feat: adding pyspark_job properties and archive_uris options

dbt-labs · Dec 7, 2023 · b353877 · b353877
1 parent c489332
commit b353877
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/dbt/adapters/bigquery/python_submissions.py b/dbt/adapters/bigquery/python_submissions.py
@@ -103,6 +103,14 @@ def _submit_dataproc_job(self) -> dataproc_v1.types.jobs.Job:
             "placement": {"cluster_name": self._get_cluster_name()},
             "pyspark_job": {
                 "main_python_file_uri": self.gcs_location,
+                "archive_uris": self.parsed_model["config"].get(
+                    "dataproc_archive_uris",
+                    [],
+                ),
+                "properties": self.parsed_model["config"].get(
+                    "dataproc_properties",
+                    {},
+                ),
             },
         }
         operation = self.job_client.submit_job_as_operation(  # type: ignore