refactor to dataproc_pyspark_job to feed any parameter to pyspark_job

dbt-labs · Dec 7, 2023 · 18bde3a · 18bde3a
1 parent b353877
commit 18bde3a
Showing 1 changed file with 3 additions and 9 deletions.
diff --git a/dbt/adapters/bigquery/python_submissions.py b/dbt/adapters/bigquery/python_submissions.py
@@ -102,17 +102,11 @@ def _submit_dataproc_job(self) -> dataproc_v1.types.jobs.Job:
         job = {
             "placement": {"cluster_name": self._get_cluster_name()},
             "pyspark_job": {
-                "main_python_file_uri": self.gcs_location,
-                "archive_uris": self.parsed_model["config"].get(
-                    "dataproc_archive_uris",
-                    [],
-                ),
-                "properties": self.parsed_model["config"].get(
-                    "dataproc_properties",
-                    {},
-                ),
+                "main_python_file_uri": self.gcs_location
             },
         }
+        job["pyspark_job"].update(self.parsed_model["config"].get("dataproc_pyspark_job", {}))
+
         operation = self.job_client.submit_job_as_operation(  # type: ignore
             request={
                 "project_id": self.credential.execution_project,