From be7ce069bb5c7ffcce07f25c68aeeef9966a54fd Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Wed, 8 Nov 2023 10:40:47 -0500 Subject: [PATCH 1/6] Add allow_non_incremental_definition option to BigQuery materialized views --- dbt/adapters/bigquery/impl.py | 1 + dbt/adapters/bigquery/relation_configs/_options.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 7d9b003b8..6ee9eb55f 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -92,6 +92,7 @@ class BigqueryConfig(AdapterConfig): enable_refresh: Optional[bool] = None refresh_interval_minutes: Optional[int] = None max_staleness: Optional[str] = None + allow_non_incremental_definition: Optional[bool] = None class BigQueryAdapter(BaseAdapter): diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index 72f9d73e6..4c373b06c 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -21,6 +21,7 @@ class BigQueryOptionsConfig(BigQueryBaseRelationConfig): refresh_interval_minutes: Optional[float] = 30 expiration_timestamp: Optional[datetime] = None max_staleness: Optional[str] = None + allow_non_incremental_definition: Optional[bool] = None kms_key_name: Optional[str] = None description: Optional[str] = None labels: Optional[Dict[str, str]] = None @@ -57,6 +58,7 @@ def array(x): "refresh_interval_minutes": numeric, "expiration_timestamp": interval, "max_staleness": interval, + "allow_non_incremental_definition": boolean, "kms_key_name": string, "description": escaped_string, "labels": array, @@ -84,6 +86,7 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "BigQueryOptionsConfig": "refresh_interval_minutes": float_setting, "expiration_timestamp": None, "max_staleness": None, + "allow_non_incremental_definition": bool_setting, "kms_key_name": None, "description": None, "labels": None, @@ -114,6 +117,7 @@ def parse_model_node(cls, model_node: ModelNode) -> Dict[str, Any]: "refresh_interval_minutes", "expiration_timestamp", "max_staleness", + "allow_non_incremental_definition", "kms_key_name", "description", "labels", @@ -137,6 +141,7 @@ def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]: "refresh_interval_minutes": table.mview_refresh_interval.seconds / 60, "expiration_timestamp": table.expires, "max_staleness": None, + "allow_non_incremental_definition": table._properties.get("materializedView", {}).get("allowNonIncrementalDefinition"), "description": table.description, } From 85e3426019dbf804b049d5328e2ab37ab07b450c Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Wed, 8 Nov 2023 19:05:15 -0500 Subject: [PATCH 2/6] Add changelog --- .changes/unreleased/Features-20231108-140752.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20231108-140752.yaml diff --git a/.changes/unreleased/Features-20231108-140752.yaml b/.changes/unreleased/Features-20231108-140752.yaml new file mode 100644 index 000000000..2a9a41a07 --- /dev/null +++ b/.changes/unreleased/Features-20231108-140752.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support allow_non_incremental_definition option in BigQuery materialized views +time: 2023-11-08T14:07:52.28972-05:00 +custom: + Author: bnaul + Issue: "672" From c976c81c229f58892651cc1ae9c7f5135fab2379 Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Wed, 8 Nov 2023 19:18:02 -0500 Subject: [PATCH 3/6] Fix default value for flag --- dbt/adapters/bigquery/relation_configs/_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index 4c373b06c..e7a434da7 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -21,7 +21,7 @@ class BigQueryOptionsConfig(BigQueryBaseRelationConfig): refresh_interval_minutes: Optional[float] = 30 expiration_timestamp: Optional[datetime] = None max_staleness: Optional[str] = None - allow_non_incremental_definition: Optional[bool] = None + allow_non_incremental_definition: Optional[bool] = False kms_key_name: Optional[str] = None description: Optional[str] = None labels: Optional[Dict[str, str]] = None From b3e9eea12d0650ed72186e2e1bdaf13a9ae9774e Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Wed, 8 Nov 2023 19:18:07 -0500 Subject: [PATCH 4/6] Set flag in test --- tests/functional/adapter/materialized_view_tests/_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/materialized_view_tests/_files.py b/tests/functional/adapter/materialized_view_tests/_files.py index 86714036a..9f8fe14d3 100644 --- a/tests/functional/adapter/materialized_view_tests/_files.py +++ b/tests/functional/adapter/materialized_view_tests/_files.py @@ -36,7 +36,8 @@ cluster_by=["id", "value"], enable_refresh=True, refresh_interval_minutes=60, - max_staleness="INTERVAL 45 MINUTE" + max_staleness="INTERVAL 45 MINUTE", + allow_non_incremental_definition=True ) }} select id, From 97332e6c5451d5c35ca1dbab5c1fd5ad02ac6bc9 Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Thu, 9 Nov 2023 20:33:32 -0500 Subject: [PATCH 5/6] Recreate if allow_non_incremental_definition changed --- dbt/adapters/bigquery/relation.py | 10 +++++++++- dbt/adapters/bigquery/relation_configs/_options.py | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index c14dba238..c4977be8d 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -78,8 +78,16 @@ def materialized_view_config_changeset( new_materialized_view = cls.materialized_view_from_model_node(runtime_config.model) if new_materialized_view.options != existing_materialized_view.options: + # allow_non_incremental_definition cannot be changed via ALTER, must recreate + if ( + new_materialized_view.options.allow_non_incremental_definition + == existing_materialized_view.options.allow_non_incremental_definition + ): + action = RelationConfigChangeAction.alter + else: + action = RelationConfigChangeAction.create config_change_collection.options = BigQueryOptionsConfigChange( - action=RelationConfigChangeAction.alter, + action=action, context=new_materialized_view.options, ) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index e7a434da7..a36babcf2 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -8,6 +8,7 @@ from dbt.adapters.bigquery.relation_configs._base import BigQueryBaseRelationConfig from dbt.adapters.bigquery.utility import bool_setting, float_setting, sql_escape +from dbt.adapters.relation_configs import RelationConfigChangeAction @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -160,4 +161,4 @@ class BigQueryOptionsConfigChange(RelationConfigChange): @property def requires_full_refresh(self) -> bool: - return False + return self.action != RelationConfigChangeAction.alter From e75ccacab9a740448af7f917e2fb4b7da6282f19 Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Sun, 12 Nov 2023 08:09:33 -0500 Subject: [PATCH 6/6] Parse maxStaleness --- dbt/adapters/bigquery/relation_configs/_options.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index a36babcf2..924e1b98d 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -141,7 +141,11 @@ def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]: "enable_refresh": table.mview_enable_refresh, "refresh_interval_minutes": table.mview_refresh_interval.seconds / 60, "expiration_timestamp": table.expires, - "max_staleness": None, + "max_staleness": ( + f"INTERVAL '{table._properties.get('maxStaleness')}' YEAR TO SECOND" + if table._properties.get('maxStaleness') + else None + ), "allow_non_incremental_definition": table._properties.get("materializedView", {}).get("allowNonIncrementalDefinition"), "description": table.description, }