From a1aab639b4edcdd69108021a2dd7df8db4155ac8 Mon Sep 17 00:00:00 2001 From: Theo Kiehn <162969167+tkiehn@users.noreply.github.com> Date: Tue, 27 Aug 2024 18:17:08 +0200 Subject: [PATCH 1/3] add parsing macros for global variables(beginning and end of all times, date_format) --- .../beginning_of_all_times_date.sql | 155 ++++++++++++++++++ macros/supporting/date_format.sql | 154 +++++++++++++++++ macros/supporting/end_of_all_times_date.sql | 152 +++++++++++++++++ 3 files changed, 461 insertions(+) create mode 100644 macros/supporting/beginning_of_all_times_date.sql create mode 100644 macros/supporting/date_format.sql create mode 100644 macros/supporting/end_of_all_times_date.sql diff --git a/macros/supporting/beginning_of_all_times_date.sql b/macros/supporting/beginning_of_all_times_date.sql new file mode 100644 index 00000000..dea1dfd8 --- /dev/null +++ b/macros/supporting/beginning_of_all_times_date.sql @@ -0,0 +1,155 @@ +{%- macro beginning_of_all_times_date() %} + + {{ return( adapter.dispatch('beginning_of_all_times_date', 'datavault4dbt')() ) }} + +{%- endmacro -%} + + +{%- macro default__beginning_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.beginning_of_all_times_date', none) -%} +{%- set beginning_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'bigquery' in global_var.keys()|map('lower') -%} + {% set beginning_of_all_times_date = global_var['bigquery'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.beginning_of_all_times_date' to a dictionary, but have not included the adapter you use (bigquery) as a key. Applying the default value.") -%} + {% endif %} + {%- set beginning_of_all_times_date = "0001-01-01" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set beginning_of_all_times_date = global_var -%} +{%- else -%} + {%- set beginning_of_all_times_date = "0001-01-01" -%} +{%- endif -%} + +{{ return(beginning_of_all_times_date) }} + +{%- endmacro -%} + + +{%- macro snowflake__beginning_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.beginning_of_all_times_date', none) -%} +{%- set beginning_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'snowflake' in global_var.keys()|map('lower') -%} + {% set beginning_of_all_times_date = global_var['snowflake'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.beginning_of_all_times_date' to a dictionary, but have not included the adapter you use (snowflake) as a key. Applying the default value.") -%} + {% endif %} + {%- set beginning_of_all_times_date = "0001-01-01" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set beginning_of_all_times_date = global_var -%} +{%- else -%} + {%- set beginning_of_all_times_date = "0001-01-01" -%} +{%- endif -%} + +{{ return(beginning_of_all_times_date) }} + +{%- endmacro -%} + + +{%- macro exasol__beginning_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.beginning_of_all_times_date', none) -%} +{%- set beginning_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'exasol' in global_var.keys()|map('lower') -%} + {% set beginning_of_all_times_date = global_var['exasol'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.beginning_of_all_times_date' to a dictionary, but have not included the adapter you use (exasol) as a key. Applying the default value.") -%} + {% endif %} + {%- set beginning_of_all_times_date = "0001-01-01" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set beginning_of_all_times_date = global_var -%} +{%- else -%} + {%- set beginning_of_all_times_date = "0001-01-01" -%} +{%- endif -%} + +{{ return(beginning_of_all_times_date) }} + +{%- endmacro -%} + + +{%- macro synapse__beginning_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.beginning_of_all_times_date', none) -%} +{%- set beginning_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'synapse' in global_var.keys()|map('lower') -%} + {% set beginning_of_all_times_date = global_var['synapse'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.beginning_of_all_times_date' to a dictionary, but have not included the adapter you use (synapse) as a key. Applying the default value.") -%} + {% endif %} + {%- set beginning_of_all_times_date = "1901-01-01" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set beginning_of_all_times_date = global_var -%} +{%- else -%} + {%- set beginning_of_all_times_date = "1901-01-01" -%} +{%- endif -%} + +{{ return(beginning_of_all_times_date) }} + +{%- endmacro -%} + + +{%- macro postgres__beginning_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.beginning_of_all_times_date', none) -%} +{%- set beginning_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'postgres' in global_var.keys()|map('lower') -%} + {% set beginning_of_all_times_date = global_var['postgres'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.beginning_of_all_times_date' to a dictionary, but have not included the adapter you use (postgres) as a key. Applying the default value.") -%} + {% endif %} + {%- set beginning_of_all_times_date = "0001-01-01" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set beginning_of_all_times_date = global_var -%} +{%- else -%} + {%- set beginning_of_all_times_date = "0001-01-01" -%} +{%- endif -%} + +{{ return(beginning_of_all_times_date) }} + +{%- endmacro -%} + + +{%- macro redshift__beginning_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.beginning_of_all_times_date', none) -%} +{%- set beginning_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'redshift' in global_var.keys()|map('lower') -%} + {% set beginning_of_all_times_date = global_var['redshift'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.beginning_of_all_times_date' to a dictionary, but have not included the adapter you use (redshift) as a key. Applying the default value.") -%} + {% endif %} + {%- set beginning_of_all_times_date = "0001-01-01" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set beginning_of_all_times_date = global_var -%} +{%- else -%} + {%- set beginning_of_all_times_date = "0001-01-01" -%} +{%- endif -%} + +{{ return(beginning_of_all_times_date) }} + +{%- endmacro -%} \ No newline at end of file diff --git a/macros/supporting/date_format.sql b/macros/supporting/date_format.sql new file mode 100644 index 00000000..b050981c --- /dev/null +++ b/macros/supporting/date_format.sql @@ -0,0 +1,154 @@ +{%- macro date_format() %} + + {{ return(adapter.dispatch('date_format', 'datavault4dbt')()) }} + +{%- endmacro -%} + + +{%- macro default__date_format() %} + +{%- set global_var = var('datavault4dbt.date_format', none) -%} +{%- set date_format = '' -%} + +{%- if global_var is mapping -%} + {%- if 'bigquery' in global_var.keys()|map('lower') -%} + {% set date_format = global_var['bigquery'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.date_format' to a dictionary, but have not included the adapter you use (bigquery) as a key. Applying the default value.") -%} + {% endif %} + {%- set date_format = "%Y-%m-%d" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set date_format = global_var -%} +{%- else -%} + {%- set date_format = "%Y-%m-%d" -%} +{%- endif -%} + +{{ return(date_format) }} + +{%- endmacro -%} + + +{%- macro snowflake__date_format() %} + +{%- set global_var = var('datavault4dbt.date_format', none) -%} +{%- set date_format = '' -%} + +{%- if global_var is mapping -%} + {%- if 'snowflake' in global_var.keys()|map('lower') -%} + {% set date_format = global_var['snowflake'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.date_format' to a dictionary, but have not included the adapter you use (snowflake) as a key. Applying the default value.") -%} + {% endif %} + {%- set date_format = "YYYY-MM-DD" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set date_format = global_var -%} +{%- else -%} + {%- set date_format = "YYYY-MM-DD" -%} +{%- endif -%} + +{{ return(date_format) }} + +{%- endmacro -%} + + +{%- macro exasol__date_format() %} + +{%- set global_var = var('datavault4dbt.date_format', none) -%} +{%- set date_format = '' -%} + +{%- if global_var is mapping -%} + {%- if 'exasol' in global_var.keys()|map('lower') -%} + {% set date_format = global_var['exasol'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.date_format' to a dictionary, but have not included the adapter you use (exasol) as a key. Applying the default value.") -%} + {% endif %} + {%- set date_format = "YYYY-mm-dd" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set date_format = global_var -%} +{%- else -%} + {%- set date_format = "YYYY-mm-dd" -%} +{%- endif -%} + +{{ return(date_format) }} + +{%- endmacro -%} + + +{%- macro synapse__date_format() %} + +{%- set global_var = var('datavault4dbt.date_format', none) -%} +{%- set date_format = '' -%} + +{%- if global_var is mapping -%} + {%- if 'synapse' in global_var.keys()|map('lower') -%} + {% set date_format = global_var['synapse'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.date_format' to a dictionary, but have not included the adapter you use (synapse) as a key. Applying the default value.") -%} + {% endif %} + {%- set date_format = "yyyy-MM-dd" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set date_format = global_var -%} +{%- else -%} + {%- set date_format = "yyyy-MM-dd" -%} +{%- endif -%} + +{{ return(date_format) }} + +{%- endmacro -%} + + +{%- macro postgres__date_format() %} + +{%- set global_var = var('datavault4dbt.date_format', none) -%} +{%- set date_format = '' -%} + +{%- if global_var is mapping -%} + {%- if 'postgres' in global_var.keys()|map('lower') -%} + {% set date_format = global_var['postgres'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.date_format' to a dictionary, but have not included the adapter you use (postgres) as a key. Applying the default value.") -%} + {% endif %} + {%- set date_format = "YYYY-MM-DD" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set date_format = global_var -%} +{%- else -%} + {%- set date_format = "YYYY-MM-DD" -%} +{%- endif -%} + +{{ return(date_format) }} + +{%- endmacro -%} + +{%- macro redshift__date_format() %} + +{%- set global_var = var('datavault4dbt.date_format', none) -%} +{%- set date_format = '' -%} + +{%- if global_var is mapping -%} + {%- if 'redshift' in global_var.keys()|map('lower') -%} + {% set date_format = global_var['redshift'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.date_format' to a dictionary, but have not included the adapter you use (redshift) as a key. Applying the default value.") -%} + {% endif %} + {%- set date_format = "YYYY-MM-DD" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set date_format = global_var -%} +{%- else -%} + {%- set date_format = "YYYY-MM-DD" -%} +{%- endif -%} + +{{ return(date_format) }} + +{%- endmacro -%} \ No newline at end of file diff --git a/macros/supporting/end_of_all_times_date.sql b/macros/supporting/end_of_all_times_date.sql new file mode 100644 index 00000000..43750f74 --- /dev/null +++ b/macros/supporting/end_of_all_times_date.sql @@ -0,0 +1,152 @@ +{%- macro end_of_all_times_date() %} + + {{ return(adapter.dispatch('end_of_all_times_date', 'datavault4dbt')()) }} + +{%- endmacro -%} + + +{%- macro default__end_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.end_of_all_times_date', none) -%} +{%- set end_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'bigquery' in global_var.keys()|map('lower') -%} + {% set end_of_all_times_date = global_var['bigquery'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.end_of_all_times_date' to a dictionary, but have not included the adapter you use (bigquery) as a key. Applying the default value.") -%} + {% endif %} + {%- set end_of_all_times_date = "8888-12-31" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set end_of_all_times_date = global_var -%} +{%- else -%} + {%- set end_of_all_times_date = "8888-12-31" -%} +{%- endif -%} + +{{ return(end_of_all_times_date) }} + +{%- endmacro -%} + + +{%- macro snowflake__end_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.end_of_all_times_date', none) -%} +{%- set end_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'snowflake' in global_var.keys()|map('lower') -%} + {% set end_of_all_times_date = global_var['snowflake'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.end_of_all_times_date' to a dictionary, but have not included the adapter you use (snowflake) as a key. Applying the default value.") -%} + {% endif %} + {%- set end_of_all_times_date = "8888-12-31" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set end_of_all_times_date = global_var -%} +{%- else -%} + {%- set end_of_all_times_date = "8888-12-31" -%} +{%- endif -%} + +{{ return(end_of_all_times_date) }} + +{%- endmacro -%} + + +{%- macro exasol__end_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.end_of_all_times_date', none) -%} +{%- set end_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'exasol' in global_var.keys()|map('lower') -%} + {% set end_of_all_times_date = global_var['exasol'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.end_of_all_times_date' to a dictionary, but have not included the adapter you use (exasol) as a key. Applying the default value.") -%} + {% endif %} + {%- set end_of_all_times_date = "8888-12-31" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set end_of_all_times_date = global_var -%} +{%- else -%} + {%- set end_of_all_times_date = "8888-12-31" -%} +{%- endif -%} + +{{ return(end_of_all_times_date) }} + +{%- endmacro -%} + + +{%- macro postgres__end_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.end_of_all_times_date', none) -%} +{%- set end_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'postgres' in global_var.keys()|map('lower') -%} + {% set end_of_all_times_date = global_var['postgres'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.end_of_all_times_date' to a dictionary, but have not included the adapter you use (postgres) as a key. Applying the default value.") -%} + {% endif %} + {%- set end_of_all_times_date = "8888-12-31" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set end_of_all_times_date = global_var -%} +{%- else -%} + {%- set end_of_all_times_date = "8888-12-31" -%} +{%- endif -%} + +{{ return(end_of_all_times_date) }} +{%- endmacro -%} + + +{%- macro redshift__end_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.end_of_all_times_date', none) -%} +{%- set end_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'redshift' in global_var.keys()|map('lower') -%} + {% set end_of_all_times_date = global_var['redshift'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.end_of_all_times_date' to a dictionary, but have not included the adapter you use (redshift) as a key. Applying the default value.") -%} + {% endif %} + {%- set end_of_all_times_date = "8888-12-31" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set end_of_all_times_date = global_var -%} +{%- else -%} + {%- set end_of_all_times_date = "8888-12-31" -%} +{%- endif -%} + +{{ return(end_of_all_times_date) }} +{%- endmacro -%} + + +{%- macro synapse__end_of_all_times_date() %} + +{%- set global_var = var('datavault4dbt.end_of_all_times_date', none) -%} +{%- set end_of_all_times_date = '' -%} + +{%- if global_var is mapping -%} + {%- if 'synapse' in global_var.keys()|map('lower') -%} + {% set end_of_all_times_date = global_var['synapse'] %} + {%- else -%} + {%- if execute -%} + {%- do exceptions.warn("Warning: You have set the global variable 'datavault4dbt.end_of_all_times_date' to a dictionary, but have not included the adapter you use (synapse) as a key. Applying the default value.") -%} + {% endif %} + {%- set end_of_all_times_date = "8888-12-31" -%} + {% endif %} +{%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%} + {%- set end_of_all_times_date = global_var -%} +{%- else -%} + {%- set end_of_all_times_date = "8888-12-31" -%} +{%- endif -%} + +{{ return(end_of_all_times_date) }} +{%- endmacro -%} \ No newline at end of file From c00f7a4243a18ad33b460f2d390419aadaf2242b Mon Sep 17 00:00:00 2001 From: Theo Kiehn <162969167+tkiehn@users.noreply.github.com> Date: Tue, 27 Aug 2024 18:19:02 +0200 Subject: [PATCH 2/3] change ghostrecords to use the parsing-macros and align synapse ghostrecords --- .../supporting/ghost_record_per_datatype.sql | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/macros/supporting/ghost_record_per_datatype.sql b/macros/supporting/ghost_record_per_datatype.sql index 3f67a249..2ae161b6 100644 --- a/macros/supporting/ghost_record_per_datatype.sql +++ b/macros/supporting/ghost_record_per_datatype.sql @@ -18,9 +18,9 @@ {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%} {%- set timestamp_format = datavault4dbt.timestamp_format() -%} -{%- set beginning_of_all_times_date = var('datavault4dbt.beginning_of_all_times_date', '0001-01-01') -%} -{%- set end_of_all_times_date = var('datavault4dbt.end_of_all_times_date', '8888-12-31') -%} -{%- set date_format = var('datavault4dbt.date_format', '%Y-%m-%d') -%} +{%- set beginning_of_all_times_date = datavault4dbt.beginning_of_all_times_date() -%} +{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%} +{%- set date_format = datavault4dbt.date_format() -%} {%- set datatype = datatype | string | upper | trim -%} @@ -58,9 +58,9 @@ {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%} {%- set timestamp_format = datavault4dbt.timestamp_format() -%} -{%- set beginning_of_all_times_date = var('datavault4dbt.beginning_of_all_times_date', '0001-01-01') -%} -{%- set end_of_all_times_date = var('datavault4dbt.end_of_all_times_date', '8888-12-31') -%} -{%- set date_format = var('datavault4dbt.date_format', 'YYYY-mm-dd') -%} +{%- set beginning_of_all_times_date = datavault4dbt.beginning_of_all_times_date() -%} +{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%} +{%- set date_format = datavault4dbt.date_format() -%} {%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%} {%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%} @@ -147,9 +147,9 @@ {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%} {%- set timestamp_format = datavault4dbt.timestamp_format() -%} -{%- set beginning_of_all_times_date = var('datavault4dbt.beginning_of_all_times_date', '0001-01-01') -%} -{%- set end_of_all_times_date = var('datavault4dbt.end_of_all_times_date', '8888-12-31') -%} -{%- set date_format = var('datavault4dbt.date_format', 'YYYY-mm-dd') -%} +{%- set beginning_of_all_times_date = datavault4dbt.beginning_of_all_times_date() -%} +{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%} +{%- set date_format = datavault4dbt.date_format() -%} {%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%} {%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%} @@ -220,22 +220,30 @@ {%- macro synapse__ghost_record_per_datatype(column_name, datatype, ghost_record_type, col_size, alias) -%} +{%- set beginning_of_all_times = datavault4dbt.beginning_of_all_times() -%} +{%- set end_of_all_times = datavault4dbt.end_of_all_times() -%} +{%- set timestamp_format = datavault4dbt.timestamp_format() -%} + +{%- set beginning_of_all_times_date = datavault4dbt.beginning_of_all_times_date() -%} +{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%} +{%- set date_format = datavault4dbt.date_format() -%} {%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%} {%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%} {%- set unknown_value_alt__STRING = var('datavault4dbt.unknown_value_alt__STRING', 'u') -%} {%- set error_value_alt__STRING = var('datavault4dbt.error_value_alt__STRING', 'e') -%} -{%- set format_date = var('datavault4dbt.format_date', 'YYYY-mm-dd') -%} + {%- set hash = datavault4dbt.hash_method() -%} {%- set hash_default_values = datavault4dbt.hash_default_values(hash_function=hash) -%} {%- set hash_alg= hash_default_values['hash_alg'] -%} + {%- set unknown_value__HASHTYPE = hash_default_values['unknown_key'] -%} -{%- set error_value__HASHTYPE = hash_default_values['error_key'] -%} +{%- set error_value__HASHTYPE = hash_default_values['error_key'] -%} {%- set datatype = datatype | string | upper | trim -%} {%- if ghost_record_type == 'unknown' -%} - {%- if datatype == 'DATETIME' or datatype == 'DATETIME2' %} {{- datavault4dbt.string_to_timestamp( timestamp_format , beginning_of_all_times) }} as "{{ column_name }}" + {%- if datatype in ['DATETIME', 'DATETIME2', 'DATETIMEOFFSET'] %} CONVERT({{ datatype }}, {{- datavault4dbt.string_to_timestamp( timestamp_format , beginning_of_all_times) }}) as "{{ column_name }}" {%- elif 'CHAR' in datatype -%} {%- if col_size is not none -%} {%- if (col_size | int) == -1 -%} @@ -264,7 +272,7 @@ {%- elif ghost_record_type == 'error' -%} - {%- if datatype == 'DATETIME' or datatype == 'DATETIME2' %} {{- datavault4dbt.string_to_timestamp( timestamp_format , end_of_all_times) }} as "{{ column_name }}" + {%- if datatype in ['DATETIME', 'DATETIME2', 'DATETIMEOFFSET'] %} CONVERT({{ datatype }}, {{- datavault4dbt.string_to_timestamp( timestamp_format , end_of_all_times) }}) as "{{ column_name }}" {%- elif 'CHAR' in datatype -%} {%- if col_size is not none -%} {%- if (col_size | int) == -1 -%} @@ -308,9 +316,9 @@ {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%} {%- set timestamp_format = datavault4dbt.timestamp_format() -%} -{%- set beginning_of_all_times_date = var('datavault4dbt.beginning_of_all_times_date', '0001-01-01') -%} -{%- set end_of_all_times_date = var('datavault4dbt.end_of_all_times_date', '8888-12-31') -%} -{%- set date_format = var('datavault4dbt.date_format', 'YYYY-mm-dd') -%} +{%- set beginning_of_all_times_date = datavault4dbt.beginning_of_all_times_date() -%} +{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%} +{%- set date_format = datavault4dbt.date_format() -%} {%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%} {%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%} @@ -350,9 +358,9 @@ {%- set end_of_all_times = datavault4dbt.end_of_all_times() -%} {%- set timestamp_format = datavault4dbt.timestamp_format() -%} -{%- set beginning_of_all_times_date = var('datavault4dbt.beginning_of_all_times_date', '0001-01-01') -%} -{%- set end_of_all_times_date = var('datavault4dbt.end_of_all_times_date', '8888-12-31') -%} -{%- set date_format = var('datavault4dbt.date_format', 'YYYY-mm-dd') -%} +{%- set beginning_of_all_times_date = datavault4dbt.beginning_of_all_times_date() -%} +{%- set end_of_all_times_date = datavault4dbt.end_of_all_times_date() -%} +{%- set date_format = datavault4dbt.date_format() -%} {%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%} {%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%} From 9351066ace80b14f0cd9401e021465ce2e926940 Mon Sep 17 00:00:00 2001 From: Theo Kiehn <162969167+tkiehn@users.noreply.github.com> Date: Wed, 4 Sep 2024 09:45:57 +0200 Subject: [PATCH 3/3] Update dbt_project.yml, reactivate variables for ghost records date values --- dbt_project.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 52ad4fc0..ebaf45bb 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -50,9 +50,9 @@ vars: datavault4dbt.end_of_all_times: {"bigquery":"8888-12-31T23-59-59","snowflake":"8888-12-31T23:59:59", "exasol": "8888-12-31 23:59:59", "postgres": "8888-12-31 23:59:59", "redshift": "8888-12-31 23:59:59", "synapse": "8888-12-31T23:59:59"} datavault4dbt.timestamp_format: {"bigquery":"%Y-%m-%dT%H-%M-%S","snowflake":"YYYY-MM-DDTHH24:MI:SS", "exasol": "YYYY-mm-dd HH:MI:SS", "postgres": "YYYY-MM-DD HH24:MI:SS", "redshift": "YYYY-MM-DD HH24:MI:SS", "synapse": 126} - #datavault4dbt.beginning_of_all_times_date: {"bigquery":"0001-01-01","snowflake":"0001-01-01", "exasol": "0001-01-01", "postgres": "0001-01-01", "redshift": "0001-01-01", "synapse": "1901-01-01"} - #datavault4dbt.end_of_all_times_date: {"bigquery":"8888-12-31","snowflake":"8888-12-31", "exasol": "8888-12-31", "postgres": "8888-12-31", "redshift": "8888-12-31", "synapse": "8888-12-31"} - #datavault4dbt.date_format: {"bigquery":"%Y-%m-%d","snowflake":"YYYY-MM-DD", "exasol": "YYYY-mm-dd", "postgres": "YYYY-MM-DD", "redshift": "YYYY-MM-DD", "synapse": "yyyy-MM-dd"} + datavault4dbt.beginning_of_all_times_date: {"bigquery":"0001-01-01","snowflake":"0001-01-01", "exasol": "0001-01-01", "postgres": "0001-01-01", "redshift": "0001-01-01", "synapse": "1901-01-01"} + datavault4dbt.end_of_all_times_date: {"bigquery":"8888-12-31","snowflake":"8888-12-31", "exasol": "8888-12-31", "postgres": "8888-12-31", "redshift": "8888-12-31", "synapse": "8888-12-31"} + datavault4dbt.date_format: {"bigquery":"%Y-%m-%d","snowflake":"YYYY-MM-DD", "exasol": "YYYY-mm-dd", "postgres": "YYYY-MM-DD", "redshift": "YYYY-MM-DD", "synapse": "yyyy-MM-dd"} datavault4dbt.default_unknown_rsrc: 'SYSTEM' datavault4dbt.default_error_rsrc: 'ERROR'