New PR for DBT 1.8.5 (#139)

CHANGELOG.md

@@ -3,6 +3,30 @@
 - "Breaking changes" listed under a version may require action from end users.
 
 
+### 1.8.3
+
+#### Features:
+- Update base adapter references as part of decoupling migration to support dbt core 1.8.3 https://github.com/vertica/dbt-vertica/issues/131
+- Support --empty tests
+- Support all types for unit testing in dbt-vertica, expand coverage of safe_cast macro https://github.com/dbt-labs/dbt-core/discussions/9798
+- Add new workflow for internal patch releases
+- Support limiting get_catalog by object name
+- add --empty value to jinja context as flags.EMPTY
+
+#### Fixes:
+- Unit-test check tests
+  - test_types.BaseUnitTestingTypes
+  - test_case_insensitivity.BaseUnitTestCaseInsensivity
+  - test_invalid_input.BaseUnitTestInvalidInput
+- Flags.EMPTY check tests
+  - BaseTestEmpty
+- Support limiting get_catalog check tests
+  - BaseChangeRelationTypeValidator
+- warning on unit_test config in dbt_project.yaml file
+- Additional tests
+  - TestBaseContext
+  - BaseIncrementalOnSchemaChangeSetup
+
 
 ### 1.7.13
 

README.md

@@ -13,8 +13,8 @@ dbt-vertica has been developed using the following software and versions:
 * Vertica Server 23.4.0-0
 * Python 3.11
 * vertica-python client 1.3.1
-* dbt-core 1.6.0
-* dbt-tests-adapter 1.6.0
+* dbt-core 1.8.3
+* dbt-tests-adapter 1.8.0
 
 ## Supported Features
 ### dbt Core Features
@@ -34,6 +34,7 @@ Below is a table for what features the current Vertica adapter supports for dbt.
 | Tests                                             | Yes         |
 | Documentation                                     | Yes         |
 | External Tables                                   | Untested    |
+| Unit Testing                                      | Yes         |
 * **Yes** - Supported, and tests pass.
 * **No** - Not supported or implemented.
 * **Untested** - May support out of the box, though hasn't been tested.
@@ -125,4 +126,8 @@ Run tests via:
     # run an individual test 
     pytest tests/functional/adapter/test_basic.py
 
+Run Unit test via:
+
+    dbt test --select /{foldername}/{unit_test_file}
+
 

dbt/adapters/vertica/__init__.py

@@ -18,6 +18,7 @@
 from dbt.adapters.vertica.impl import verticaAdapter
 from dbt.adapters.vertica.column import VerticaColumn
 
+
 from dbt.adapters.base import AdapterPlugin
 from dbt.include import vertica
 

dbt/adapters/vertica/__version__.py

@@ -14,4 +14,4 @@
 
 
 
-version = "1.7.13"
+version = "1.8.3"

dbt/adapters/vertica/column.py

@@ -18,6 +18,7 @@
 from typing import Dict, ClassVar
 
 from dbt.adapters.base.column import Column
+#from dbt.common.exceptions import DbtRuntimeError
 
 @dataclass(init=False)
 class VerticaColumn(Column):

dbt/adapters/vertica/connections.py

@@ -19,14 +19,14 @@
 from typing import Any, List, Optional, Tuple, Union
 
 import agate
-import dbt.clients.agate_helper
-import dbt.exceptions
+import dbt_common.clients.agate_helper
+import dbt_common.exceptions
+import dbt.adapters.sql.connections
 import requests
 import vertica_python
-from dbt.adapters.base import Credentials
-from dbt.adapters.sql import SQLConnectionManager
-from dbt.contracts.connection import AdapterResponse
-from dbt.events import AdapterLogger
+from dbt.adapters.contracts.connection import Credentials, AdapterResponse
+from dbt.adapters.sql.connections import SQLConnectionManager
+from dbt.adapters.events.logging import AdapterLogger
 
 logger = AdapterLogger("vertica")
 
@@ -91,7 +91,7 @@ def open(cls, connection):
                 'database': credentials.database,
                 'connection_timeout': credentials.timeout,
                 'connection_load_balance':credentials.connection_load_balance,
-                'session_label': f'dbt_{credentials.username}',
+                'session_label': credentials.username,
                 'retries': credentials.retries,
                 'oauth_access_token': credentials.oauth_access_token,
                 'autocommit': credentials.autocommit,
@@ -132,7 +132,7 @@ def connect():
             logger.debug(f':P Error connecting to database: {exc}')
             connection.state = 'fail'
             connection.handle = None
-            raise dbt.exceptions.DbtFailedToConnectErroe(str(exc))
+            raise dbt.adapters.exceptions.connection.FailedToConnectError(str(exc))
 
         # This is here mainly to support dbt-integration-tests.
         # It globally enables WITH materialization for every connection dbt
@@ -153,7 +153,7 @@ def connect():
 
         retryable_exceptions = [
         Exception,
-        dbt.exceptions.FailedToConnectError
+        dbt.adapters.exceptions.connection.FailedToConnectError
         ]
 
         return cls.retry_connection(
@@ -199,11 +199,11 @@ def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> agate.Tabl
                 if isinstance(check, vertica_python.vertica.messages.ErrorResponse):
                     logger.debug(f'Cursor message is: {check}')
                     self.release()
-                    raise dbt.exceptions.DbtDatabaseError(str(check))
+                    raise dbt_common.exceptions.DbtDatabaseError(str(check))
 
             data = cls.process_results(column_names, rows)
 
-        return dbt.clients.agate_helper.table_from_data_flat(data, column_names)
+        return dbt_common.clients.agate_helper.table_from_data_flat(data, column_names)
 
     def execute(
         self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None
@@ -214,13 +214,13 @@ def execute(
         if fetch:
             table = self.get_result_from_cursor(cursor,limit)
         else:
-            table = dbt.clients.agate_helper.empty_table()
+            table = dbt_common.clients.agate_helper.empty_table()
             while cursor.nextset():
                 check = cursor._message
                 if isinstance(check, vertica_python.vertica.messages.ErrorResponse):
                     logger.debug(f'Cursor message is: {check}')
                     self.release()
-                    raise dbt.exceptions.DbtDatabaseError(str(check))
+                    raise dbt_common.exceptions.DbtDatabaseError(str(check))
         return response, table
 
     @contextmanager
@@ -230,11 +230,11 @@ def exception_handler(self, sql):
         except vertica_python.DatabaseError as exc:
             logger.debug(f':P Database error: {exc}')
             self.release()
-            raise dbt.exceptions.DbtDatabaseError(str(exc))
+            raise dbt_common.exceptions.DbtDatabaseError(str(exc))
         except Exception as exc:
             logger.debug(f':P Error: {exc}')
             self.release()
-            raise dbt.exceptions.DbtRuntimeError(str(exc))
+            raise dbt_common.exceptions.DbtRuntimeError(str(exc))
 
     @classmethod
     def data_type_code_to_name(cls, type_code: Union[int, str]) -> str:

dbt/adapters/vertica/impl.py

@@ -13,17 +13,20 @@
 # limitations under the License.
 
 
-from dbt.adapters.sql import SQLAdapter
+from dbt.adapters.sql.impl import SQLAdapter
 from dbt.adapters.vertica import verticaConnectionManager
 #from dbt.adapters.vertica import VerticaRelation
 from dbt.adapters.vertica.column import VerticaColumn
 from typing import Optional, List, Union, Dict
 
+
 from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability
 from dbt.adapters.base import available
-from dbt.exceptions import (
+from dbt_common.exceptions import (
 
-    DbtRuntimeError
+    DbtRuntimeError,
+    CompilationError,
+    DbtDatabaseError
 )
 
 import agate
@@ -32,7 +35,8 @@
 from dbt.adapters.sql import SQLAdapter  # type: ignore
 
 from dbt.adapters.base.impl import AdapterConfig,ConstraintSupport
-from dbt.contracts.graph.nodes import ConstraintType
+from dbt_common.contracts.constraints import ConstraintType
+
 
 @dataclass
 class VerticaConfig(AdapterConfig):

dbt/include/vertica/macros/adapters/catalog.sql

@@ -159,4 +159,4 @@
             {%- if not loop.last %} or {% endif -%}
         {%- endfor -%}
     )
-{%- endmacro %}
+{%- endmacro %}

dbt/include/vertica/macros/adapters/columns.sql

@@ -64,6 +64,7 @@
         , ordinal_position
         from v_catalog.columns
         where  table_name = '{{ relation.identifier }}'
+        and table_schema = '{{ relation.schema }}'
         union all
         select
         column_name
@@ -74,6 +75,7 @@
         , ordinal_position
         from v_catalog.view_columns
         where table_name = '{{ relation.identifier }}'
+        and table_schema = '{{ relation.schema }}'
     ) t
     order by ordinal_position
   {% endcall %}

dbt/include/vertica/macros/adapters/relation.sql

@@ -3,7 +3,7 @@
     {% do return(base_relation.incorporate(
                                   path={
                                     "identifier": tmp_identifier,
-                                    "schema": none,
+                                    "schema": 'v_temp_schema',
                                     "database": none
                                   })) -%}
 {% endmacro %}
@@ -27,4 +27,6 @@
   No need to implement drop_relation(). Syntax supported by default. 
   No need to implement drop_relation_if_exists(). Syntax supported by default.
   No need to implement get_or_create_relation(). Syntax supported by default.
-#}
+#}
+
+{%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}

dbt/include/vertica/macros/materializations/tests/unit.sql

@@ -0,0 +1,33 @@
+{%- materialization unit, default -%}
+
+  {% set relations = [] %}
+
+  {% set expected_rows = config.get('expected_rows') %}
+  {% set expected_sql = config.get('expected_sql') %}
+  {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %}
+
+  {%- set target_relation = this.incorporate(type='table') -%}
+  {%- set temp_relation = make_temp_relation(target_relation)-%}
+  {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}
+  {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}
+  {%- set column_name_to_data_types = {} -%}
+  {%- for column in columns_in_relation -%}
+  {%-   do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}
+  {%- endfor -%}
+
+  {% if not expected_sql %}
+  {%   set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %}
+  {% endif %}
+  {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %}
+
+  {% call statement('main', fetch_result=True) -%}
+
+    {{ unit_test_sql }}
+
+  {%- endcall %}
+
+  {% do adapter.drop_relation(temp_relation) %}
+
+  {{ return({'relations': relations}) }}
+
+{%- endmaterialization -%}

dbt/include/vertica/macros/unit_test_sql/get_fixture_sql.sql

@@ -0,0 +1,104 @@
+{% macro get_fixture_sql(rows, column_name_to_data_types) %}
+-- Fixture for {{ model.name }}
+{% set default_row = {} %}
+
+{%- if not column_name_to_data_types -%}
+{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}
+{%-   set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}
+{%-   set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}
+
+{%-   set column_name_to_data_types = {} -%}
+{%-   for column in columns_in_relation -%}
+{#-- This needs to be a case-insensitive comparison --#}
+{%-     do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}
+{%-   endfor -%}
+{%- endif -%}
+
+{%- if not column_name_to_data_types -%}
+    {{ exceptions.raise_compiler_error("Not able to get columns for unit test '" ~ model.name ~ "' from relation " ~ this ~ " because the relation doesn't exist") }}
+{%- endif -%}
+
+{%- for column_name, column_type in column_name_to_data_types.items() -%}
+    {%- do default_row.update({column_name: (safe_cast("null", column_type) | trim )}) -%}
+{%- endfor -%}
+
+{{ validate_fixture_rows(rows, row_number) }}
+
+{%- for row in rows -%}
+{%-   set formatted_row = format_row(row, column_name_to_data_types) -%}
+{%-   set default_row_copy = default_row.copy() -%}
+{%-   do default_row_copy.update(formatted_row) -%}
+select
+{%-   for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}
+{%-   endfor %}
+{%-   if not loop.last %}
+union all
+{%    endif %}
+{%- endfor -%}
+
+{%- if (rows | length) == 0 -%}
+    select
+    {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}
+    {%- endfor %}
+    limit 0
+{%- endif -%}
+{% endmacro %}
+
+
+{% macro get_expected_sql(rows, column_name_to_data_types) %}
+
+{%- if (rows | length) == 0 -%}
+    select * from dbt_internal_unit_test_actual
+    limit 0
+{%- else -%}
+{%- for row in rows -%}
+{%- set formatted_row = format_row(row, column_name_to_data_types) -%}
+select
+{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}
+{%- endfor %}
+{%- if not loop.last %}
+union all
+{% endif %}
+{%- endfor -%}
+{%- endif -%}
+
+{% endmacro %}
+
+{%- macro format_row(row, column_name_to_data_types) -%}
+    {#-- generate case-insensitive formatted row --#}
+    {% set formatted_row = {} %}
+    {%- for column_name, column_value in row.items() -%}
+        {% set column_name = column_name|lower %}
+
+        {%- if column_name not in column_name_to_data_types %}
+            {#-- if user-provided row contains column name that relation does not contain, raise an error --#}
+            {% set fixture_name = "expected output" if model.resource_type == 'unit_test' else ("'" ~ model.name ~ "'") %}
+            {{ exceptions.raise_compiler_error(
+                "Invalid column name: '" ~ column_name ~ "' in unit test fixture for " ~ fixture_name ~ "."
+                "\nAccepted columns for " ~ fixture_name ~ " are: " ~ (column_name_to_data_types.keys()|list)
+            ) }}
+        {%- endif -%}
+
+        {%- set column_type = column_name_to_data_types[column_name] %}
+
+        {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}
+        {%- set column_value_clean = column_value -%}
+        {%- if column_value is string -%}
+            {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}
+        {%- elif column_value is none -%}
+            {%- set column_value_clean = 'null' -%}
+        {%- endif -%}
+
+        {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}
+        {%- do formatted_row.update(row_update) -%}
+    {%- endfor -%}
+    {{ return(formatted_row) }}
+{%- endmacro -%}
+
+{%- macro validate_fixture_rows(rows, row_number) -%}
+  {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}
+{%- endmacro -%}
+
+{%- macro default__validate_fixture_rows(rows, row_number) -%}
+  {# This is an abstract method for adapter overrides as needed #}
+{%- endmacro -%}

dbt/include/vertica/macros/utils/cast.sql

@@ -0,0 +1,7 @@
+{% macro cast(field, type) %}
+  {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}
+{% endmacro %}
+
+{% macro vertica__cast(field, type) %}
+    cast({{field}} as {{type}})
+{% endmacro %}