Merge pull request #404 from dbt-labs/monorepo/migrate-dbt-spark

ADAP-1132: Migrate dbt-spark into dbt-adapters

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,12 +1,35 @@
-resolves #<issue-number>
+resolves #
+[docs](https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose) dbt-labs/docs.getdbt.com/#
 
-# Description
+<!---
+  Include the number of the issue addressed by this PR above if applicable.
+  PRs for code changes without an associated issue *will not be merged*.
+  See CONTRIBUTING.md for more information.
 
-<!-- What approach was taken to resolve the issue and why?  -->
+  Include the number of the docs issue that was opened for this PR. If
+  this change has no user-facing implications, "N/A" suffices instead. New
+  docs tickets can be created by clicking the link above or by going to
+  https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose.
+-->
 
-## Checklist
+### Problem
 
-- [ ] This PR considers the [contributing guide](https://github.com/dbt-labs/dbt-adapters#contributing)
-- [ ] This PR is small and focused on a single feature or bug fix
-- [ ] This PR includes unit testing, or unit testing is not necessary
-- [ ] This PR includes functional testing, or functional testing is not necessary
+<!---
+  Describe the problem this PR is solving. What is the application state
+  before this PR is merged?
+-->
+
+### Solution
+
+<!---
+  Describe the way this PR solves the above problem. Add as much detail as you
+  can to help reviewers understand your changes. Include any alternatives and
+  tradeoffs you considered.
+-->
+
+### Checklist
+
+- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md) and understand what's expected of me
+- [ ] I have run this code in development and it appears to resolve the stated issue
+- [ ] This PR includes tests, or tests are not required/relevant for this PR
+- [ ] This PR has no interface changes (e.g. macros, cli, logs, json artifacts, config files, adapter interface, etc) or this PR has already received feedback and approval from Product or DX

.github/dependabot.yml

@@ -6,13 +6,23 @@ updates:
     -   "/dbt-tests-adapter"
     -   "/dbt-athena"
     -   "/dbt-athena-community"
+    -   "/dbt-spark"
     schedule:
         interval: "daily"
     rebase-strategy: "disabled"
     ignore:
     -   dependency-name: "*"
         update-types:
         -   version-update:semver-patch
+-   package-ecosystem: "docker"
+    directory: "dbt-spark/docker"
+    schedule:
+        interval: "weekly"
+    rebase-strategy: "disabled"
+    ignore:
+    -   dependency-name: "*"
+        update-types:
+        -   version-update:semver-patch
 -   package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/workflows/_generate-changelog.yml

@@ -35,6 +35,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             merge:
                 description: "Choose whether to merge the changelog branch"
                 type: boolean

.github/workflows/_integration-tests.yml

@@ -31,6 +31,7 @@ on:
                 options:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             branch:
                 description: "Choose the branch to test"
                 type: string
@@ -52,6 +53,17 @@ permissions:
     id-token: write
     contents: read
 
+env:
+    DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }}
+    DD_CIVISIBILITY_AGENTLESS_ENABLED: ${{ vars.DD_CIVISIBILITY_AGENTLESS_ENABLED }}
+    DD_API_KEY: ${{ secrets.DD_API_KEY }}
+    DD_SITE: ${{ vars.DD_SITE }}
+    DD_ENV: ${{ vars.DD_ENV }}
+    DD_SERVICE: ${{ github.event.repository.name }}  # this can change per run because of forks
+    DBT_TEST_USER_1: ${{ vars.DBT_TEST_USER_1 }}
+    DBT_TEST_USER_2: ${{ vars.DBT_TEST_USER_2 }}
+    DBT_TEST_USER_3: ${{ vars.DBT_TEST_USER_3 }}
+
 jobs:
     integration-tests-athena:
         if: ${{ inputs.package == 'dbt-athena' || inputs.package == 'dbt-athena-community' }}
@@ -83,3 +95,38 @@ jobs:
                 aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }}
         -   run: hatch run integration-tests
             working-directory: ./${{ inputs.package }}
+
+    integration-tests-spark:
+        if: ${{ inputs.package == 'dbt-spark' }}
+        runs-on: ${{ inputs.os }}
+        environment:
+            name: "dbt-spark"
+        env:
+            DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }}
+            DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }}
+            DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }}
+            DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }}
+            DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USER }}
+        strategy:
+            fail-fast: false
+            matrix:
+                profile:
+                -   "apache_spark"
+                -   "spark_session"
+                -   "spark_http_odbc"
+                -   "databricks_sql_endpoint"
+                -   "databricks_cluster"
+                -   "databricks_http_cluster"
+        steps:
+        -   uses: actions/checkout@v4
+            with:
+                ref: ${{ inputs.branch }}
+                repository: ${{ inputs.repository }}
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ inputs.python-version }}
+        -   uses: pypa/hatch@install
+        -   run: hatch run pip install -r dagger/requirements.txt
+            working-directory: ./${{ inputs.package }}
+        -   run: hatch run integration-tests --profile ${{ matrix.profile }}
+            working-directory: ./${{ inputs.package }}

.github/workflows/_publish-internal.yml

@@ -23,6 +23,7 @@ on:
                 options:
                 -   "dbt-adapters"
                 -   "dbt-athena"
+                -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"
                 type: environment

.github/workflows/_publish-pypi.yml

@@ -24,6 +24,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"
                 type: environment

.github/workflows/_unit-tests.yml

@@ -32,6 +32,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             branch:
                 description: "Choose the branch to test"
                 type: string

.github/workflows/_verify-build.yml

@@ -33,6 +33,7 @@ on:
                 -   "dbt-tests-adapter"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             branch:
                 description: "Choose the branch to build"
                 type: string

.github/workflows/publish.yml

@@ -12,6 +12,7 @@ on:
                 -   "dbt-tests-adapter"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"
                 type: environment

.github/workflows/pull-request-checks.yml

@@ -34,6 +34,7 @@ jobs:
                 -   "dbt-tests-adapter"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
                 os: [ubuntu-22.04]
                 python-version: ["3.9", "3.10", "3.11", "3.12"]
         with:
@@ -52,6 +53,7 @@ jobs:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
                 os: [ ubuntu-22.04 ]
                 python-version: ["3.9", "3.10", "3.11", "3.12"]
         with:
@@ -69,6 +71,7 @@ jobs:
                 package:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
                 os: [ubuntu-22.04]
                 python-version: ["3.9", "3.10", "3.11", "3.12"]
         with:

.gitignore

@@ -149,6 +149,9 @@ dmypy.json
 # pytype static type analyzer
 .pytype/
 
+# Pyenv
+.python-version
+
 # Cython debug symbols
 cython_debug/
 
@@ -164,3 +167,10 @@ cython_debug/
 
 # Vim
 *.swp
+
+# other
+.hive-metastore/
+.spark-warehouse/
+dbt-integration-tests
+/.tool-versions
+/.hypothesis/*

.pre-commit-config.yaml

@@ -28,12 +28,13 @@ repos:
         -   --target-version=py311
         -   --target-version=py312
         -   --force-exclude=dbt/adapters/events/adapter_types_pb2.py
+        additional_dependencies: [flaky]
 
 -   repo: https://github.com/pycqa/flake8
     rev: 7.1.1
     hooks:
     -   id: flake8
-        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/
+        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/
         args:
         -   --max-line-length=99
         -   --select=E,F,W
@@ -54,4 +55,6 @@ repos:
         additional_dependencies:
         -   types-PyYAML
         -   types-protobuf
+        -   types-python-dateutil
         -   types-pytz
+        -   types-requests

dbt-spark/.changes/0.0.0.md

@@ -0,0 +1,12 @@
+## Previous Releases
+For information on prior major and minor releases, see their changelogs:
+- [1.6](https://github.com/dbt-labs/dbt-spark/blob/1.6.latest/CHANGELOG.md)
+- [1.5](https://github.com/dbt-labs/dbt-spark/blob/1.5.latest/CHANGELOG.md)
+- [1.4](https://github.com/dbt-labs/dbt-spark/blob/1.4.latest/CHANGELOG.md)
+- [1.3](https://github.com/dbt-labs/dbt-spark/blob/1.3.latest/CHANGELOG.md)
+- [1.2](https://github.com/dbt-labs/dbt-spark/blob/1.2.latest/CHANGELOG.md)
+- [1.1](https://github.com/dbt-labs/dbt-spark/blob/1.1.latest/CHANGELOG.md)
+- [1.0](https://github.com/dbt-labs/dbt-spark/blob/1.0.latest/CHANGELOG.md)
+- [0.21](https://github.com/dbt-labs/dbt-spark/blob/0.21.latest/CHANGELOG.md)
+- [0.20](https://github.com/dbt-labs/dbt-spark/blob/0.20.latest/CHANGELOG.md)
+- [0.19 and earlier](https://github.com/dbt-labs/dbt-spark/blob/0.19.latest/CHANGELOG.md)

dbt-spark/.changes/README.md

@@ -0,0 +1,3 @@
+# CHANGELOG
+
+To view information about the changelog operation we suggest reading this [README](https://github.com/dbt-labs/dbt-spark/blob/main/.changes/README.md) found in `dbt-spark`.

dbt-spark/.changes/header.tpl.md

@@ -0,0 +1,6 @@
+# dbt-spark Changelog
+
+- This file provides a full account of all changes to `dbt-spark`.
+- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases.
+- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
+- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.md#adding-changelog-entry)

dbt-spark/.changes/unreleased/Under the Hood-20241207-181814.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move from setup.py to pyproject.toml and to hatch as a dev tool
+time: 2024-12-07T18:18:14.85697-05:00
+custom:
+  Author: mikealfare
+  Issue: "1150"