Merge pull request #516 from dbt-labs/monorepo/migrate-dbt-bigquery

ADAP-1130: Migrate dbt-bigquery into dbt-adapters
dbt-labs · Jan 14, 2025 · 886f7da · 886f7da
2 parents 5983011 + e2a2ea6
commit 886f7da
Show file tree

Hide file tree

Showing 211 changed files with 15,677 additions and 2 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -35,6 +35,10 @@ body:
             required: false
         -   label: dbt-athena-community
             required: false
+        -   label: dbt-bigquery
+            required: false
+        -   label: dbt-spark
+            required: false
 -   type: textarea
     attributes:
         label: Current Behavior

diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml
@@ -30,6 +30,10 @@ body:
             required: false
         -   label: dbt-athena-community
             required: false
+        -   label: dbt-bigquery
+            required: false
+        -   label: dbt-spark
+            required: false
 -   type: textarea
     attributes:
         label: Current Behavior

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -6,6 +6,7 @@ updates:
     -   "/dbt-tests-adapter"
     -   "/dbt-athena"
     -   "/dbt-athena-community"
+    -   "/dbt-bigquery"
     -   "/dbt-spark"
     schedule:
         interval: "daily"
@@ -15,7 +16,9 @@ updates:
         update-types:
         -   version-update:semver-patch
 -   package-ecosystem: "docker"
-    directory: "dbt-spark/docker"
+    directories:
+    -   "/dbt-bigquery/docker"
+    -   "/dbt-spark/docker"
     schedule:
         interval: "weekly"
     rebase-strategy: "disabled"

diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml
@@ -35,6 +35,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
             merge:
                 description: "Choose whether to merge the changelog branch"

diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml
@@ -31,6 +31,7 @@ on:
                 options:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
             branch:
                 description: "Choose the branch to test"
@@ -96,6 +97,36 @@ jobs:
         -   run: hatch run integration-tests
             working-directory: ./${{ inputs.package }}
 
+    integration-tests-bigquery:
+        if: ${{ inputs.package == 'dbt-bigquery' }}
+        runs-on: ${{ inputs.os }}
+        environment:
+            name: "dbt-bigquery"
+        env:
+            BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }}
+            BIGQUERY_TEST_ALT_DATABASE: ${{ vars.BIGQUERY_TEST_ALT_DATABASE }}
+            BIGQUERY_TEST_NO_ACCESS_DATABASE: ${{ vars.BIGQUERY_TEST_NO_ACCESS_DATABASE }}
+            DBT_TEST_USER_1: ${{ vars.DBT_TEST_USER_1 }}
+            DBT_TEST_USER_2: ${{ vars.DBT_TEST_USER_2 }}
+            DBT_TEST_USER_3: ${{ vars.DBT_TEST_USER_3 }}
+            DATAPROC_REGION: ${{ vars.DATAPROC_REGION }}
+            DATAPROC_CLUSTER_NAME: ${{ vars.DATAPROC_CLUSTER_NAME }}
+            GCS_BUCKET: ${{ vars.GCS_BUCKET }}
+        steps:
+        -   uses: actions/checkout@v4
+            with:
+                ref: ${{ inputs.branch }}
+                repository: ${{ inputs.repository }}
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ inputs.python-version }}
+        -   uses: pypa/hatch@install
+        -   run: hatch run integration-tests tests/functional -k "not TestPython"
+            working-directory: ./${{ inputs.package }}
+        -   run: hatch run integration-tests tests/functional -n1 -k "TestPython"
+            if: ${{ inputs.python-version == '3.9' }}  # we only run this for one version to run in series
+            working-directory: ./${{ inputs.package }}
+
     integration-tests-spark:
         if: ${{ inputs.package == 'dbt-spark' }}
         runs-on: ${{ inputs.os }}

diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml
@@ -23,6 +23,7 @@ on:
                 options:
                 -   "dbt-adapters"
                 -   "dbt-athena"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"

diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml
@@ -24,6 +24,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"

diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml
@@ -32,6 +32,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
             branch:
                 description: "Choose the branch to test"

diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml
@@ -33,6 +33,7 @@ on:
                 -   "dbt-tests-adapter"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
             branch:
                 description: "Choose the branch to build"

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -12,6 +12,7 @@ on:
                 -   "dbt-tests-adapter"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"

diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml
@@ -34,6 +34,7 @@ jobs:
                 -   "dbt-tests-adapter"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
                 os: [ubuntu-22.04]
                 python-version: ["3.9", "3.10", "3.11", "3.12"]
@@ -53,6 +54,7 @@ jobs:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
                 os: [ ubuntu-22.04 ]
                 python-version: ["3.9", "3.10", "3.11", "3.12"]
@@ -71,6 +73,7 @@ jobs:
                 package:
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-bigquery"
                 -   "dbt-spark"
                 os: [ubuntu-22.04]
                 python-version: ["3.9", "3.10", "3.11", "3.12"]

diff --git a/.gitignore b/.gitignore
@@ -158,6 +158,9 @@ cython_debug/
 # PyCharm
 .idea/
 
+# AWS credentials
+.aws/
+
 # MacOS
 .DS_Store
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -34,7 +34,7 @@ repos:
     rev: 7.1.1
     hooks:
     -   id: flake8
-        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/
+        exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/|dbt-spark/tests/|dbt-bigquery/tests/
         args:
         -   --max-line-length=99
         -   --select=E,F,W

diff --git a/dbt-bigquery/.changes/0.0.0.md b/dbt-bigquery/.changes/0.0.0.md
@@ -0,0 +1,9 @@
+## Previous Releases
+For information on prior major and minor releases, see their changelogs:
+- [1.6](https://github.com/dbt-labs/dbt-bigquery/blob/1.6.latest/CHANGELOG.md)
+- [1.5](https://github.com/dbt-labs/dbt-bigquery/blob/1.5.latest/CHANGELOG.md)
+- [1.4](https://github.com/dbt-labs/dbt-bigquery/blob/1.4.latest/CHANGELOG.md)
+- [1.3](https://github.com/dbt-labs/dbt-bigquery/blob/1.3.latest/CHANGELOG.md)
+- [1.2](https://github.com/dbt-labs/dbt-bigquery/blob/1.2.latest/CHANGELOG.md)
+- [1.1](https://github.com/dbt-labs/dbt-bigquery/blob/1.1.latest/CHANGELOG.md)
+- [1.0](https://github.com/dbt-labs/dbt-bigquery/blob/1.0.latest/CHANGELOG.md)
diff --git a/dbt-bigquery/.changes/README.md b/dbt-bigquery/.changes/README.md
@@ -0,0 +1,3 @@
+# CHANGELOG
+
+To view information about the changelog operation we suggest reading this [README](https://github.com/dbt-labs/dbt-bigquery/blob/main/.changes/README.md) found in `dbt-bigquery`.
diff --git a/dbt-bigquery/.changes/header.tpl.md b/dbt-bigquery/.changes/header.tpl.md
@@ -0,0 +1,6 @@
+# dbt-bigquery Changelog
+
+- This file provides a full account of all changes to `dbt-bigquery`.
+- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases.
+- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
+- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-bigquery/blob/main/CONTRIBUTING.md#adding-changelog-entry)
diff --git a/dbt-bigquery/.changes/unreleased/.gitkeep b/dbt-bigquery/.changes/unreleased/.gitkeep
diff --git a/dbt-bigquery/.changes/unreleased/Features-20241202-223835.yaml b/dbt-bigquery/.changes/unreleased/Features-20241202-223835.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Allow copy_partitions in microbatch
+time: 2024-12-02T22:38:35.479052Z
+custom:
+    Author: borjavb
+    Issue: "1414"
diff --git a/dbt-bigquery/.changes/unreleased/Fixes-20241120-163101.yaml b/dbt-bigquery/.changes/unreleased/Fixes-20241120-163101.yaml
@@ -0,0 +1,7 @@
+kind: Fixes
+body: Fix issue where dbt-bigquery was not retrying in certain retryable scenarios,
+  e.g. 503's
+time: 2024-11-20T16:31:01.60689-05:00
+custom:
+  Author: mikealfare
+  Issue: "682"
diff --git a/dbt-bigquery/.changes/unreleased/Fixes-20241204-105846.yaml b/dbt-bigquery/.changes/unreleased/Fixes-20241204-105846.yaml
@@ -0,0 +1,7 @@
+kind: Fixes
+body: Cast `event_time` to a timestamp prior to comparing against microbatch start/end
+  time
+time: 2024-12-04T10:58:46.573608-05:00
+custom:
+  Author: michelleark
+  Issue: "1422"
diff --git a/dbt-bigquery/.changes/unreleased/Fixes-20241205-133606.yaml b/dbt-bigquery/.changes/unreleased/Fixes-20241205-133606.yaml
@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix issue where rate limit errors on table service calls are not retried
+time: 2024-12-05T13:36:06.436005-05:00
+custom:
+  Author: mikealfare
+  Issue: "1423"
diff --git a/dbt-bigquery/.changes/unreleased/Fixes-20241211-144752.yaml b/dbt-bigquery/.changes/unreleased/Fixes-20241211-144752.yaml
@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix retry scenarios so that dbt always retries when BigQuery recommends a retry
+time: 2024-12-11T14:47:52.36905-05:00
+custom:
+  Author: mikealfare
+  Issue: "263"
diff --git a/dbt-bigquery/.changes/unreleased/Under the Hood-20241117-194746.yaml b/dbt-bigquery/.changes/unreleased/Under the Hood-20241117-194746.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move from setup.py to pyproject.toml and to hatch as a dev tool
+time: 2024-11-17T19:47:46.341-05:00
+custom:
+  Author: mikealfare
+  Issue: "1407"
diff --git a/dbt-bigquery/.changie.yaml b/dbt-bigquery/.changie.yaml
@@ -0,0 +1,131 @@
+changesDir: .changes
+unreleasedDir: unreleased
+headerPath: header.tpl.md
+versionHeaderPath: ""
+changelogPath: CHANGELOG.md
+versionExt: md
+envPrefix: "CHANGIE_"
+versionFormat: '## dbt-bigquery {{.Version}} - {{.Time.Format "January 02, 2006"}}'
+kindFormat: '### {{.Kind}}'
+changeFormat: |-
+  {{- $IssueList := list }}
+  {{- $changes := splitList " " $.Custom.Issue }}
+  {{- range $issueNbr := $changes }}
+    {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-bigquery/issues/nbr)" | replace "nbr" $issueNbr }}
+    {{- $IssueList = append $IssueList $changeLink  }}
+  {{- end -}}
+  - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}})
+
+kinds:
+- label: Breaking Changes
+- label: Features
+- label: Fixes
+- label: Under the Hood
+- label: Dependencies
+  changeFormat: |-
+    {{- $PRList := list }}
+    {{- $changes := splitList " " $.Custom.PR }}
+    {{- range $pullrequest := $changes }}
+      {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-bigquery/pull/nbr)" | replace "nbr" $pullrequest }}
+      {{- $PRList = append $PRList $changeLink  }}
+    {{- end -}}
+    - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}})
+  skipGlobalChoices: true
+  additionalChoices:
+    - key: Author
+      label: GitHub Username(s) (separated by a single space if multiple)
+      type: string
+      minLength: 3
+    - key: PR
+      label: GitHub Pull Request Number (separated by a single space if multiple)
+      type: string
+      minLength: 1
+- label: Security
+  changeFormat: |-
+    {{- $PRList := list }}
+    {{- $changes := splitList " " $.Custom.PR }}
+    {{- range $pullrequest := $changes }}
+      {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-bigquery/pull/nbr)" | replace "nbr" $pullrequest }}
+      {{- $PRList = append $PRList $changeLink  }}
+    {{- end -}}
+    - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}})
+  skipGlobalChoices: true
+  additionalChoices:
+    - key: Author
+      label: GitHub Username(s) (separated by a single space if multiple)
+      type: string
+      minLength: 3
+    - key: PR
+      label: GitHub Pull Request Number (separated by a single space if multiple)
+      type: string
+      minLength: 1
+
+newlines:
+  afterChangelogHeader: 1
+  afterKind: 1
+  afterChangelogVersion: 1
+  beforeKind: 1
+  endOfVersion: 1
+
+custom:
+- key: Author
+  label: GitHub Username(s) (separated by a single space if multiple)
+  type: string
+  minLength: 3
+- key: Issue
+  label: GitHub Issue Number (separated by a single space if multiple)
+  type: string
+  minLength: 1
+
+footerFormat: |
+  {{- $contributorDict := dict }}
+  {{- /* ensure all names in this list are all lowercase for later matching purposes */}}
+  {{- $core_team := splitList " " .Env.CORE_TEAM }}
+  {{- /* ensure we always skip snyk and dependabot in addition to the core team */}}
+  {{- $maintainers := list "dependabot[bot]" "snyk-bot"}}
+  {{- range $team_member := $core_team }}
+    {{- $team_member_lower := lower $team_member }}
+    {{- $maintainers = append $maintainers $team_member_lower }}
+  {{- end }}
+  {{- range $change := .Changes }}
+    {{- $authorList := splitList " " $change.Custom.Author }}
+    {{- /* loop through all authors for a single changelog */}}
+    {{- range $author := $authorList }}
+      {{- $authorLower := lower $author }}
+      {{- /* we only want to include non-core team contributors */}}
+      {{- if not (has $authorLower $maintainers)}}
+        {{- $changeList := splitList " " $change.Custom.Author }}
+          {{- $IssueList := list }}
+          {{- $changeLink := $change.Kind }}
+          {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }}
+            {{- $changes := splitList " " $change.Custom.PR }}
+            {{- range $issueNbr := $changes }}
+              {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-bigquery/pull/nbr)" | replace "nbr" $issueNbr }}
+              {{- $IssueList = append $IssueList $changeLink  }}
+            {{- end -}}
+          {{- else }}
+            {{- $changes := splitList " " $change.Custom.Issue }}
+            {{- range $issueNbr := $changes }}
+              {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-bigquery/issues/nbr)" | replace "nbr" $issueNbr }}
+              {{- $IssueList = append $IssueList $changeLink  }}
+            {{- end -}}
+          {{- end }}
+          {{- /* check if this contributor has other changes associated with them already */}}
+          {{- if hasKey $contributorDict $author }}
+            {{- $contributionList := get $contributorDict $author }}
+            {{- $contributionList = concat $contributionList $IssueList  }}
+            {{- $contributorDict := set $contributorDict $author $contributionList }}
+          {{- else }}
+            {{- $contributionList := $IssueList }}
+            {{- $contributorDict := set $contributorDict $author $contributionList }}
+          {{- end }}
+        {{- end}}
+    {{- end}}
+  {{- end }}
+  {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}}
+  {{- if $contributorDict}}
+  ### Contributors
+  {{- range $k,$v := $contributorDict }}
+  - [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}})
+  {{- end }}
+  {{- end }}
diff --git a/dbt-bigquery/CHANGELOG.md b/dbt-bigquery/CHANGELOG.md
@@ -0,0 +1,16 @@
+# dbt-bigquery Changelog
+
+- This file provides a full account of all changes to `dbt-bigquery`.
+- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases.
+- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
+- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-bigquery/blob/main/CONTRIBUTING.md#adding-changelog-entry)
+
+## Previous Releases
+For information on prior major and minor releases, see their changelogs:
+- [1.6](https://github.com/dbt-labs/dbt-bigquery/blob/1.6.latest/CHANGELOG.md)
+- [1.5](https://github.com/dbt-labs/dbt-bigquery/blob/1.5.latest/CHANGELOG.md)
+- [1.4](https://github.com/dbt-labs/dbt-bigquery/blob/1.4.latest/CHANGELOG.md)
+- [1.3](https://github.com/dbt-labs/dbt-bigquery/blob/1.3.latest/CHANGELOG.md)
+- [1.2](https://github.com/dbt-labs/dbt-bigquery/blob/1.2.latest/CHANGELOG.md)
+- [1.1](https://github.com/dbt-labs/dbt-bigquery/blob/1.1.latest/CHANGELOG.md)
+- [1.0](https://github.com/dbt-labs/dbt-bigquery/blob/1.0.latest/CHANGELOG.md)
-Original file line number
+Diff line change
@@ Expand Up / @@ -158,6 +158,9 @@ cython_debug/ @@
     # PyCharm
     .idea/
+    # AWS credentials
+    .aws/
     # MacOS
     .DS_Store
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# CHANGELOG

		To view information about the changelog operation we suggest reading this [README](https://github.com/dbt-labs/dbt-bigquery/blob/main/.changes/README.md) found in `dbt-bigquery`.