diff --git a/jaffle_shop/models/final/finance/_exposures.yml b/jaffle_shop/models/final/finance/_exposures.yml deleted file mode 100644 index b38da1da7..000000000 --- a/jaffle_shop/models/final/finance/_exposures.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 - -exposures: - - name: customer_return_total - description: Inksacio App to show the total sum of customer returns, per customer (for both completed and upcoming) - type: dashboard - url: https://inksacio.eks.octojaffle.engineering/customer_return_totals/ - owner: - name: 'Elliot Taylor' - email: elliot.taylor@octojaffle.com - depends_on: - - ref('fnl_finance_customerreturnstotal') \ No newline at end of file diff --git a/jaffle_shop/models/final/finance/_models.yml b/jaffle_shop/models/final/finance/_models.yml index ced7510c5..37e4cfe18 100644 --- a/jaffle_shop/models/final/finance/_models.yml +++ b/jaffle_shop/models/final/finance/_models.yml @@ -1,7 +1,22 @@ version: 2 +exposures: + - name: customer_return_total + description: + Inksacio App to show the total sum of customer returns + , per customer (for both completed and upcoming) + type: dashboard + url: https://inksacio.eks.octojaffle.engineering/customer_return_totals/ + owner: + name: 'Elliot Taylor' + email: elliot.taylor@octojaffle.com + depends_on: + - ref('fnl_finance_customerreturnstotal') models: - name: fnl_finance_customerreturnstotal + meta: + owner: elliot.taylor@octojaffle.com + team_owner: '!subteam^S7K9CR63D' #@datateam description: > This table gives one row per customer, with the total of their completed returns, pending_returns and the sum of the 2 @@ -10,4 +25,4 @@ models: description: primary key tests: - unique - - not_null + - not_null \ No newline at end of file diff --git a/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql index a832f0dae..5243d3758 100644 --- a/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql +++ b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql @@ -1,6 +1,6 @@ {% set return_states = ['returned', 'return_pending'] %} -select +SELECT orders.customer_id {% for return_state in return_states -%} @@ -9,6 +9,6 @@ select , SUM(orders.amount_dollars) AS sum_return_amount_dollars -from {{ ref('wh_orders') }} AS orders -where orders.status IN ('returned', 'return_pending') +FROM {{ ref('wh_orders') }} AS orders +WHERE orders.status IN ('returned', 'return_pending') GROUP BY orders.customer_id \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/_exposures.yml b/jaffle_shop/models/final/sales/_exposures.yml deleted file mode 100644 index c12bb1be5..000000000 --- a/jaffle_shop/models/final/sales/_exposures.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 - -exposures: - - name: new_customer_history - description: Inksacio App to show monthly new customer history - type: dashboard - url: https://inksacio.eks.octojaffle.engineering/new_customer_history/ - owner: - name: 'Elliot Taylor' - email: elliot.taylor@octojaffle.com - depends_on: - - ref('fnl_sales_newcustomerhistory') \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/_models.yml b/jaffle_shop/models/final/sales/_models.yml index 8186417cf..0466a221c 100644 --- a/jaffle_shop/models/final/sales/_models.yml +++ b/jaffle_shop/models/final/sales/_models.yml @@ -1,7 +1,20 @@ version: 2 +exposures: + - name: new_customer_history + description: Inksacio App to show monthly new customer history + type: dashboard + url: https://inksacio.eks.octojaffle.engineering/new_customer_history/ + owner: + name: 'Elliot Taylor' + email: elliot.taylor@octojaffle.com + depends_on: + - ref('fnl_sales_newcustomerhistory') models: - name: fnl_sales_newcustomerhistory + meta: + owner: elliot.taylor@octojaffle.com + team_owner: '!subteam^S7K9CR63D' #@datateam description: > This table gives one row per truncated month with the count of customers that have created their first order within this month diff --git a/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql b/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql index 8258f25f4..bbc8063c5 100644 --- a/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql +++ b/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql @@ -1,4 +1,4 @@ -select - date_trunc('month', first_order) AS first_order_month - , count(*) AS number_customers -from {{ ref('wh_customers') }} \ No newline at end of file +SELECT + DATE_TRUNC('month', first_order) AS first_order_month + , COUNT(DISTINCT customer_id) AS number_customers +FROM {{ ref('wh_customers') }} \ No newline at end of file diff --git a/jaffle_shop/models/staging/src_seed/_models.yml b/jaffle_shop/models/staging/src_seed/_models.yml index 981a7e3b9..420ed23d2 100644 --- a/jaffle_shop/models/staging/src_seed/_models.yml +++ b/jaffle_shop/models/staging/src_seed/_models.yml @@ -13,7 +13,6 @@ models: tests: - accepted_values: values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - - name: stg_payments description: staging layer for all payments with a FK to order columns: @@ -26,7 +25,21 @@ models: tests: - accepted_values: values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] - + - name: stg_customers_pii + description: > + staging layer for customers, PII hashed + columns: + - name: customer_id + description: Primary Key + tests: + - unique + - not_null + - name: first_name + meta: + sensitive: true + - name: last_name + meta: + sensitive: true - name: stg_customers description: staging layer for all customers, PII hashed columns: @@ -34,4 +47,10 @@ models: description: Primary Key tests: - unique - - not_null \ No newline at end of file + - not_null + - name: first_name_hash + tests: + - dbt_expectations.expect_column_to_exist + - name: last_name_hash + tests: + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml deleted file mode 100644 index e75f52c6c..000000000 --- a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 2 - -models: - - name: stg_customers_pii - description: > - Table that includes all info about all customers, with PII hashed - columns: - - name: customer_id - description: Primary Key - tests: - - unique - - not_null - - name: first_name - meta: - sensitive: true - - name: last_name - meta: - sensitive: true \ No newline at end of file diff --git a/jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql b/jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql deleted file mode 100644 index cad047269..000000000 --- a/jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql +++ /dev/null @@ -1,22 +0,0 @@ -with source as ( - - {#- - Normally we would select from the table here, but we are using seeds to load - our data in this project - #} - select * from {{ ref('raw_customers') }} - -), - -renamed as ( - - select - id as customer_id, - first_name, - last_name - - from source - -) - -select * from renamed diff --git a/jaffle_shop/models/staging/src_seed/stg_customers_pii.sql b/jaffle_shop/models/staging/src_seed/stg_customers_pii.sql new file mode 100644 index 000000000..7fed1e68d --- /dev/null +++ b/jaffle_shop/models/staging/src_seed/stg_customers_pii.sql @@ -0,0 +1,23 @@ +WITH source AS ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + SELECT * + FROM {{ ref('raw_customers') }} + +), + +renamed AS ( + + SELECT + id AS customer_id + , first_name + , last_name + FROM source + +) + +SELECT * +FROM renamed diff --git a/jaffle_shop/models/staging/src_seed/stg_orders.sql b/jaffle_shop/models/staging/src_seed/stg_orders.sql index a654dcb94..2fa70c84a 100644 --- a/jaffle_shop/models/staging/src_seed/stg_orders.sql +++ b/jaffle_shop/models/staging/src_seed/stg_orders.sql @@ -1,23 +1,24 @@ -with source as ( +WITH source AS ( {#- Normally we would select from the table here, but we are using seeds to load our data in this project #} - select * from {{ ref('raw_orders') }} + SELECT * + FROM {{ ref('raw_orders') }} ), -renamed as ( +renamed AS ( - select - id as order_id, - user_id as customer_id, - order_date, - status - - from source + SELECT + id AS order_id + , user_id AS customer_id + , order_date + , status + FROM source ) -select * from renamed +SELECT * +FROM renamed diff --git a/jaffle_shop/models/staging/src_seed/stg_payments.sql b/jaffle_shop/models/staging/src_seed/stg_payments.sql index 502a3ecdc..d62c38fd3 100644 --- a/jaffle_shop/models/staging/src_seed/stg_payments.sql +++ b/jaffle_shop/models/staging/src_seed/stg_payments.sql @@ -1,25 +1,24 @@ -with source as ( +WITH source AS ( {#- Normally we would select from the table here, but we are using seeds to load our data in this project #} - select * from {{ ref('raw_payments') }} + SELECT * FROM {{ ref('raw_payments') }} ), -renamed as ( +renamed AS ( - select - id as payment_id, - order_id, - payment_method, - - -- `amount` is currently stored in cents, so we convert it to dollars - amount / 100 as amount_dollars - - from source + SELECT + id AS payment_id + , order_id + , payment_method + -- `amount` is currently stored in cents, so we convert it to dollars + , amount / 100 AS amount_dollars + FROM source ) -select * from renamed +SELECT * +FROM renamed diff --git a/jaffle_shop/seeds/_seeds.yml b/jaffle_shop/seeds/_seeds.yml new file mode 100644 index 000000000..e24326136 --- /dev/null +++ b/jaffle_shop/seeds/_seeds.yml @@ -0,0 +1,38 @@ +version: 2 + +seeds: + - name: dbt_project_evaluator_exceptions + description: > + Table exceptions to avoid staging to staging failures + config: + column_types: + fct_name: string + column_name: string + id_to_exclude: string + comment: string + - name: raw_customers + description: > + Customer names and IDs + config: + column_types: + id: bigint + first_name: string + last_name: string + - name: raw_orders + description: > + Customer orders by id, date, and status + config: + column_types: + id: bigint + user_id: bigint + order_date: date + status: string + - name: raw_payments + description: > + Customer payments by id, payment method, and amount paid + config: + column_types: + id: bigint + order_id: bigint + payment_method: string + amount: bigint \ No newline at end of file