From b6772e53e50c5d05902388b08a00943f815728fa Mon Sep 17 00:00:00 2001 From: Elliot Taylor Date: Wed, 4 Jan 2023 14:44:41 +0000 Subject: [PATCH] Refactor Repo to follow convention --- jaffle_shop/models/intermediate/_models.yml | 0 .../staging/{schema.yml => src_seed/_models.yml} | 8 -------- jaffle_shop/models/staging/src_seed/_sources.yml | 0 .../models/staging/src_seed/sensitive/_models.yml | 15 +++++++++++++++ .../sensitive/stg_customers_pii.sql} | 0 .../models/staging/src_seed/stg_customers.sql | 3 +++ .../models/staging/{ => src_seed}/stg_orders.sql | 0 .../staging/{ => src_seed}/stg_payments.sql | 0 .../models/{schema.yml => warehouse/_models.yml} | 0 jaffle_shop/models/{ => warehouse}/customers.sql | 0 jaffle_shop/models/{ => warehouse}/docs.md | 0 jaffle_shop/models/{ => warehouse}/orders.sql | 0 .../seeds/dbt_project_evaluator_exceptions.csv | 2 ++ 13 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 jaffle_shop/models/intermediate/_models.yml rename jaffle_shop/models/staging/{schema.yml => src_seed/_models.yml} (81%) create mode 100644 jaffle_shop/models/staging/src_seed/_sources.yml create mode 100644 jaffle_shop/models/staging/src_seed/sensitive/_models.yml rename jaffle_shop/models/staging/{stg_customers.sql => src_seed/sensitive/stg_customers_pii.sql} (100%) create mode 100644 jaffle_shop/models/staging/src_seed/stg_customers.sql rename jaffle_shop/models/staging/{ => src_seed}/stg_orders.sql (100%) rename jaffle_shop/models/staging/{ => src_seed}/stg_payments.sql (100%) rename jaffle_shop/models/{schema.yml => warehouse/_models.yml} (100%) rename jaffle_shop/models/{ => warehouse}/customers.sql (100%) rename jaffle_shop/models/{ => warehouse}/docs.md (100%) rename jaffle_shop/models/{ => warehouse}/orders.sql (100%) create mode 100644 jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv diff --git a/jaffle_shop/models/intermediate/_models.yml b/jaffle_shop/models/intermediate/_models.yml new file mode 100644 index 000000000..e69de29bb diff --git a/jaffle_shop/models/staging/schema.yml b/jaffle_shop/models/staging/src_seed/_models.yml similarity index 81% rename from jaffle_shop/models/staging/schema.yml rename to jaffle_shop/models/staging/src_seed/_models.yml index c207e4cf5..d55944bd1 100644 --- a/jaffle_shop/models/staging/schema.yml +++ b/jaffle_shop/models/staging/src_seed/_models.yml @@ -1,13 +1,5 @@ version: 2 -models: - - name: stg_customers - columns: - - name: customer_id - tests: - - unique - - not_null - - name: stg_orders columns: - name: order_id diff --git a/jaffle_shop/models/staging/src_seed/_sources.yml b/jaffle_shop/models/staging/src_seed/_sources.yml new file mode 100644 index 000000000..e69de29bb diff --git a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml new file mode 100644 index 000000000..f5e4e844d --- /dev/null +++ b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml @@ -0,0 +1,15 @@ +version: 2 + +models: + - name: stg_customers + columns: + - name: customer_id + tests: + - unique + - not_null + - name: first_name + meta: + sensitive: true + - name: last_name + meta: + sensitive: true \ No newline at end of file diff --git a/jaffle_shop/models/staging/stg_customers.sql b/jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql similarity index 100% rename from jaffle_shop/models/staging/stg_customers.sql rename to jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql diff --git a/jaffle_shop/models/staging/src_seed/stg_customers.sql b/jaffle_shop/models/staging/src_seed/stg_customers.sql new file mode 100644 index 000000000..b9eadce2c --- /dev/null +++ b/jaffle_shop/models/staging/src_seed/stg_customers.sql @@ -0,0 +1,3 @@ +SELECT + {{ hash_sensitive_columns('stg_customers_pii') }} +FROM {{ ref('stg_customers_pii') }} \ No newline at end of file diff --git a/jaffle_shop/models/staging/stg_orders.sql b/jaffle_shop/models/staging/src_seed/stg_orders.sql similarity index 100% rename from jaffle_shop/models/staging/stg_orders.sql rename to jaffle_shop/models/staging/src_seed/stg_orders.sql diff --git a/jaffle_shop/models/staging/stg_payments.sql b/jaffle_shop/models/staging/src_seed/stg_payments.sql similarity index 100% rename from jaffle_shop/models/staging/stg_payments.sql rename to jaffle_shop/models/staging/src_seed/stg_payments.sql diff --git a/jaffle_shop/models/schema.yml b/jaffle_shop/models/warehouse/_models.yml similarity index 100% rename from jaffle_shop/models/schema.yml rename to jaffle_shop/models/warehouse/_models.yml diff --git a/jaffle_shop/models/customers.sql b/jaffle_shop/models/warehouse/customers.sql similarity index 100% rename from jaffle_shop/models/customers.sql rename to jaffle_shop/models/warehouse/customers.sql diff --git a/jaffle_shop/models/docs.md b/jaffle_shop/models/warehouse/docs.md similarity index 100% rename from jaffle_shop/models/docs.md rename to jaffle_shop/models/warehouse/docs.md diff --git a/jaffle_shop/models/orders.sql b/jaffle_shop/models/warehouse/orders.sql similarity index 100% rename from jaffle_shop/models/orders.sql rename to jaffle_shop/models/warehouse/orders.sql diff --git a/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv new file mode 100644 index 000000000..b9f94e579 --- /dev/null +++ b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv @@ -0,0 +1,2 @@ +fct_name,column_name,id_to_exclude,comment +fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer. \ No newline at end of file