chore(#27): Add dbt unit test brac (#28)

* Add tests to contact views working models * Add tests to use views working models * feat: make model chnages and temporarily remove dashboard models * feat: fix failing models * Merge conflicts * feat: fix failing models * Add users.yml to gitignore * Add tests for contact_views models * Add tests to useviews models * Fix some test * Add tests for main models * Add relationships and accepted_values tests * Add relationships tests on contact views models * Add relationships tests to main models * Fix Did not find matching node for patch error * Fix accepted values errors * Fix tests * Fixing failing tests * Delete some failing tests * First views model test * Add useviews model tests * Change the way columns are referenced in couchdb * Trying to see if the problem are the seeds * Add test to pncview_actual_enrrollments model * Add test for form_metadata model * Add test for impactview_month * Add test to formview_inmunization_follow_up model * Add test for formview_assessment_follow_up model * Add test for formview_muac_follow_up * Add test on couples of views models * Fix views.yml duplicate * Add more test on views models * Add more test to views models * Delete failed tests * Add first test for impact anc models * Delete exclude command from run dbt * Revert deletion of failing tests * Reverse change * reverse deletions * Fix brac name * Finish test for views models * Move and rename impact test file. Implement impact tests * Delete commented lines --------- Co-authored-by: Maria Lorena Rodriguez Viruel <[email protected]> Co-authored-by: Njuguna Ndung'u <[email protected]>
medic · Nov 1, 2023 · 194b3d5 · 194b3d5
1 parent 2b7957b
commit 194b3d5
Show file tree

Hide file tree

Showing 26 changed files with 793 additions and 321 deletions.
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,7 @@ dbt_packages/
 logs/
 */.env
 */dbt_packages/
+tests/.user.yml
+dbt-env/*
+.user.yml
+dbt-env/*
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,34 +1,35 @@
 # Name your project! Project names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: "pipeline"
-version: "0.0.1"
+name: 'pipeline'
+version: '0.0.1'
 config-version: 2
 
+vars:
+  # The `start_date` variable will be accessible in all resources
+  end_date: '2023-06-01'
+
 # This setting configures which "profile" dbt uses for this project.
-profile: "default"
+profile: 'default'
 
 # These configurations specify where dbt should look for different types of files.
 # The `source-paths` config, for example, states that models in this project can be
 # found in the "models/" directory. You probably won't need to change these!
-model-paths: ["models"]
-analysis-paths: ["analysis"]
-test-paths: ["tests"]
-seed-paths: ["seeds"]
-macro-paths: ["macros"]
-snapshot-paths: ["snapshots"]
+model-paths: ['models']
+analysis-paths: ['analysis']
+test-paths: ['tests']
+seed-paths: ['seeds']
+macro-paths: ['macros']
+snapshot-paths: ['snapshots']
 
-target-path: "target" # directory which will store compiled SQL files
+target-path: 'target' # directory which will store compiled SQL files
 clean-targets: # directories to be removed by `dbt clean`
-  - "target"
-  - "dbt_modules"
+  - 'target'
+  - 'dbt_modules'
 
-# Configuring models
-# Full documentation: https://docs.getdbt.com/docs/configuring-models
+vars:
+  end_date: "2020-12-12"
 
-# In this example config, we tell dbt to build all models in the example/ directory
-# as tables. These settings can be overridden in the individual model files
-# using the `{{ config(...) }}` macro.
 models:
   type_partitions:
     root:

diff --git a/models/brac/brac.yml b/models/brac/brac.yml
@@ -0,0 +1,74 @@
+version: 2
+
+models:
+  - name: brac_hmis
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+          - unique
+          - relationships:
+              to: ref('contactview_chp')
+              field: uuid
+      - name: households_registered
+        tests:
+          - not_null
+      - name: num_of_hh_visited
+        tests:
+          - not_null
+      - name: num_of_hh_bed_nets
+        tests:
+          - not_null
+      - name: num_of_hh_latrines
+        tests:
+          - not_null
+      - name: num_of_hh_safe_water
+        tests:
+          - not_null
+      - name: num_of_comm_members
+        tests:
+          - not_null
+      - name: num_of_health_forum
+        tests:
+          - not_null
+      - name: num_pregnant_care
+        tests:
+          - not_null
+      - name: num_pregnant_identified
+        tests:
+          - not_null
+      - name: num_facility_deliveries
+        tests:
+          - not_null
+      - name: num_home_deliveries
+        tests:
+          - not_null
+  - name: brac-kpis
+    columns:
+      - name: chp_uuid
+        tests:
+          - not_null
+          - relationships:
+              to: ref('contactview_chp')
+              field: uuid
+      - name: count_all_assessments
+        tests:
+          - not_null
+      - name: u5_malaria_treatment
+        tests:
+          - not_null
+      - name: u5_diarrhea_treatment
+        tests:
+          - not_null
+      - name: u5_pneumonia_treatment
+        tests:
+          - not_null
+      - name: u1_malaria_treatment
+        tests:
+          - not_null
+      - name: u1_diarrhea_treatment
+        tests:
+          - not_null
+      - name: u1_pneumonia_treatment
+        tests:
+          - not_null
diff --git a/models/brac/contact_views/contact_views.yml b/models/brac/contact_views/contact_views.yml
@@ -0,0 +1,143 @@
+version: 2
+
+models:
+  - name: contactview_person
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+          - unique
+          - relationships:
+              to: ref('couchdb')
+              field: doc ->> '_id'
+      - name: name
+        tests:
+          - not_null
+      - name: patient_id
+        tests:
+          - not_null
+  - name: contactview_person_fields
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+          - unique
+          - relationships:
+              to: ref('contactview_metadata')
+              field: uuid
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('contactview_metadata')
+  - name: contactview_chw
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+          - unique
+          - relationships:
+              to: ref('contactview_person_fields')
+              field: uuid
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('contactview_metadata')
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('contactview_person_fields')
+  - name: contactview_chp
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+          - unique
+          - relationships:
+              to: ref('contactview_chw')
+              field: uuid
+      - name: branch_uuid
+        tests:
+          - not_null
+      - name: supervisor_uuid
+        tests:
+          - not_null
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('contactview_metadata')
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('contactview_person_fields')
+  - name: contactview_muted
+    columns:
+      - name: contact_uuid
+        tests:
+          - not_null
+          - unique
+          - relationships:
+              to: ref('contactview_metadata')
+              field: uuid
+      - name: type
+        tests:
+          - not_null
+          - accepted_values:
+              values:
+                [contact, clinic, district_hospital, health_center, person]
+      - name: mute_status
+        tests:
+          - not_null
+      - name: date
+        tests:
+          - not_null
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('contactview_metadata')
+  - name: contactview_hospital
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+          - unique
+          - relationships:
+              to: ref('contactview_metadata')
+              field: uuid
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('contactview_metadata')
+  - name: contactview_hierarchy
+    columns:
+      - name: branch_uuid
+        tests:
+          - not_null
+          - relationships:
+              to: ref('contactview_branch')
+              field: uuid
+      - name: chw_uuid
+        tests:
+          - not_null
+          - relationships:
+              to: ref('contactview_chp')
+              field: uuid
+      - name: chw_area_uuid
+        tests:
+          - not_null
+  - name: contactview_branch
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+          - unique
+          - relationships:
+              to: ref('contactview_hospital')
+              field: uuid
+  - name: contactview_clinic
+    columns:
+      - name: uuid
+        tests:
+          - not_null
+          - relationships:
+              to: ref('contactview_metadata')
+              field: uuid
+      - name: chw_uuid
+        tests:
+          - not_null
+          - relationships:
+              to: ref('contactview_chw')
+              field: uuid
+    tests:
+      - dbt_utils.fewer_rows_than:
+          compare_model: ref('contactview_metadata')
diff --git a/models/brac/contact_views/contactview_chp.sql b/models/brac/contact_views/contactview_chp.sql
@@ -47,8 +47,7 @@ SELECT
     chp.doc #>> '{chp_profile, g_language, mother_tongue}'::text[] AS mother_tongue,
     chp.doc #>> '{chp_profile, g_language, other_languages}'::text[] AS other_languages,
     chp.doc #>> '{chp_profile, g_other_details, incentives}'::text[] AS incentives,
-    chp.doc #>> '{chp_profile, g_other_details, chp_services}'::text[] AS chp_services,
-    contactview_chw."@timestamp" AS "@timestamp"
+    chp.doc #>> '{chp_profile, g_other_details, chp_services}'::text[] AS chp_services
   FROM
     {{ ref("contactview_chw") }}
   JOIN {{ ref("raw_contacts") }} ON contactview_chw.area_uuid = (raw_contacts.doc ->> '_id'::text)

diff --git a/models/brac/contact_views/contactview_chw.sql b/models/brac/contact_views/contactview_chw.sql
@@ -5,8 +5,7 @@ SELECT chw.name,
     pplfields.date_of_birth,
     pplfields.parent_type,
     chwarea.uuid AS area_uuid,
-    chwarea.parent_uuid AS branch_uuid,
-    pplfields."@timestamp" AS "@timestamp"
+    chwarea.parent_uuid AS branch_uuid
 FROM {{ ref("contactview_person_fields") }} pplfields
     JOIN {{ ref("contactview_metadata") }} chw ON chw.uuid = pplfields.uuid
     JOIN {{ ref("contactview_metadata") }} chwarea ON chw.parent_uuid = chwarea.uuid

diff --git a/models/brac/contact_views/contactview_person_fields.sql b/models/brac/contact_views/contactview_person_fields.sql
@@ -2,8 +2,7 @@ SELECT person.uuid,
 		person.phone,
 		person.phone2,
 		person.date_of_birth,
-		parent.type AS parent_type,
-		person."@timestamp" AS "@timestamp"
+		parent.type AS parent_type
 	FROM {{ ref("contactview_metadata") }} person
 	LEFT JOIN {{ ref("contactview_metadata") }} parent ON person.parent_uuid = parent.uuid
 	WHERE person.type = 'person'::text
diff --git a/models/brac/impact/ANC/impact_ancview_danger_sign.sql b/models/brac/impact/ANC/impact_ancview_danger_sign.sql
@@ -43,4 +43,4 @@ SELECT
   reported_by_parent,
   reported
 FROM
-  danger_sign_CTE;
+  danger_sign_CTE
diff --git a/models/brac/impact/ANC/impact_ancview_delivery.sql b/models/brac/impact/ANC/impact_ancview_delivery.sql
@@ -36,4 +36,4 @@ LEFT JOIN preg ON (pnc.reported > preg.reported AND pnc.reported < (preg.reporte
 WHERE
   pnc.follow_up_count = '1'
   AND (pnc.pregnancy_outcome IN (VALUES ('healthy'), ('still_birth')))
-  AND pnc.patient_id IS NOT NULL AND pnc.patient_id <> '';
+  AND pnc.patient_id IS NOT NULL AND pnc.patient_id <> ''
diff --git a/models/brac/impact/ANC/impact_ancview_pregnancy.sql b/models/brac/impact/ANC/impact_ancview_pregnancy.sql
diff --git a/models/brac/impact/ANC/impact_ancview_pregnancy_visit.sql b/models/brac/impact/ANC/impact_ancview_pregnancy_visit.sql
@@ -15,4 +15,4 @@ SELECT
 FROM
   {{ ref('useview_visit') }}
 WHERE
-  visit_type = 'anc';
+  visit_type = 'anc'