Merge pull request #270 from CanDIG/stable-candidate-v5.0.0

v5.0.0: MoH data model 3.1
CanDIG · Oct 4, 2024 · 13c81fd · 13c81fd
2 parents c550625 + 889bc1b
commit 13c81fd
Show file tree

Hide file tree

Showing 203 changed files with 16,906 additions and 5,192,208 deletions.
diff --git a/.github/workflows/dispatch-actions.yml b/.github/workflows/dispatch-actions.yml
@@ -1,7 +1,8 @@
 name: Submodule PR
 on:
-    push:
+    pull_request:
         branches: [develop]
+        types: [closed]
 jobs:
   CanDIG-dispatch:
     runs-on: ubuntu-latest
@@ -10,21 +11,23 @@ jobs:
       CHECKOUT_BRANCH: 'develop'
       PR_AGAINST_BRANCH: 'develop'
       OWNER: 'CanDIG'
+    if: github.event.pull_request.merged == true
     steps:
             - name: Check out repository code
               uses: actions/checkout@v4
             - name: get PR data
-              uses: actions/github-script@v7
-              id: get_pr_data
-              with:
-                script: |
-                  return (
-                    await github.rest.repos.listPullRequestsAssociatedWithCommit({
-                      commit_sha: context.sha,
-                      owner: context.repo.owner,
-                      repo: context.repo.repo,
-                    })
-                  ).data[0];
+              shell: python
+              run: |
+                  import json
+                  import os
+                  with open('${{ github.event_path }}') as fh:
+                      event = json.load(fh)
+                      escaped = event['pull_request']['title'].replace("'", '"')
+                      pr_number = event["number"]
+                      print(escaped)    
+                  with open(os.environ['GITHUB_ENV'], 'a') as fh:
+                      print(f'PR_TITLE={escaped}', file=fh)
+                      print(f'PR_NUMBER={pr_number}', file=fh)
             - name: Create PR in CanDIGv2
               id: make_pr
               uses: CanDIG/github-action-pr-expanded@v4
@@ -33,7 +36,7 @@ jobs:
                   parent_repository: ${{ env.PARENT_REPOSITORY }}
                   checkout_branch: ${{ env.CHECKOUT_BRANCH}}
                   pr_against_branch: ${{ env.PR_AGAINST_BRANCH }}
-                  pr_title: '${{ github.repository }} merging: ${{ fromJson(steps.get_pr_data.outputs.result).title }}'
-                  pr_description: "PR triggered by update to develop branch on ${{ github.repository }}. Commit hash: `${{ github.sha }}`. PR link: [#${{ fromJson(steps.get_pr_data.outputs.result).number }}](https://github.com/${{ github.repository }}/pull/${{ fromJson(steps.get_pr_data.outputs.result).number }})"
+                  pr_title: "${{ github.repository }} merging: ${{ env.PR_TITLE }}"
+                  pr_description: "PR triggered by update to develop branch on ${{ github.repository }}. Commit hash: `${{ github.sha }}`. PR link: [#${{ env.PR_NUMBER }}](https://github.com/${{ github.repository }}/pull/${{ env.PR_NUMBER }})"
                   owner: ${{ env.OWNER }}
                   submodule_path: lib/katsu/katsu_service
diff --git a/.github/workflows/update-schema.yaml b/.github/workflows/update-schema.yaml
@@ -1,13 +1,14 @@
 name: Docs
 
 on:
-  push:
+  pull_request:
+    branches:
+      - develop
     paths:
-      - 'chord_metadata_service/mohpackets/schemas/**'
-      - 'chord_metadata_service/mohpackets/models.py'
-      - 'chord_metadata_service/mohpackets/docs/schema.json'
+      - "chord_metadata_service/mohpackets/schemas/**"
+      - "chord_metadata_service/mohpackets/models.py"
+      - "chord_metadata_service/mohpackets/permissible_values.py"
 
-
 jobs:
   generate-moh-schema:
     name: Update MoH schema
@@ -34,38 +35,37 @@ jobs:
         run: |
           export DJANGO_SETTINGS_MODULE=config.settings.base
           echo "CACHE_DURATION = 0" >> config/settings/base.py
-          python manage.py export_openapi_schema --api chord_metadata_service.mohpackets.apis.core.api | python -m json.tool > chord_metadata_service/mohpackets/docs/schema.json
-      
-      - name: Commit new schema.json
-        uses: stefanzweifel/git-auto-commit-action@v5
-        id: auto-commit-action
-        with:
-          commit_message: Update schema.json
-          file_pattern: 'chord_metadata_service/mohpackets/docs/schema.json'
-
-      - name: Update schema.json with new SHA
-        if: steps.auto-commit-action.outputs.changes_detected == 'true'
-        run: |
-          REPO_NAME=${{ github.repository }}
-          SHA=${{ steps.auto-commit-action.outputs.commit_hash }}
-          SCHEMA_PATH=chord_metadata_service/mohpackets/docs/schema.json
-          sed -i 's|"description": "This is the RESTful API for the MoH Service."|"description": "This is the RESTful API for the MoH Service. Based on https://raw.githubusercontent.com/'"$REPO_NAME"'/'"$SHA"'/'"$SCHEMA_PATH"'"|' $SCHEMA_PATH
+          echo "AGGREGATE_COUNT_THRESHOLD = 5" >> config/settings/base.py
+          python manage.py export_openapi_schema --api chord_metadata_service.mohpackets.apis.core.api | python -m json.tool > chord_metadata_service/mohpackets/docs/schemas/schema.json
 
       - name: Install widdershins
         run: npm install -g widdershins
 
       - name: Convert schema to OpenAPI documentation
         run: |
-          npx widdershins ./chord_metadata_service/mohpackets/docs/schema.json -o ./chord_metadata_service/mohpackets/docs/schema.md -u ./chord_metadata_service/mohpackets/docs/widdershins/templates/openapi3 -c true --omitHeader true
-      
+          npx widdershins ./chord_metadata_service/mohpackets/docs/schemas/schema.json -o ./chord_metadata_service/mohpackets/docs/schemas/schema.md -u ./chord_metadata_service/mohpackets/docs/widdershins/templates/openapi3 -c true --omitHeader true
+
       - name: Install PyYAML
         run: pip install PyYAML
 
       - name: Convert schema.json to schema.yml
-        run: python -c 'import json, yaml; json.load(open("chord_metadata_service/mohpackets/docs/schema.json")); print(yaml.dump(json.load(open("chord_metadata_service/mohpackets/docs/schema.json"))))' > chord_metadata_service/mohpackets/docs/schema.yml
+        run: python -c 'import json, yaml; json.load(open("chord_metadata_service/mohpackets/docs/schemas/schema.json")); print(yaml.dump(json.load(open("chord_metadata_service/mohpackets/docs/schemas/schema.json"))))' > chord_metadata_service/mohpackets/docs/schemas/schema.yml
+
+      - name: Update schemas
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: Update schemas
+          file_pattern: "chord_metadata_service/mohpackets/docs/schemas/schema.json chord_metadata_service/mohpackets/docs/schemas/schema.md chord_metadata_service/mohpackets/docs/schemas/schema.yml"
+
+      - name: Get SHA commit of schema.json
+        id: get-sha
+        run: echo "SHA=$(git log -1 --format=%H chord_metadata_service/mohpackets/docs/schemas/schema.json)" >> $GITHUB_ENV
+
+      - name: Write SHA to schema_version.txt
+        run: echo ${{ env.SHA }} > chord_metadata_service/mohpackets/docs/schema_version.txt
 
-      - name: Update schemas with SHA
+      - name: Commit updated schema_version.txt
         uses: stefanzweifel/git-auto-commit-action@v5
         with:
-          commit_message: Update schemas with SHA
-          file_pattern: 'chord_metadata_service/mohpackets/docs/schema.json chord_metadata_service/mohpackets/docs/schema.md chord_metadata_service/mohpackets/docs/schema.yml '
+          commit_message: Update schema version
+          file_pattern: "chord_metadata_service/mohpackets/docs/schema_version.txt"
diff --git a/.gitignore b/.gitignore
@@ -24,3 +24,5 @@ katsu.code-workspace
 
 # Staticfiles
 config/staticfiles/*
+
+chord_metadata_service/mohpackets/data/custom*
diff --git a/Dockerfile b/Dockerfile
@@ -9,24 +9,23 @@ USER root
 RUN groupadd -r candig && useradd -r -g candig candig
 
 RUN apt-get update && apt-get -y install \
-	postgresql-client
-
+    postgresql-client
+
+RUN mkdir -p /home/candig && chown -R candig:candig /home/candig
+
 RUN mkdir /app
 WORKDIR /app
 
 COPY ./requirements /app/requirements
 
 # Conditionally install dependencies based on the environment
-ARG katsu_env
-RUN if [ ${katsu_env} = "dev" ]; then \
-    echo "Installing dev.txt" && \
+ARG debug_mode 
+RUN if [ ${debug_mode} = 1 ]; then \
     pip install --no-cache-dir -r requirements/dev.txt; \
 else \
-    echo "Installing prod.txt" && \
     pip install --no-cache-dir -r requirements/prod.txt; \
 fi
 
-
 COPY . /app/chord_metadata_service
 
 WORKDIR /app/chord_metadata_service
@@ -41,4 +40,3 @@ RUN chmod +x /app/chord_metadata_service/create_db.sh
 USER candig
 
 CMD ["/app/chord_metadata_service/entrypoint.sh"]
-
diff --git a/README.md b/README.md
@@ -64,15 +64,49 @@ This will install all the packages needed for development.
 
 ### Step 4: Set up PostgreSQL
 
-Replace "dbname", "username", and "password" with the values in [local.py](config/settings/local.py):
+Install for macOS:
 
-```sql
+```bash
+# Install PostgreSQL
+brew install postgresql@16
+
+# Set PATH environment variable
+echo 'export PATH="/opt/homebrew/opt/postgresql@16/bin:$PATH"' >> ~/.zshrc
+source ~/.zshrc
+
+# Start PostgreSQL service
+brew services start postgresql@16
+
+# Log in to PostgreSQL
+psql -d postgres
+```
+
+Install for Linux:
+
+```bash
+# Install PostgreSQL
+sudo apt install postgresql
+
+# Start PostgreSQL service
+sudo systemctl start postgresql.service
+
+# Log in to PostgreSQL
 sudo -u postgres psql
-CREATE DATABASE dbname;
-CREATE USER username WITH PASSWORD 'password';
-GRANT ALL PRIVILEGES ON DATABASE dbname TO username;
 ```
 
+Create a Role and Database:
+
+```sql
+CREATE ROLE admin_local WITH LOGIN PASSWORD 'password_local' CREATEDB CREATEROLE;
+CREATE DATABASE katsu_local WITH OWNER = admin_local;
+GRANT ALL PRIVILEGES ON DATABASE katsu_local TO admin_local;
+
+-- Quit PostgreSQL
+\q
+```
+
+Note: In some cases, existing PostgreSQL might cause issues. You may need to clean them up manually.
+
 ### Step 5: Set up database
 
 With the database configured, run the following command to create the necessary database tables:
@@ -135,7 +169,6 @@ coverage html
 │       │   └── ingestion.py
 │       ├── data/
 │       │   ├── ...
-│       │   ├── data_converter.py
 │       │   ├── data_loader.py
 │       │   └── README.md
 │       ├── docs/
@@ -153,10 +186,12 @@ coverage html
 │       ├── tests/
 │       │   ├── endpoints/
 │       │   │   ├── base.py
-│       │   │   ├── factories.py
-│       │   │   ├── test_chemotherapy.py
+│       │   │   ├── test_donor.py
+│       │   │   └── ...
+│       │   ├── models/
+│       │   │   ├── test_biomarker.py
 │       │   │   └── ...
-│       │   └── test_models.py
+│       │   └── factories.py
 │       ├── apps.py
 │       ├── models.py
 │       ├── pagination.py
@@ -188,7 +223,7 @@ coverage html
 
 ## MOHCCN Clinical Data Model
 
-Katsu uses an underlying data model that is a compatible interpretation, but does not exactly match the MOHCCN data model. Katsu is currently compliant with version 2 of the model, released February 2023. Some relationships between objects have been modified to avoid excessive complexity in the katsu database and allow for the submission of data that is incomplete compared to the MOHCCN gold standard requirements. Permissable values for controlled fields are not validated by the underlying database.
+Katsu uses an underlying data model that is a compatible interpretation, but does not exactly match the MOHCCN data model. Katsu is currently compliant with version 3 of the model, released May 2024. Some relationships between objects have been modified to avoid excessive complexity in the katsu database and allow for the submission of data that is incomplete compared to the MOHCCN gold standard requirements. Permissable values for controlled fields are not validated by the underlying database.
 
 The katsu MoH model is explicitly defined as a set of classes in [models.py](chord_metadata_service/mohpackets/models.py). Permissable values for controlled fields, conditionally required fields and relationships are enforced by the [serializers](chord_metadata_service/mohpackets), [clinical ETL](https://github.com/CanDIG/clinical_ETL_code) and [ingest](https://github.com/CanDIG/candigv2-ingest) validation steps.
 
@@ -201,42 +236,20 @@ title: katsu object level MoH ER diagram
 erDiagram
 
 Program ||--o{ Donor : ""
-Program ||--o{ PrimaryDiagnosis : ""
-Program ||--o{ Comorbidity : ""
-Program ||--o{ Biomarker : ""
-Program ||--o{ Exposure : ""
-Program ||--o{ FollowUp : ""
-Program ||--o{ Specimen : ""
-Program ||--o{ Treatment : ""
-Program ||--o{ SampleRegistration : ""
-Program ||--o{ Chemotherapy : ""
-Program ||--o{ HormoneTherapy : ""
-Program ||--o{ Immunotherapy : ""
-Program ||--o{ Radiation : ""
-Program ||--o{ Surgery : ""
-Donor ||--o{ PrimaryDiagnosis : ""
 Donor ||--o{ Comorbidity : ""
 Donor ||--o{ Biomarker : "" 
 Donor ||--o{ Exposure : "" 
 Donor ||--o{ FollowUp : "" 
-Donor ||--o{ Specimen : "" 
+Donor ||--o{ PrimaryDiagnosis : ""
 Donor ||--o{ Treatment : "" 
-Donor ||--o{ SampleRegistration : "" 
-Donor ||--o{ Chemotherapy : "" 
-Donor ||--o{ HormoneTherapy : "" 
-Donor ||--o{ Immunotherapy : "" 
-Donor ||--o{ Radiation : "" 
-Donor ||--o{ Surgery : "" 
 PrimaryDiagnosis ||--o{ Specimen : "" 
 PrimaryDiagnosis ||--o{ Treatment : "" 
-PrimaryDiagnosis ||--o{ FollowUp : "" 
+PrimaryDiagnosis o|--o{ FollowUp : ""
 Specimen ||--o{ SampleRegistration : "" 
-Treatment ||--o{ Chemotherapy : "" 
-Treatment ||--o{ HormoneTherapy : "" 
-Treatment ||--o{ Immunotherapy : "" 
 Treatment ||--o| Radiation : "" 
 Treatment ||--o| Surgery : "" 
-Treatment ||--o{ FollowUp : "" 
+Treatment ||--o{ Systemictherapy : "" 
+Treatment o|--o{ FollowUp : "" 
 
 ```
 
@@ -253,9 +266,9 @@ Treatment ||--o{ FollowUp : ""
 
 ### References
 
-[Clinical Data Model](https://www.marathonofhopecancercentres.ca/docs/default-source/policies-and-guidelines/mohccn-clinical-data-model_v1_endorsed6oct-2022.pdf?Status=Master&sfvrsn=7f6bd159_7)
+[Clinical Data Model](https://www.marathonofhopecancercentres.ca/docs/default-source/policies-and-guidelines/clinical-data-model-v3/mohccn_clinical_data_modelv3-0_may2024.xlsx?sfvrsn=80e1f917_1)
 
-[ER Diagram](https://www.marathonofhopecancercentres.ca/docs/default-source/policies-and-guidelines/mohccn_data_standard_er_diagram_endorsed6oct22.pdf?Status=Master&sfvrsn=dd57a75e_5)
+[ER Diagram](https://www.marathonofhopecancercentres.ca/docs/default-source/policies-and-guidelines/clinical-data-model-v3/mohccn_clinical_data_modelv3-0_er_diagram_may2024.png?sfvrsn=cdbcd90b_1)
 
 ## Authentication and Authorization
Original file line number	Diff line number	Diff line change
Expand Up		@@ -24,3 +24,5 @@ katsu.code-workspace

		# Staticfiles
		config/staticfiles/*

		chord_metadata_service/mohpackets/data/custom*