From 56121a4ff749ed3b4a0d6c79cdc061da4632c701 Mon Sep 17 00:00:00 2001 From: Son Chau Date: Mon, 27 Nov 2023 10:17:10 -0800 Subject: [PATCH 1/5] Update test_integration.py --- etc/tests/test_integration.py | 232 +++++++++++++++------------------- 1 file changed, 103 insertions(+), 129 deletions(-) diff --git a/etc/tests/test_integration.py b/etc/tests/test_integration.py index 0c0381618..c4d38f5a5 100644 --- a/etc/tests/test_integration.py +++ b/etc/tests/test_integration.py @@ -231,14 +231,14 @@ def assert_datasets_should_not_exist(datasets): If any of the dataset names is found, the assertion will fail. """ response = requests.get( - f"{ENV['CANDIG_URL']}/katsu/v2/discovery/donors/", headers=get_headers() + f"{ENV['CANDIG_URL']}/katsu/v2/discovery/programs/", headers=get_headers() ) data = response.json() - dataset_names = list(data["discovery_donor"].keys()) + dataset_names = data["cohort_list"] assert all( dataset_name not in dataset_names for dataset_name in datasets - ), f"Expected none of {datasets} to exist, but at least one was found." + ), f"Expected none of {datasets} to exist, but at least one was found. Attempt to clean up. Please run the test again" def ingest_data(endpoint, data, is_admin=False): @@ -269,7 +269,7 @@ def perform_ingest_and_assert_status(endpoint, data, is_admin=False): Performs data ingest and asserts the response status code depend on permission """ response = ingest_data(endpoint, data, is_admin) - expected_status = HTTPStatus.CREATED if is_admin else HTTPStatus.FORBIDDEN + expected_status = HTTPStatus.CREATED if is_admin else HTTPStatus.UNAUTHORIZED assert_ingest_response_status(response, expected_status, endpoint) @@ -278,7 +278,7 @@ def clean_up_program(test_id): Deletes a dataset and all related objects. Expected 204 """ delete_response = requests.delete( - f"{ENV['CANDIG_URL']}/katsu/v2/authorized/programs/{test_id}/", + f"{ENV['CANDIG_URL']}/katsu/v2/authorized/program/{test_id}/", headers=get_headers(is_admin=True), ) assert ( @@ -288,180 +288,152 @@ def clean_up_program(test_id): def check_program_ingest(test_id, is_admin=False): - endpoint = "programs" - data = [{"program_id": test_id}] + endpoint = "program" + data = {"program_id": test_id} perform_ingest_and_assert_status(endpoint, data, is_admin) def check_donor_ingest(test_id, is_admin=False): - endpoint = "donors" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - }, - ] + endpoint = "donor" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_diagnosis_ingest(test_id, is_admin=False): - endpoint = "primary_diagnoses" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_primary_diagnosis_id": test_id, - }, - ] + endpoint = "primary_diagnosis" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_primary_diagnosis_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_specimen_ingest(test_id, is_admin=False): - endpoint = "specimens" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_primary_diagnosis_id": test_id, - "submitter_specimen_id": test_id, - }, - ] + endpoint = "specimen" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_primary_diagnosis_id": test_id, + "submitter_specimen_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_sample_ingest(test_id, is_admin=False): - endpoint = "sample_registrations" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_specimen_id": test_id, - "submitter_sample_id": test_id, - }, - ] + endpoint = "sample_registration" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_specimen_id": test_id, + "submitter_sample_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_treatment_ingest(test_id, is_admin=False): - endpoint = "treatments" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_primary_diagnosis_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "treatment" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_primary_diagnosis_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_chemotherapy_ingest(test_id, is_admin=False): - endpoint = "chemotherapies" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "chemotherapy" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_radiation_ingest(test_id, is_admin=False): - endpoint = "radiations" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "radiation" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_hormonetherapy_ingest(test_id, is_admin=False): - endpoint = "hormone_therapies" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "hormone_therapy" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_immunotherapy_ingest(test_id, is_admin=False): - endpoint = "immunotherapies" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "immunotherapy" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_surgery_ingest(test_id, is_admin=False): - endpoint = "surgeries" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - "margin_types_involved": [], - "margin_types_not_involved": [], - "margin_types_not_assessed": [], - }, - ] + endpoint = "surgery" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + "margin_types_involved": [], + "margin_types_not_involved": [], + "margin_types_not_assessed": [], + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_follow_up_ingest(test_id, is_admin=False): - endpoint = "follow_ups" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_primary_diagnosis_id": test_id, - "submitter_treatment_id": test_id, - "submitter_follow_up_id": test_id, - }, - ] + endpoint = "follow_up" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_primary_diagnosis_id": test_id, + "submitter_treatment_id": test_id, + "submitter_follow_up_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_biomarker_ingest(test_id, is_admin=False): - endpoint = "biomarkers" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - }, - ] + endpoint = "biomarker" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_comorbidity_ingest(test_id, is_admin=False): - endpoint = "comorbidities" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - }, - ] + endpoint = "comorbidity" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_exposure_ingest(test_id, is_admin=False): - endpoint = "exposures" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - }, - ] + endpoint = "exposure" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) @@ -473,7 +445,7 @@ def check_datasets_access(is_admin, authorized_datasets, unauthorized_datasets): f"{ENV['CANDIG_URL']}/katsu/v2/authorized/programs/", headers=get_headers(is_admin=is_admin), ) - programs = list(map(lambda x: x["program_id"], response.json()["results"])) + programs = list(map(lambda x: x["program_id"], response.json()["items"])) # Assert that all authorized datasets are present in programs assert all( @@ -551,7 +523,7 @@ def test_unauthorized_ingests(): - Attempt to clean up after in case any ingests go through but expected None Expected result: - - HTTP 403 even with valid ingest data. + - HTTP 401 even with valid ingest data. """ # to simplify the test data, only 1 unique id is needed test_id = "TEST-" + str(uuid.uuid4()) @@ -609,10 +581,12 @@ def test_katsu_users_data_access(): assert_datasets_should_not_exist(synthetic_datasets) # create synthetic datasets that matches OPA access - endpoint = "programs" - program_data = [{"program_id": dataset_id} for dataset_id in synthetic_datasets] - response = ingest_data(endpoint, program_data, is_admin=True) - assert response.status_code == HTTPStatus.CREATED, "Failed to create programs." + endpoint = "program" + program_data_list = [{"program_id": dataset_id} for dataset_id in synthetic_datasets] + for program_data in program_data_list: + response = ingest_data(endpoint, program_data, is_admin=True) + assert response.status_code == HTTPStatus.CREATED, "Failed to create program." + # Assert access for admin user check_datasets_access( From 8a4e76e56a87520a4a97157186fdb8d9efa0ccfa Mon Sep 17 00:00:00 2001 From: Daisie Huang Date: Thu, 30 Nov 2023 12:02:22 -0800 Subject: [PATCH 2/5] Update katsu_service --- lib/katsu/katsu_service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/katsu/katsu_service b/lib/katsu/katsu_service index 0f82bdc31..6675eacc6 160000 --- a/lib/katsu/katsu_service +++ b/lib/katsu/katsu_service @@ -1 +1 @@ -Subproject commit 0f82bdc314230149e980322fd20132a189c7bb0f +Subproject commit 6675eacc60ac1d7eddbff28648ec1e0a898df655 From 724e28a0d7bc46e69308a41f6068af0eccac9b81 Mon Sep 17 00:00:00 2001 From: Daisie Huang Date: Tue, 5 Dec 2023 10:12:34 -0800 Subject: [PATCH 3/5] Update test_integration.py --- etc/tests/test_integration.py | 237 ++++++++++++++++------------------ 1 file changed, 108 insertions(+), 129 deletions(-) diff --git a/etc/tests/test_integration.py b/etc/tests/test_integration.py index cb54dd90e..c82b4263b 100644 --- a/etc/tests/test_integration.py +++ b/etc/tests/test_integration.py @@ -262,7 +262,8 @@ def assert_datasets_should_not_exist(datasets): f"{ENV['CANDIG_URL']}/katsu/v2/discovery/donors/", headers=get_headers() ) data = response.json() - dataset_names = list(data["discovery_donor"].keys()) + print(data) + dataset_names = list(data["donors_by_cohort"].keys()) assert all( dataset_name not in dataset_names for dataset_name in datasets @@ -274,6 +275,7 @@ def ingest_data(endpoint, data, is_admin=False): Ingests data into the katsu API at the specified endpoint using a POST request. """ headers = get_headers(is_admin) + print(json.dumps(data)) response = requests.post( f"{ENV['CANDIG_URL']}/katsu/v2/ingest/{endpoint}/", headers=headers, @@ -297,7 +299,7 @@ def perform_ingest_and_assert_status(endpoint, data, is_admin=False): Performs data ingest and asserts the response status code depend on permission """ response = ingest_data(endpoint, data, is_admin) - expected_status = HTTPStatus.CREATED if is_admin else HTTPStatus.FORBIDDEN + expected_status = HTTPStatus.CREATED if is_admin else HTTPStatus.UNAUTHORIZED assert_ingest_response_status(response, expected_status, endpoint) @@ -316,180 +318,152 @@ def clean_up_program(test_id): def check_program_ingest(test_id, is_admin=False): - endpoint = "programs" - data = [{"program_id": test_id}] + endpoint = "program" + data = {"program_id": test_id} perform_ingest_and_assert_status(endpoint, data, is_admin) def check_donor_ingest(test_id, is_admin=False): - endpoint = "donors" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - }, - ] + endpoint = "donor" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_diagnosis_ingest(test_id, is_admin=False): - endpoint = "primary_diagnoses" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_primary_diagnosis_id": test_id, - }, - ] + endpoint = "primary_diagnosis" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_primary_diagnosis_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_specimen_ingest(test_id, is_admin=False): - endpoint = "specimens" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_primary_diagnosis_id": test_id, - "submitter_specimen_id": test_id, - }, - ] + endpoint = "specimen" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_primary_diagnosis_id": test_id, + "submitter_specimen_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_sample_ingest(test_id, is_admin=False): - endpoint = "sample_registrations" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_specimen_id": test_id, - "submitter_sample_id": test_id, - }, - ] + endpoint = "sample_registration" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_specimen_id": test_id, + "submitter_sample_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_treatment_ingest(test_id, is_admin=False): - endpoint = "treatments" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_primary_diagnosis_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "treatment" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_primary_diagnosis_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_chemotherapy_ingest(test_id, is_admin=False): - endpoint = "chemotherapies" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "chemotherapy" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_radiation_ingest(test_id, is_admin=False): - endpoint = "radiations" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "radiation" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_hormonetherapy_ingest(test_id, is_admin=False): - endpoint = "hormone_therapies" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "hormone_therapy" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_immunotherapy_ingest(test_id, is_admin=False): - endpoint = "immunotherapies" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - }, - ] + endpoint = "immunotherapy" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_surgery_ingest(test_id, is_admin=False): - endpoint = "surgeries" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_treatment_id": test_id, - "margin_types_involved": [], - "margin_types_not_involved": [], - "margin_types_not_assessed": [], - }, - ] + endpoint = "surgery" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_treatment_id": test_id, + "margin_types_involved": [], + "margin_types_not_involved": [], + "margin_types_not_assessed": [], + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_follow_up_ingest(test_id, is_admin=False): - endpoint = "follow_ups" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - "submitter_primary_diagnosis_id": test_id, - "submitter_treatment_id": test_id, - "submitter_follow_up_id": test_id, - }, - ] + endpoint = "follow_up" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + "submitter_primary_diagnosis_id": test_id, + "submitter_treatment_id": test_id, + "submitter_follow_up_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_biomarker_ingest(test_id, is_admin=False): - endpoint = "biomarkers" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - }, - ] + endpoint = "biomarker" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_comorbidity_ingest(test_id, is_admin=False): - endpoint = "comorbidities" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - }, - ] + endpoint = "comorbidity" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) def check_exposure_ingest(test_id, is_admin=False): - endpoint = "exposures" - data = [ - { - "submitter_donor_id": test_id, - "program_id": test_id, - }, - ] + endpoint = "exposure" + data = { + "submitter_donor_id": test_id, + "program_id": test_id, + } perform_ingest_and_assert_status(endpoint, data, is_admin) @@ -501,7 +475,8 @@ def check_datasets_access(is_admin, authorized_datasets, unauthorized_datasets): f"{ENV['CANDIG_URL']}/katsu/v2/authorized/programs/", headers=get_headers(is_admin=is_admin), ) - programs = list(map(lambda x: x["program_id"], response.json()["results"])) + print(response.json()) + programs = list(map(lambda x: x["program_id"], response.json()["items"])) # Assert that all authorized datasets are present in programs assert all( @@ -637,10 +612,12 @@ def test_katsu_users_data_access(): assert_datasets_should_not_exist(synthetic_datasets) # create synthetic datasets that matches OPA access - endpoint = "programs" - program_data = [{"program_id": dataset_id} for dataset_id in synthetic_datasets] - response = ingest_data(endpoint, program_data, is_admin=True) - assert response.status_code == HTTPStatus.CREATED, "Failed to create programs." + endpoint = "program" + program_data_list = [{"program_id": dataset_id} for dataset_id in synthetic_datasets] + for program_data in program_data_list: + response = ingest_data(endpoint, program_data, is_admin=True) + print(response.json()) + assert response.status_code == HTTPStatus.CREATED, "Failed to create program." # Assert access for admin user check_datasets_access( @@ -665,6 +642,7 @@ def test_katsu_users_data_access(): def test_ingest_permissions(): clean_up_program("SYNTHETIC-2") clean_up_program("TEST_2") + assert_datasets_should_not_exist(["SYNTHETIC-2", "TEST_2"]) test_loc = "https://raw.githubusercontent.com/CanDIG/candigv2-ingest/develop/tests/clinical_ingest.json" test_data = requests.get(test_loc).json() @@ -678,10 +656,10 @@ def test_ingest_permissions(): "Content-Type": "application/json; charset=utf-8", } - response = requests.post(f"{ENV['CANDIG_URL']}/ingest/clinical", headers=headers, json=test_data) - # when the user has no admin access, they should not be allowed - assert response.status_code == 403 - + # response = requests.post(f"{ENV['CANDIG_URL']}/ingest/clinical", headers=headers, json=test_data) + # # when the user has no admin access, they should not be allowed + # assert_datasets_should_not_exist(["SYNTHETIC-2", "TEST_2"]) + # assert response.status_code == 401 token = get_token( username=ENV["CANDIG_SITE_ADMIN_USER"], password=ENV["CANDIG_SITE_ADMIN_PASSWORD"], @@ -701,6 +679,7 @@ def test_ingest_permissions(): clean_up_program("SYNTHETIC-2") clean_up_program("TEST_2") + assert_datasets_should_not_exist(["SYNTHETIC-2", "TEST_2"]) ## Htsget tests: From 4c90f2015664a7b327539e80b7b13e5c779e69ed Mon Sep 17 00:00:00 2001 From: Daisie Huang Date: Tue, 5 Dec 2023 10:24:07 -0800 Subject: [PATCH 4/5] Update candigv2-ingest --- lib/candig-ingest/candigv2-ingest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/candig-ingest/candigv2-ingest b/lib/candig-ingest/candigv2-ingest index 11911ea84..8822ca243 160000 --- a/lib/candig-ingest/candigv2-ingest +++ b/lib/candig-ingest/candigv2-ingest @@ -1 +1 @@ -Subproject commit 11911ea8466f41aeed8e7a480cc7b0d9ecab1f34 +Subproject commit 8822ca243fca96906259c8201da652c7d3827738 From b887d19b5858ab1c80811826ca8510a6f1a6197b Mon Sep 17 00:00:00 2001 From: Daisie Huang Date: Tue, 5 Dec 2023 10:24:22 -0800 Subject: [PATCH 5/5] update error --- etc/tests/test_integration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/tests/test_integration.py b/etc/tests/test_integration.py index c4d38f5a5..2b063dedd 100644 --- a/etc/tests/test_integration.py +++ b/etc/tests/test_integration.py @@ -586,7 +586,7 @@ def test_katsu_users_data_access(): for program_data in program_data_list: response = ingest_data(endpoint, program_data, is_admin=True) assert response.status_code == HTTPStatus.CREATED, "Failed to create program." - + # Assert access for admin user check_datasets_access( @@ -626,7 +626,7 @@ def test_ingest_permissions(): response = requests.post(f"{ENV['CANDIG_URL']}/ingest/clinical", headers=headers, json=test_data) # when the user has no admin access, they should not be allowed - assert response.status_code == 403 + assert response.status_code == 401 token = get_token( username=ENV["CANDIG_SITE_ADMIN_USER"],