From b00686958c00b753f4aca4fdb2d5e9b1b1d8c98f Mon Sep 17 00:00:00 2001 From: Michael Folz Date: Fri, 15 Nov 2024 10:56:57 +0100 Subject: [PATCH 01/10] No issue - bump version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99b634e1..8ba4b1aa 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ de.medizininformatik-initiative DataportalBackend - 6.0.0-alpha.6 + 6.0.0-SNAPSHOT Dataportal Backend Backend of the Dataportal From e09b853246a8a366c9eeea05d2c4d495562c55ba Mon Sep 17 00:00:00 2001 From: Michael Folz Date: Mon, 18 Nov 2024 15:20:15 +0100 Subject: [PATCH 02/10] #398 - Add referencedProfiles string array to dse profile - add "referencedProfiles" to dse Field class --- .../java/de/numcodex/feasibility_gui_backend/dse/api/Field.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/de/numcodex/feasibility_gui_backend/dse/api/Field.java b/src/main/java/de/numcodex/feasibility_gui_backend/dse/api/Field.java index dce952a5..184f2c26 100644 --- a/src/main/java/de/numcodex/feasibility_gui_backend/dse/api/Field.java +++ b/src/main/java/de/numcodex/feasibility_gui_backend/dse/api/Field.java @@ -12,6 +12,7 @@ public record Field( @JsonProperty String id, @JsonProperty DisplayEntry display, @JsonProperty DisplayEntry description, + @JsonProperty List referencedProfiles, @JsonProperty String type, @JsonProperty boolean recommended, @JsonProperty boolean required, From 73e5740eb8954ecf9390cb25a5597374957f40ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gr=C3=BCndner?= Date: Tue, 19 Nov 2024 19:18:30 +0100 Subject: [PATCH 03/10] update to develop 4 --- docker-compose.yml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1e01ce85..ee0836b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,7 +139,7 @@ services: environment: ES_HOST: http://dataportal-elastic ES_PORT: 9200 - ONTO_GIT_TAG: v3.0.0-alpha.1 + ONTO_GIT_TAG: develop.4 ONTO_REPO: https://github.com/medizininformatik-initiative/fhir-ontology-generator/raw/ ONTO_RELATIVE_PATH: /example/fdpg-ontology/ DOWNLOAD_FILENAME: elastic.zip diff --git a/pom.xml b/pom.xml index 8ba4b1aa..f8f4e009 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 17 4.10.0 4.10.0 - v3.0.0-alpha.1 + develop.4 From 107df5443433a3e3f8e0f2ccd3ab6f360657c5ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gr=C3=BCndner?= Date: Wed, 20 Nov 2024 20:29:36 +0100 Subject: [PATCH 04/10] Update Ontology to v3.0.0 --- docker-compose.yml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ee0836b4..04e1205f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,7 +139,7 @@ services: environment: ES_HOST: http://dataportal-elastic ES_PORT: 9200 - ONTO_GIT_TAG: develop.4 + ONTO_GIT_TAG: v3.0.0 ONTO_REPO: https://github.com/medizininformatik-initiative/fhir-ontology-generator/raw/ ONTO_RELATIVE_PATH: /example/fdpg-ontology/ DOWNLOAD_FILENAME: elastic.zip diff --git a/pom.xml b/pom.xml index f8f4e009..5ff02438 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 17 4.10.0 4.10.0 - develop.4 + v3.0.0 From 8697602fe147acf511f9ab633037905dd80e8197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gr=C3=BCndner?= Date: Wed, 20 Nov 2024 21:21:54 +0100 Subject: [PATCH 05/10] bump sq2cql version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5ff02438..5a0534cd 100644 --- a/pom.xml +++ b/pom.xml @@ -220,7 +220,7 @@ de.medizininformatik-initiative sq2cql - v0.5.0-alpha.1 + 0.5.0 From de1a605b89b588985ae31eb6567a5aff0bfa10cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gr=C3=BCndner?= Date: Wed, 20 Nov 2024 21:55:56 +0100 Subject: [PATCH 06/10] new-dev --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8bfb4c9b..0b977d98 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ de.medizininformatik-initiative DataportalBackend - 6.0.0 + 6.1.0-SNAPSHOT Dataportal Backend Backend of the Dataportal From 5e401f3a003153d3c579a2703e6b8edca73c5736 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 25 Nov 2024 13:06:34 +0100 Subject: [PATCH 07/10] #403 - Update spring boot to 3.4.0 (#404) * #403 - Update spring boot to 3.4.0 - update spring boot to 3.4.0 - remove manual inclusion of nimbus-jose-jwt - adapt to changes in elastic search client --- pom.xml | 15 +-------------- .../terminology/es/TerminologyEsService.java | 18 +++++++----------- .../es/CodeableConceptServiceTest.java | 2 +- .../es/TerminologyEsServiceTest.java | 2 +- 4 files changed, 10 insertions(+), 27 deletions(-) diff --git a/pom.xml b/pom.xml index 0b977d98..da9a155b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.springframework.boot spring-boot-starter-parent - 3.3.5 + 3.4.0 @@ -72,7 +72,6 @@ spring-boot-starter-validation - org.springframework.boot spring-boot-starter-data-elasticsearch @@ -81,18 +80,6 @@ org.springframework.boot spring-boot-starter-oauth2-resource-server - - - com.nimbusds - nimbus-jose-jwt - - - - - - com.nimbusds - nimbus-jose-jwt - 9.46 diff --git a/src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsService.java b/src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsService.java index 6c54ee55..ee346a7e 100644 --- a/src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsService.java +++ b/src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsService.java @@ -1,12 +1,10 @@ package de.numcodex.feasibility_gui_backend.terminology.es; import co.elastic.clients.elasticsearch._types.FieldValue; -import co.elastic.clients.elasticsearch._types.InlineScript; import co.elastic.clients.elasticsearch._types.Script; import co.elastic.clients.elasticsearch._types.aggregations.Aggregation; import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket; import co.elastic.clients.elasticsearch._types.query_dsl.*; -import co.elastic.clients.json.JsonData; import de.numcodex.feasibility_gui_backend.terminology.api.EsSearchResult; import de.numcodex.feasibility_gui_backend.terminology.api.EsSearchResultEntry; import de.numcodex.feasibility_gui_backend.terminology.es.model.*; @@ -121,10 +119,12 @@ private SearchHits findByNameOrTermcode(String keyword List filterTerms = new ArrayList<>(); if (availability) { - var availabilityFilter = new RangeQuery.Builder() - .field("availability") - .gt(JsonData.of("0")) - .build(); + var availabilityFilter = RangeQuery.of(r -> r + .number(n -> n + .field("availability") + .gt(0.0) + ) + ); filterTerms.add(availabilityFilter._toQuery()); } @@ -164,12 +164,8 @@ private SearchHits findByNameOrTermcode(String keyword .withPageable(pageRequest) .build(); - var inlineScript = new InlineScript.Builder() - .source("doc['availability'].value == 0 ? _score : _score + 100") - .build(); - var availabilityScoreScript = new Script.Builder() - .inline(inlineScript) + .source("doc['availability'].value == 0 ? _score : _score + 100") .build(); var function = FunctionScoreBuilders.scriptScore() diff --git a/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/CodeableConceptServiceTest.java b/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/CodeableConceptServiceTest.java index 0a8b6bc0..5daa7d1b 100644 --- a/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/CodeableConceptServiceTest.java +++ b/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/CodeableConceptServiceTest.java @@ -143,7 +143,7 @@ private SearchHits createDummySearchHitsPage(int totalH ) ); } - return new SearchHitsImpl<>(totalHits, TotalHitsRelation.OFF, 10.0F, null, null, searchHitsList, null, null, null); + return new SearchHitsImpl<>(totalHits, TotalHitsRelation.OFF, 10.0F, null, null, null, searchHitsList, null, null, null); } private CodeableConceptDocument createDummyCodeableConceptDocument(String id) { diff --git a/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsServiceTest.java b/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsServiceTest.java index b71ed0ec..c8f3e12f 100644 --- a/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsServiceTest.java +++ b/src/test/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsServiceTest.java @@ -303,6 +303,6 @@ private SearchHits createDummySearchHitsPage(int total ) ); } - return new SearchHitsImpl<>(totalHits, TotalHitsRelation.OFF, 10.0F, null, null, searchHitsList, null, null, null); + return new SearchHitsImpl<>(totalHits, TotalHitsRelation.OFF, 10.0F, null, null, null, searchHitsList, null, null, null); } } From 6233eb86999c0fa63e0149d544b14e031a0c7ec7 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 28 Nov 2024 14:23:53 +0100 Subject: [PATCH 08/10] #401 - Update sq2cql and validate time restrictions (#402) * #401 - Update sq2cql and validate time restrictions - update sq2cql to 0.6.0 - check if timerestrictions are correct (before date is after afterdate) - add new error code for invalid timerestrictin --- pom.xml | 2 +- .../query/api/status/ValidationIssue.java | 3 +- .../validation/StructuredQueryValidation.java | 20 ++++++++ .../StructuredQueryValidationTest.java | 51 +++++++++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index da9a155b..71513e90 100644 --- a/pom.xml +++ b/pom.xml @@ -213,7 +213,7 @@ de.medizininformatik-initiative sq2cql - 0.5.0 + 0.6.0 diff --git a/src/main/java/de/numcodex/feasibility_gui_backend/query/api/status/ValidationIssue.java b/src/main/java/de/numcodex/feasibility_gui_backend/query/api/status/ValidationIssue.java index f42212a1..6692323c 100644 --- a/src/main/java/de/numcodex/feasibility_gui_backend/query/api/status/ValidationIssue.java +++ b/src/main/java/de/numcodex/feasibility_gui_backend/query/api/status/ValidationIssue.java @@ -7,7 +7,8 @@ @Slf4j @JsonSerialize(using = ValidationIssueSerializer.class) public enum ValidationIssue { - TERMCODE_CONTEXT_COMBINATION_INVALID(20001, "The combination of context and termcode(s) is not found."); + TERMCODE_CONTEXT_COMBINATION_INVALID(20001, "The combination of context and termcode(s) is not found."), + TIMERESTRICTION_INVALID(20002, "The TimeRestriction is invalid. 'beforeDate' must not be before 'afterDate'"); private static final ValidationIssue[] VALUES; diff --git a/src/main/java/de/numcodex/feasibility_gui_backend/terminology/validation/StructuredQueryValidation.java b/src/main/java/de/numcodex/feasibility_gui_backend/terminology/validation/StructuredQueryValidation.java index a1cf677b..d688580b 100644 --- a/src/main/java/de/numcodex/feasibility_gui_backend/terminology/validation/StructuredQueryValidation.java +++ b/src/main/java/de/numcodex/feasibility_gui_backend/terminology/validation/StructuredQueryValidation.java @@ -5,8 +5,11 @@ import de.numcodex.feasibility_gui_backend.common.api.TermCode; import de.numcodex.feasibility_gui_backend.query.api.MutableStructuredQuery; import de.numcodex.feasibility_gui_backend.query.api.StructuredQuery; +import de.numcodex.feasibility_gui_backend.query.api.TimeRestriction; import de.numcodex.feasibility_gui_backend.query.api.status.ValidationIssue; import de.numcodex.feasibility_gui_backend.terminology.TerminologyService; + +import java.time.LocalDate; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -86,6 +89,10 @@ private void annotateCriteria(List criteria, boolean skipValid criterion.setValidationIssues(List.of(ValidationIssue.TERMCODE_CONTEXT_COMBINATION_INVALID)); continue; } + if (isTimeRestrictionInvalid(criterion.getTimeRestriction())) { + criterion.setValidationIssues(List.of(ValidationIssue.TIMERESTRICTION_INVALID)); + continue; + } for (TermCode termCode : criterion.getTermCodes()) { if (terminologyService.isExistingTermCode(termCode.system(), termCode.code())) { log.trace("termcode ok: {} - {}", termCode.system(), termCode.code()); @@ -103,6 +110,11 @@ private boolean containsInvalidCriteria(List criteria) { if (criterion.context() == null) { return true; } + if (isTimeRestrictionInvalid(criterion.timeRestriction())) { + log.debug("TimeRestriction invalid. 'beforeDate' ({}) must be after 'afterDate' ({}) but is not", + criterion.timeRestriction().beforeDate(), criterion.timeRestriction().afterDate()); + return true; + } for (TermCode termCode : criterion.termCodes()) { if (terminologyService.isExistingTermCode(termCode.system(), termCode.code())) { log.trace("termcode ok: {} - {}", termCode.system(), termCode.code()); @@ -115,4 +127,12 @@ private boolean containsInvalidCriteria(List criteria) { } return false; } + + private boolean isTimeRestrictionInvalid(TimeRestriction timeRestriction) { + // If no timeRestriction is set, it is not invalid + if (timeRestriction == null) { + return false; + } + return LocalDate.parse(timeRestriction.beforeDate()).isBefore(LocalDate.parse(timeRestriction.afterDate())); + } } diff --git a/src/test/java/de/numcodex/feasibility_gui_backend/terminology/validation/StructuredQueryValidationTest.java b/src/test/java/de/numcodex/feasibility_gui_backend/terminology/validation/StructuredQueryValidationTest.java index aaddc12a..e4ab71b5 100644 --- a/src/test/java/de/numcodex/feasibility_gui_backend/terminology/validation/StructuredQueryValidationTest.java +++ b/src/test/java/de/numcodex/feasibility_gui_backend/terminology/validation/StructuredQueryValidationTest.java @@ -3,6 +3,7 @@ import de.numcodex.feasibility_gui_backend.common.api.Criterion; import de.numcodex.feasibility_gui_backend.common.api.TermCode; import de.numcodex.feasibility_gui_backend.query.api.StructuredQuery; +import de.numcodex.feasibility_gui_backend.query.api.TimeRestriction; import de.numcodex.feasibility_gui_backend.terminology.TerminologyService; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeEach; @@ -62,6 +63,13 @@ void testIsValid_falseOnMissingContext() { assertFalse(isValid); } + @Test + void testIsValid_falseOnInvalidTimeRestriction() { + var isValid = structuredQueryValidation.isValid(createStructuredQueryWithInvalidTimeRestriction()); + + assertFalse(isValid); + } + @ParameterizedTest @CsvSource({"true,true", "true,false", "false,true", "false,false"}) void testAnnotateStructuredQuery_emptyIssuesOnValidCriteriaOrSkippedValidation(String withExclusionCriteriaString, String skipValidationString) { @@ -106,6 +114,18 @@ void testAnnotateStructuredQuery_nonEmptyIssuesOnMissingContext(boolean skipVali } } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testAnnotateStructuredQuery_nonEmptyIssuesOnInvalidTimeRestriction(boolean skipValidation) { + var annotatedStructuredQuery = structuredQueryValidation.annotateStructuredQuery(createStructuredQueryWithInvalidTimeRestriction(), skipValidation); + + if (skipValidation) { + assertTrue(annotatedStructuredQuery.inclusionCriteria().get(0).get(0).validationIssues().isEmpty()); + } else { + assertFalse(annotatedStructuredQuery.inclusionCriteria().get(0).get(0).validationIssues().isEmpty()); + } + } + @NotNull private static StructuredQuery createValidStructuredQuery(boolean withExclusionCriteria) { var context = TermCode.builder() @@ -150,4 +170,35 @@ private static StructuredQuery createStructuredQueryWithoutContext() { .display("foo") .build(); } + + @NotNull + private static StructuredQuery createStructuredQueryWithInvalidTimeRestriction() { + var context = TermCode.builder() + .code("Laboruntersuchung") + .system("fdpg.mii.cds") + .display("Laboruntersuchung") + .version("1.0.0") + .build(); + var termCode = TermCode.builder() + .code("19113-0") + .system("http://loinc.org") + .display("IgE") + .build(); + var timeRestriction = TimeRestriction.builder() + .afterDate("1998-05-09") + .beforeDate("1991-06-15") + .build(); + var criterion = Criterion.builder() + .termCodes(List.of(termCode)) + .context(context) + .attributeFilters(List.of()) + .timeRestriction(timeRestriction) + .build(); + return StructuredQuery.builder() + .version(URI.create("http://to_be_decided.com/draft-2/schema#")) + .inclusionCriteria(List.of(List.of(criterion))) + .exclusionCriteria(List.of()) + .display("foo") + .build(); + } } From d3480432b835fdadb201e1a9d9631607700c79f1 Mon Sep 17 00:00:00 2001 From: Michael Folz Date: Thu, 28 Nov 2024 14:31:36 +0100 Subject: [PATCH 09/10] #405 - Fix code scanning alert - org.hl7.fhir.convertors: org.hl7.fhir.dstu2: org.hl7.fhir.dstu2016may: org.hl7.fhir.dstu3... MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - update fhir-structures-r4 - uüdate mockwebserver --- pom.xml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 71513e90..d9322547 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 17 - 4.10.0 + 4.12.0 4.10.0 v3.0.0 @@ -159,7 +159,7 @@ ca.uhn.hapi.fhir hapi-fhir-structures-r4 - 7.4.5 + 7.6.0 @@ -168,19 +168,12 @@ 6.4.0 - - ca.uhn.hapi.fhir - org.hl7.fhir.utilities - 6.3.26 - - com.bucket4j bucket4j-core 8.10.1 -