Skip to content

Commit

Permalink
#382 - Change display and translations structure for criteria and con…
Browse files Browse the repository at this point in the history
…cepts

- remove nested declaration from elastic search documents (seems to be unnecessary, but should be monitored)
- modify swagger file to match new responses
- adapt tests to the new structure
- Change responses to codeable concept and terminology replies to the new structure
- change de-De to de and en-US to us for the elastic search related entries (the ones sent to the frontend are unchanged...might have to change as well)
- modify elastic search tests in integration tests
- remove GET of documents with fixed ids from elastic search integration test since those ids differ with each new ontology
- add translations to CriteriaProfileData
- adapt swagger file
  • Loading branch information
michael-82 committed Jan 15, 2025
1 parent 5792b49 commit 5b7638c
Show file tree
Hide file tree
Showing 41 changed files with 1,908 additions and 737 deletions.
49 changes: 0 additions & 49 deletions .github/scripts/post-elastic-test-queries.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,39 +2,6 @@

curl "$ELASTIC_HOST/_cat/indices"

response=$(curl -s -w "%{http_code}" -o response_body "$ELASTIC_HOST/ontology/_doc/9c2328b0-ac4e-3d69-8f2f-d8b905875348")
http_code="${response: -3}"
json_body=$(cat response_body)

if [ "$http_code" -eq 200 ]; then
if echo "$json_body" | jq '.found == true' | grep -q true; then
echo "Ontology Document found in elastic search"
else
echo "Empty or nonexistent response from elastic search"
exit 1
fi
else
echo "Response code $http_code"
exit 1
fi


response=$(curl -s -w "%{http_code}" -o response_body "$ELASTIC_HOST/codeable_concept/_doc/d676be36-7f34-3ea6-9838-c0c9e1ca3dcc")
http_code="${response: -3}"
json_body=$(cat response_body)

if [ "$http_code" -eq 200 ]; then
if echo "$json_body" | jq '.found == true' | grep -q true; then
echo "Codeable Concept Document found in elastic search"
else
echo "Empty or nonexistent response from elastic search"
exit 1
fi
else
echo "Response code $http_code"
exit 1
fi

access_token="$(curl -s --request POST \
--url http://localhost:8083/auth/realms/dataportal/protocol/openid-connect/token \
--header 'Content-Type: application/x-www-form-urlencoded' \
Expand Down Expand Up @@ -76,21 +43,5 @@ else
exit 1
fi

response=$(curl -s -w "%{http_code}" --header "Authorization: Bearer $access_token" -o response_body "http://localhost:8091/api/v4/codeable-concept/entry/d676be36-7f34-3ea6-9838-c0c9e1ca3dcc")
http_code="${response: -3}"
json_body=$(cat response_body)

if [ "$http_code" -eq 200 ]; then
if echo "$json_body" | jq -e '.code and .code != ""' | grep -q true; then
echo "OK response with non-empty array"
else
echo "Empty or nonexistent response"
exit 1
fi
else
echo "Response code $http_code"
exit 1
fi

echo "All elastic search tests completed"
exit 0
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ jobs:
needs: tests
runs-on: ubuntu-latest
env:
ONTOLOGY_GIT_TAG: v3.0.1
ONTOLOGY_GIT_TAG: v3.0.2-alpha
ELASTIC_HOST: http://localhost:9200
ELASTIC_FILEPATH: https://github.com/medizininformatik-initiative/fhir-ontology-generator/releases/download/TAGPLACEHOLDER/
ELASTIC_FILENAME: elastic.zip
Expand Down
2 changes: 1 addition & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ services:
environment:
ES_HOST: http://dataportal-elastic
ES_PORT: 9200
ONTO_GIT_TAG: v3.0.1
ONTO_GIT_TAG: v3.0.2-alpha
ONTO_REPO: https://github.com/medizininformatik-initiative/fhir-ontology-generator/releases/download
DOWNLOAD_FILENAME: elastic.zip
EXIT_ON_EXISTING_INDICES: false
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package de.numcodex.feasibility_gui_backend.common.api;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.numcodex.feasibility_gui_backend.dse.api.LocalizedValue;
import de.numcodex.feasibility_gui_backend.terminology.es.model.Display;
import lombok.Builder;

import java.util.List;

@JsonInclude(JsonInclude.Include.NON_NULL)
@Builder
public record DisplayEntry(
@JsonProperty String original,
@JsonProperty List<LocalizedValue> translations
) {

public static DisplayEntry of(Display display) {
return DisplayEntry.builder()
.original(display.original())
.translations(List.of(
LocalizedValue.builder()
.language("de-DE")
.value(display.deDe())
.build(),
LocalizedValue.builder()
.language("en-US")
.value(display.enUs())
.build()
))
.build();
}
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.numcodex.feasibility_gui_backend.common.api.DisplayEntry;
import lombok.Builder;

import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.numcodex.feasibility_gui_backend.common.api.DisplayEntry;
import lombok.Builder;

import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.numcodex.feasibility_gui_backend.common.api.DisplayEntry;
import lombok.Builder;

import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,4 +108,14 @@ public List<CriteriaProfileData> getCriteriaProfileData(List<String> criteriaIds

return results;
}

public List<CriteriaProfileData> addDisplayDataToCriteriaProfileData(List<CriteriaProfileData> criteriaProfileData, List<EsSearchResultEntry> displayData) {
var result = new ArrayList<CriteriaProfileData>();

Check warning on line 113 in src/main/java/de/numcodex/feasibility_gui_backend/terminology/TerminologyService.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/de/numcodex/feasibility_gui_backend/terminology/TerminologyService.java#L113

Added line #L113 was not covered by tests
for (CriteriaProfileData cpd : criteriaProfileData) {
var searchResultEntry = displayData.stream().filter(dd -> cpd.getId().equals(dd.id())).findFirst().orElse(null);

Check warning on line 115 in src/main/java/de/numcodex/feasibility_gui_backend/terminology/TerminologyService.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/de/numcodex/feasibility_gui_backend/terminology/TerminologyService.java#L115

Added line #L115 was not covered by tests
cpd.setDisplay(searchResultEntry == null ? null : searchResultEntry.display());
result.add(cpd);
}
return result;

Check warning on line 119 in src/main/java/de/numcodex/feasibility_gui_backend/terminology/TerminologyService.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/de/numcodex/feasibility_gui_backend/terminology/TerminologyService.java#L117-L119

Added lines #L117 - L119 were not covered by tests
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@
@Builder
public class CcSearchResult {
private long totalHits;
private List<TermCode> results;
private List<CodeableConceptEntry> results;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package de.numcodex.feasibility_gui_backend.terminology.api;

import de.numcodex.feasibility_gui_backend.common.api.TermCode;
import de.numcodex.feasibility_gui_backend.common.api.DisplayEntry;
import de.numcodex.feasibility_gui_backend.dse.api.LocalizedValue;
import de.numcodex.feasibility_gui_backend.terminology.es.model.CodeableConceptDocument;
import lombok.Builder;

import java.util.List;

@Builder
public record CodeableConceptEntry(
TermCode termCode,
DisplayEntry display
) {
public static CodeableConceptEntry of(CodeableConceptDocument document) {
return CodeableConceptEntry.builder()
.termCode(document.termCode())
.display(DisplayEntry.builder()
.original(document.display().original())
.translations(List.of(
LocalizedValue.builder()
.language("de-DE")
.value(document.display().deDe())
.build(),
LocalizedValue.builder()
.language("en-US")
.value(document.display().enUs())
.build()
))
.build()
)
.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import de.numcodex.feasibility_gui_backend.common.api.DisplayEntry;
import de.numcodex.feasibility_gui_backend.common.api.TermCode;
import lombok.Builder;
import lombok.Data;
Expand All @@ -17,6 +18,8 @@
public class CriteriaProfileData {
@JsonProperty("id")
private String id;
@JsonProperty("display")
private DisplayEntry display;
@JsonProperty("context")
private TermCode context;
@JsonProperty("termCodes")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package de.numcodex.feasibility_gui_backend.terminology.api;

import de.numcodex.feasibility_gui_backend.common.api.DisplayEntry;
import de.numcodex.feasibility_gui_backend.terminology.es.model.OntologyListItemDocument;
import lombok.Builder;

@Builder
public record EsSearchResultEntry(
String id,
String name,
DisplayEntry display,
int availability,
String context,
String terminology,
Expand All @@ -17,7 +18,7 @@ public record EsSearchResultEntry(
public static EsSearchResultEntry of(OntologyListItemDocument ontologyListItemDocument) {
return EsSearchResultEntry.builder()
.id(ontologyListItemDocument.id())
.name(ontologyListItemDocument.name())
.display(DisplayEntry.of(ontologyListItemDocument.display()))
.availability(ontologyListItemDocument.availability())
.context(ontologyListItemDocument.context().code())
.terminology(ontologyListItemDocument.terminology())
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package de.numcodex.feasibility_gui_backend.terminology.api;

import de.numcodex.feasibility_gui_backend.common.api.DisplayEntry;
import de.numcodex.feasibility_gui_backend.terminology.es.model.OntologyItemRelationsDocument;
import de.numcodex.feasibility_gui_backend.terminology.es.model.Relative;
import lombok.Builder;

import java.util.Collection;

@Builder
public record RelationEntry(
DisplayEntry display,
Collection<Relative> parents,
Collection<Relative> children,
Collection<Relative> relatedTerms
) {

public static RelationEntry of(OntologyItemRelationsDocument document) {
return RelationEntry.builder()
.display(DisplayEntry.of(document.display()))
.parents(document.parents())
.children(document.children())
.relatedTerms(document.relatedTerms())
.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.query_dsl.*;
import de.numcodex.feasibility_gui_backend.common.api.TermCode;
import de.numcodex.feasibility_gui_backend.terminology.api.CcSearchResult;
import de.numcodex.feasibility_gui_backend.terminology.api.CodeableConceptEntry;
import de.numcodex.feasibility_gui_backend.terminology.es.model.CodeableConceptDocument;
import de.numcodex.feasibility_gui_backend.terminology.es.repository.CodeableConceptEsRepository;
import de.numcodex.feasibility_gui_backend.terminology.es.repository.OntologyItemNotFoundException;
Expand Down Expand Up @@ -47,17 +47,18 @@ public CcSearchResult performCodeableConceptSearchWithRepoAndPaging(String keywo
}

var searchHitPage = findByCodeOrDisplay(keyword, filterList, PageRequest.of(page, pageSize));
List<TermCode> codeableConceptEntries = new ArrayList<>();
List<CodeableConceptEntry> codeableConceptEntries = new ArrayList<>();

searchHitPage.getSearchHits().forEach(hit -> codeableConceptEntries.add(hit.getContent().termCode()));
searchHitPage.getSearchHits().forEach(hit -> codeableConceptEntries.add(CodeableConceptEntry.of(hit.getContent())));
return CcSearchResult.builder()
.totalHits(searchHitPage.getTotalHits())
.results(codeableConceptEntries)
.build();
}

public TermCode getSearchResultEntryByCode(String code) {
return repo.findById(code).orElseThrow(OntologyItemNotFoundException::new).termCode();
public CodeableConceptEntry getSearchResultEntryByCode(String code) {
var document = repo.findById(code).orElseThrow(OntologyItemNotFoundException::new);
return CodeableConceptEntry.of(document);
}

private SearchHits<CodeableConceptDocument> findByCodeOrDisplay(String keyword,
Expand Down Expand Up @@ -88,7 +89,7 @@ private SearchHits<CodeableConceptDocument> findByCodeOrDisplay(String keyword,
} else {
var mmQuery = new MultiMatchQuery.Builder()
.query(keyword)
.fields(List.of("termcode.display", "termcode.code^2"))
.fields(List.of("display.de", "display.en", "termcode.code^2"))
.build();

boolQuery = new BoolQuery.Builder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import co.elastic.clients.elasticsearch._types.query_dsl.*;
import de.numcodex.feasibility_gui_backend.terminology.api.EsSearchResult;
import de.numcodex.feasibility_gui_backend.terminology.api.EsSearchResultEntry;
import de.numcodex.feasibility_gui_backend.terminology.api.RelationEntry;
import de.numcodex.feasibility_gui_backend.terminology.es.model.*;
import de.numcodex.feasibility_gui_backend.terminology.es.repository.OntologyItemEsRepository;
import de.numcodex.feasibility_gui_backend.terminology.es.repository.OntologyItemNotFoundException;
Expand Down Expand Up @@ -53,6 +54,13 @@ public EsSearchResultEntry getSearchResultEntryByHash(String hash) {
return EsSearchResultEntry.of(ontologyItem);
}

public List<EsSearchResultEntry> getSearchResultEntriesByHash(List<String> hashes) {
var ontologyItems = ontologyListItemEsRepository.findAllById(hashes);
List<EsSearchResultEntry> results = new ArrayList<>();
ontologyItems.forEach(oi -> results.add(EsSearchResultEntry.of(oi)));
return results;

Check warning on line 61 in src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsService.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/de/numcodex/feasibility_gui_backend/terminology/es/TerminologyEsService.java#L58-L61

Added lines #L58 - L61 were not covered by tests
}

public List<TermFilter> getAvailableFilters() {
var filterTerms = List.of(filterFields);
var list = new ArrayList<TermFilter>();
Expand Down Expand Up @@ -150,7 +158,7 @@ private SearchHits<OntologyListItemDocument> findByNameOrTermcode(String keyword
} else {
var mmQuery = new MultiMatchQuery.Builder()
.query(keyword)
.fields(List.of("name", "termcode^2"))
.fields(List.of("display.de", "display.en", "termcode^2"))
.build();

boolQuery = new BoolQuery.Builder()
Expand Down Expand Up @@ -185,18 +193,21 @@ private SearchHits<OntologyListItemDocument> findByNameOrTermcode(String keyword
.withPageable(pageRequest)
.build();

log.info(finalQuery.getQuery().toString());

return operations.search(finalQuery, OntologyListItemDocument.class);

}

public OntologyItemRelationsDocument getOntologyItemRelationsByHash(String hash) {
public RelationEntry getRelationEntryByHash(String hash) {
var ontologyItem = ontologyItemEsRepository.findById(hash).orElseThrow(OntologyItemNotFoundException::new);
return OntologyItemRelationsDocument.builder()
.translations(ontologyItem.translations())
var ontologyItemRelationsDocument = OntologyItemRelationsDocument.builder()
.display(ontologyItem.display())
.parents(ontologyItem.parents())
.children(ontologyItem.children())
.relatedTerms(ontologyItem.relatedTerms())
.build();
return RelationEntry.of(ontologyItemRelationsDocument);
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,11 @@
@Document(indexName = "codeable_concept")
public record CodeableConceptDocument(
@Id String id,
@Field(type = FieldType.Nested, includeInParent = true, name = "termcode")
@Field(name = "termcode")
TermCode termCode,
@Field(type = FieldType.Nested, includeInParent = true, name = "value_sets")
List<String> valueSets
@Field(name = "value_sets")
List<String> valueSets,
@Field(name = "display")
Display display
) {
}
Loading

0 comments on commit 5b7638c

Please sign in to comment.