diff --git a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java index 5489cf1cd..7af3f875c 100644 --- a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java +++ b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -21,6 +22,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.PropertySources; +import org.springframework.core.io.ClassPathResource; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import eu.europeana.entitymanagement.definitions.model.ZohoLabelUriMapping; @@ -165,11 +167,14 @@ public class EntityManagementConfiguration implements InitializingBean { @Value("${spring.profiles.active:}") private String activeProfileString; - @Value("${zoho.country.mapping:null}") + @Value("${zoho.country.mapping:zoho_country_mapping.json}") private String zohoCountryMappingFilename; - @Value("${zoho.role.mapping:null}") + @Value("${zoho.role.mapping:zoho_role_mapping.json}") private String zohoRoleMappingFilename; + + @Value("${europeana.role.vocabulary:role_vocabulary.xml}") + private String roleVocabularyFilename; Map countryMappings = new HashMap<>(); Map roleMappings = new HashMap<>(); @@ -224,7 +229,9 @@ private void verifyRequiredProperties() { } private void initCountryMappings() throws IOException { - try (InputStream inputStream = getClass().getResourceAsStream(getZohoCountryMappingFilename())) { + ClassPathResource resource = new ClassPathResource(getZohoCountryMappingFilename()); + + try (InputStream inputStream = resource.getInputStream()) { assert inputStream != null; try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { String contents = reader.lines().collect(Collectors.joining(System.lineSeparator())); @@ -237,7 +244,10 @@ private void initCountryMappings() throws IOException { } private void initRoleMappings() throws IOException { - try (InputStream inputStream = getClass().getResourceAsStream(getZohoRoleMappingFilename())) { + + ClassPathResource resource = new ClassPathResource(getZohoRoleMappingFilename()); + + try (InputStream inputStream = resource.getInputStream()) { assert inputStream != null; try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { String contents = reader.lines().collect(Collectors.joining(System.lineSeparator())); @@ -249,6 +259,17 @@ private void initRoleMappings() throws IOException { } } + public String loadRoleVocabulary() throws IOException { + ClassPathResource resource = new ClassPathResource(getRoleVocabularyFilename()); + + try (InputStream inputStream = resource.getInputStream()) { + assert inputStream != null; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + return reader.lines().collect(Collectors.joining(System.lineSeparator())); + } + } + } + public String getPrSolrUrl() { return prSolrUrl; } @@ -436,4 +457,8 @@ public String getZohoRoleMappingFilename() { public Map getRoleMappings() { return roleMappings; } + + public String getRoleVocabularyFilename() { + return roleVocabularyFilename; + } } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/VocabularyFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/VocabularyFields.java index 17820921e..4fb92e997 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/VocabularyFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/VocabularyFields.java @@ -7,6 +7,6 @@ private VocabularyFields() { // private constructor to prevent instantiation } - public static final String VOCABULARY_URI = "vocabularyUri"; + public static final String URI = "uri"; } \ No newline at end of file diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java index 2b39e2e1b..335f267f3 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java @@ -95,7 +95,6 @@ public Organization(Organization copy) { if (copy.getPhone() != null) this.phone = new ArrayList<>(copy.getPhone()); if (copy.getMbox() != null) this.mbox = new ArrayList<>(copy.getMbox()); //because the europeanaRoleRef is a reference to the object we keep it the same (therefore also for europeanaRole) - this.europeanaRole = copy.getEuropeanaRole(); this.europeanaRoleRefs = copy.getEuropeanaRoleRefs(); if(copy.getEuropeanaRoleIds()!=null) this.europeanaRoleIds=new ArrayList<>(copy.getEuropeanaRoleIds()); //because the countryRef is a reference to the object we keep it the same (therefore also for country) @@ -272,7 +271,7 @@ public List getEuropeanaRole() { europeanaRole=new ArrayList<>(); for(String roleId : europeanaRoleIds) { Vocabulary vocab = new Vocabulary(); - vocab.setVocabularyUri(roleId); + vocab.setUri(roleId); europeanaRole.add(vocab); } } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Vocabulary.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Vocabulary.java index ee32500cc..63de01610 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Vocabulary.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Vocabulary.java @@ -21,6 +21,9 @@ @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @dev.morphia.annotations.Entity("Vocabulary") @EntityListeners(VocabularyWatcher.class) +/** + * class used for storing static vocabularies (e.g. europeana roles) + */ public class Vocabulary { private String type = "Vocabulary"; @@ -28,7 +31,7 @@ public class Vocabulary { @Id @JsonIgnore private ObjectId dbId; @Indexed(options = @IndexOptions(unique = true)) - private String vocabularyUri; + private String uri; protected List inScheme; @@ -40,6 +43,9 @@ public class Vocabulary { @JsonIgnore private Date modified; + /** + * default constructor + */ public Vocabulary() { } @@ -69,17 +75,17 @@ public Date getModified() { @Override public String toString() { - return String.format("Vocabulary.vocabularyUri: %s", getVocabularyUri()); + return String.format("Vocabulary.uri: %s", getUri()); } @JsonGetter - public String getVocabularyUri() { - return vocabularyUri; + public String getUri() { + return uri; } @JsonSetter - public void setVocabularyUri(String vocabularyUri) { - this.vocabularyUri = vocabularyUri; + public void setUri(String vocabularyUri) { + this.uri = vocabularyUri; } @JsonGetter(IN_SCHEME) diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java index 8bef05ad7..045f40a64 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java @@ -113,7 +113,7 @@ public XmlOrganizationImpl(Organization organization) { List orgXmlRole= new ArrayList<>(); for(Vocabulary vocab : orgRole) { XmlConceptImpl xmlConcept = new XmlConceptImpl(); - xmlConcept.setAbout(vocab.getVocabularyUri()); + xmlConcept.setAbout(vocab.getUri()); xmlConcept.setPrefLabel(RdfXmlUtils.convertMapToXmlMultilingualString(vocab.getPrefLabel())); xmlConcept.setInScheme(RdfXmlUtils.convertToRdfResource(vocab.getInScheme())); orgXmlRole.add(xmlConcept); @@ -149,30 +149,18 @@ public Organization toEntityModel() throws EntityModelCreationException { entity.setAcronym(RdfXmlUtils.toLanguageMapList(getAcronym())); entity.setDescription(RdfXmlUtils.toLanguageMap(getDescription())); entity.setLogo(XmlWebResourceWrapper.toWebResource(getLogo())); - //set europeanaRole + //set europeanaRole id (external dereferencers deliver only the ids, not transitive data) if(getEuropeanaRole()!=null && !getEuropeanaRole().isEmpty()) { - List roleIds=new ArrayList<>(); - List role=new ArrayList<>(); - for(XmlConceptImpl xmlConcept : getEuropeanaRole()) { - Vocabulary vocab=new Vocabulary(); - vocab.setVocabularyUri(xmlConcept.getAbout()); - vocab.setPrefLabel(RdfXmlUtils.toLanguageMap(xmlConcept.getPrefLabel())); - vocab.setInScheme(RdfXmlUtils.toStringList(xmlConcept.getInScheme())); - role.add(vocab); - roleIds.add(xmlConcept.getAbout()); - } - entity.setEuropeanaRole(role); + List roleIds = getEuropeanaRole().stream().map(e -> e.getAbout()).toList(); entity.setEuropeanaRoleIds(roleIds); } + //set country id (external dereferencers deliver only the ids, not transitive data) if(getCountry() != null) { - //the country is not saved in the database - entity.setCountry(getCountry().toEntityModel()); //we need to extract the countryID as well (xml about holds the entityId) entity.setCountryId(getCountry().getAbout()); } - if (getHomepage() != null) { entity.setHomepage(getHomepage().getResource()); } diff --git a/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java b/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java index 0e35213c5..a49537a3c 100644 --- a/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java +++ b/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java @@ -1,7 +1,7 @@ package eu.europeana.entitymanagement.mongo.repository; import static dev.morphia.query.experimental.filters.Filters.in; -import static eu.europeana.entitymanagement.definitions.VocabularyFields.VOCABULARY_URI; +import static eu.europeana.entitymanagement.definitions.VocabularyFields.URI; import java.util.ArrayList; import java.util.List; import javax.annotation.Resource; @@ -18,24 +18,50 @@ public class VocabularyRepository { @Resource(name = AppConfigConstants.BEAN_EM_DATA_STORE) Datastore datastore; - public List findByVocabularyUris(List vocabularyUris) { + /** + * retrieve records by their URI/id + * @param vocabularyUris + * @return + */ + public List findByUri(List vocabularyUris) { List filters = new ArrayList<>(); - filters.add(in(VOCABULARY_URI, vocabularyUris)); + filters.add(in(URI, vocabularyUris)); return datastore.find(Vocabulary.class) .filter(filters.toArray(Filter[]::new)) .iterator() .toList(); } + /** + * save to database + * @param vocab record to save + * @return saved record + */ public Vocabulary save(Vocabulary vocab) { return datastore.save(vocab); } + /** + * save list of records to database + * @param vocabs list of records to save + * @return saved records + */ public List saveBulk(List vocabs) { return datastore.save(vocabs); } + /** + * clear database collection + */ public void dropCollection() { datastore.getMapper().getCollection(Vocabulary.class).drop(); } + + /** + * count the records available in the database + * @return number of database records + */ + public long countRecords() { + return datastore.find(Vocabulary.class).count(); + } } \ No newline at end of file diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java index 97b6e1def..669f1bb51 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java @@ -36,7 +36,8 @@ public class SolrOrganization extends SolrEntity { @Field(OrganizationSolrFields.FOAF_MBOX) private List mbox; - @Field(OrganizationSolrFields.EUROPEANA_ROLE) + //MAPPING TO BE ENABLED BACK IN THE NEXT VERSION - requires schema and ENtity API UPDATE + //@Field(OrganizationSolrFields.EUROPEANA_ROLE) private List europeanaRole; @Field(OrganizationSolrFields.COUNTRY) diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java index f33ed6119..ec5ea9068 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java @@ -151,8 +151,7 @@ static void setProperties(DynamicPropertyRegistry registry) { //generate Europeana ID can be set to true when using the mock service, see TextConfig class registry.add("zoho.generate.organization.europeanaid", () -> true); - registry.add("zoho.country.mapping", () -> "/zoho_country_mapping_test.json"); - registry.add("zoho.role.mapping", () -> "/zoho_role_mapping_test.json"); + registry.add("zoho.country.mapping", () -> "zoho_country_mapping_test.json"); // could be used to fix eclipse issues registry.add("scmBranch", () -> "dev"); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java index 76b510463..eb975acd7 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java @@ -14,7 +14,7 @@ import eu.europeana.entitymanagement.batch.repository.FailedTaskRepository; import eu.europeana.entitymanagement.batch.service.ScheduledTaskService; import eu.europeana.entitymanagement.batch.utils.BatchUtils; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; import eu.europeana.entitymanagement.definitions.batch.model.FailedTask; import eu.europeana.entitymanagement.definitions.batch.model.FailedTask.Builder; @@ -63,7 +63,7 @@ class ScheduledTaskServiceIT extends AbstractIntegrationTest { @Autowired EntityUpdateJobConfig updateJobConfig; - @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) + @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE) @Autowired private SolrService emSolrService; diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScoringServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScoringServiceIT.java index a78c01c61..4ebb4d700 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScoringServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScoringServiceIT.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import eu.europeana.entitymanagement.AbstractIntegrationTest; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.Agent; import eu.europeana.entitymanagement.definitions.model.Place; import eu.europeana.entitymanagement.definitions.model.TimeSpan; @@ -26,7 +26,7 @@ @AutoConfigureMockMvc class ScoringServiceIT extends AbstractIntegrationTest { - @Resource(name = AppConfig.BEAN_EM_SCORING_SERVICE) + @Resource(name = AppAutoconfig.BEAN_EM_SCORING_SERVICE) ScoringService scoringService; // @Test diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java index d4fb9d432..a9b93ef9d 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.europeana.entitymanagement.AbstractIntegrationTest; import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.Agent; import eu.europeana.entitymanagement.definitions.model.Aggregation; import eu.europeana.entitymanagement.definitions.model.Concept; @@ -48,7 +48,7 @@ public class SolrServiceIT extends AbstractIntegrationTest { // private static final String RIGHTS_PD = "https://creativecommons.org/publicdomain/zero/1.0/"; - @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) + @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE) @Autowired private SolrService emSolrService; diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java index 0476b51d4..aa99ec23e 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java @@ -23,7 +23,7 @@ import eu.europeana.entitymanagement.batch.service.EntityUpdateService; import eu.europeana.entitymanagement.batch.service.ScheduledTaskService; import eu.europeana.entitymanagement.common.config.DataSource; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType; import eu.europeana.entitymanagement.definitions.model.Entity; @@ -49,7 +49,7 @@ abstract class BaseWebControllerTest extends AbstractIntegrationTest { @Autowired private EntityUpdateService entityUpdateService; - @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) + @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE) @Autowired protected SolrService emSolrService; diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java index 591c77ff9..df9896450 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java @@ -330,6 +330,8 @@ void registerZohoOrganizationBnfWithNewFieldsShouldBeSuccessful() throws Excepti .andExpect(jsonPath("$.language", everyItem(matchesRegex("[a-z]+")))) .andExpect(jsonPath("$.hiddenLabel", hasSize(3))) .andExpect(jsonPath("$.id", any(String.class))); + // excluded as the wikidata hasGeo is not consolidated + //.andExpect(jsonPath("$.hasAddress.hasGeo").isNotEmpty()) } @Test diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java index 411df4cd7..d547cc3ad 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java @@ -498,13 +498,6 @@ public void retrieveOrganizationJsonExternalWithCountryAndRoleDereference() thro String metisResponse = loadFile(IntegrationTestUtils.PLACE_SWEDEN_XML); createEntity(europeanaMetadata, metisResponse, IntegrationTestUtils.PLACE_SWEDEN_URI); - //2. create a vocabulary for the europeanaRole - Vocabulary vocab = new Vocabulary(); - vocab.setVocabularyUri("http://data.europeana.eu/vocabulary/role/ProvidingInstitution"); - vocab.setInScheme(List.of("http://data.europeana.eu/vocabulary/role")); - vocab.setPrefLabel(Map.of("en", "Providing Institution")); - vocabRepository.save(vocab); - // //forcefully change the country mapping uri to the right one // List countryMap= entityRecordService.getCountryMapping(); // for(CountryMapping cm : countryMap) { @@ -513,7 +506,7 @@ public void retrieveOrganizationJsonExternalWithCountryAndRoleDereference() thro // } // } - //3. register zoho GFM org + //2. register zoho GFM org europeanaMetadata = loadFile(IntegrationTestUtils.ORGANIZATION_REGISTER_GFM_ZOHO_JSON); Optional zohoRecord = IntegrationTestUtils.getZohoOrganizationRecord( @@ -523,12 +516,12 @@ public void retrieveOrganizationJsonExternalWithCountryAndRoleDereference() thro String entityId = createOrganization(europeanaMetadata, zohoRecord.get()).getEntityId(); String requestPath = getEntityRequestPath(entityId); - mockMvc + ResultActions resultActions = mockMvc .perform( get(IntegrationTestUtils.BASE_SERVICE_URL + "/" + requestPath + ".jsonld") .param(WebEntityConstants.QUERY_PARAM_PROFILE, "external, dereference") - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) + .accept(MediaType.APPLICATION_JSON)); + resultActions.andExpect(status().isOk()) .andExpect(jsonPath("$.id", is(entityId))) .andExpect(jsonPath("$.type", is(EntityTypes.Organization.getEntityType()))) .andExpect(jsonPath("$.sameAs").isNotEmpty()) diff --git a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml index 8d83e5388..720ed19cb 100644 --- a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml +++ b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml @@ -485,11 +485,15 @@ - + + + + @@ -573,7 +577,8 @@ dest="text.*" /> - + + @@ -587,7 +592,8 @@ dest="text" /> - + + diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java index 4a27f9140..4ba75eeb8 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java @@ -19,7 +19,9 @@ import eu.europeana.entitymanagement.batch.service.BatchEntityUpdateExecutor; import eu.europeana.entitymanagement.batch.service.ScheduledTaskService; import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; +import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; import eu.europeana.entitymanagement.web.model.ZohoSyncReport; +import eu.europeana.entitymanagement.web.service.EntityRecordService; import eu.europeana.entitymanagement.web.service.ZohoSyncService; /** @@ -39,6 +41,8 @@ public class EntityManagementApp implements CommandLineRunner { private BatchEntityUpdateExecutor batchUpdateExecutor; @Autowired private ZohoSyncService zohoSyncService; + @Autowired + private EntityRecordService entityRecordService; /** * Main entry point of this application @@ -51,7 +55,7 @@ public class EntityManagementApp implements CommandLineRunner { */ public static void main(String[] args) { // jobType = args.length > 0 ? args[0] : ""; - if (hasCmdLineParams(args)) { + if (isScheduledTask(args)) { if (LOG.isInfoEnabled()) { LOG.info("Starting batch updates execution with args: {}", Arrays.toString(args)); } @@ -92,6 +96,10 @@ public static void main(String[] args) { } } + static boolean isScheduledTask(String[] args) { + return hasCmdLineParams(args); + } + static ScheduledTaskService getScheduledTasksService(ConfigurableApplicationContext context) { return (ScheduledTaskService) context .getBean(AppConfigConstants.BEAN_BATCH_SCHEDULED_TASK_SERVICE); @@ -108,34 +116,38 @@ static boolean hasCmdLineParams(String[] args) { @Override public void run(String... args) throws Exception { - if (hasCmdLineParams(args)) { - Set tasks = Set.of(args); - - // first zoho sync as it runs synchronuous operations - if (tasks.contains(JobType.ZOHO_SYNC.value())) { - LOG.info("Executing zoho sync"); - ZohoSyncReport zohoSyncReport = zohoSyncService.synchronizeModifiedZohoOrganizations(); - LOG.info("Synchronization Report: {}", zohoSyncReport.toString()); - } + if (isScheduledTask(args)) { + runScheduledTasks(args); + } + // if no arguments then web server should be started + return; + } - if (tasks.contains(JobType.SCHEDULE_DELETION.value())) { - // run also the deletions called through the API directly - LOG.info("Executing scheduled deletions"); - batchUpdateExecutor.runScheduledDeprecationsAndDeletions(); - // TODO: should read the number of scheduled deletions and deprecations from the database - // and write it to the logs - } - if (tasks.contains(JobType.SCHEDULE_UPDATE.value())) { - LOG.info("Executing scheduled updates"); - batchUpdateExecutor.runScheduledUpdate(); - // TODO: should read the number of scheduled deletions and deprecations from the database - // and write it to the logs - } + void runScheduledTasks(String... args) throws EntityUpdateException { + Set tasks = Set.of(args); + // first zoho sync as it runs synchronuous operations + if (tasks.contains(JobType.ZOHO_SYNC.value())) { + LOG.info("Executing zoho sync"); + ZohoSyncReport zohoSyncReport = zohoSyncService.synchronizeModifiedZohoOrganizations(); + LOG.info("Synchronization Report: {}", zohoSyncReport.toString()); + } + + if (tasks.contains(JobType.SCHEDULE_DELETION.value())) { + // run also the deletions called through the API directly + LOG.info("Executing scheduled deletions"); + batchUpdateExecutor.runScheduledDeprecationsAndDeletions(); + // TODO: should read the number of scheduled deletions and deprecations from the database + // and write it to the logs + } + + if (tasks.contains(JobType.SCHEDULE_UPDATE.value())) { + LOG.info("Executing scheduled updates"); + batchUpdateExecutor.runScheduledUpdate(); + // TODO: should read the number of scheduled deletions and deprecations from the database + // and write it to the logs } - // if no arguments then web server should be started - return; } /** validates the arguments passed */ diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppConfig.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppAutoconfig.java similarity index 63% rename from entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppConfig.java rename to entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppAutoconfig.java index a3fb651e1..7e063926b 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppConfig.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppAutoconfig.java @@ -5,9 +5,12 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.annotation.Resource; +import javax.xml.bind.JAXBContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.context.MessageSource; @@ -22,32 +25,70 @@ import eu.europeana.entitymanagement.common.config.DataSource; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; +import eu.europeana.entitymanagement.definitions.model.Vocabulary; +import eu.europeana.entitymanagement.exception.ApplicationInitializationException; +import eu.europeana.entitymanagement.mongo.repository.VocabularyRepository; +import eu.europeana.entitymanagement.web.MetisDereferenceUtils; +import eu.europeana.entitymanagement.web.xml.model.RdfXmlUtils; +import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; +import eu.europeana.entitymanagement.web.xml.model.XmlConceptImpl; /** @author GordeaS */ @Configuration -public class AppConfig extends AppConfigConstants { +public class AppAutoconfig extends AppConfigConstants { - private static final Logger LOG = LogManager.getLogger(AppConfig.class); + private static final Logger LOG = LogManager.getLogger(AppAutoconfig.class); @Resource private EntityManagementConfiguration emConfiguration; @Resource(name = BEAN_XML_MAPPER) private XmlMapper xmlMapper; - - public AppConfig() { + + @Resource(name = BEAN_VOCABULARY_REPO) + private VocabularyRepository vocabRepository; + + @Resource protected JAXBContext jaxbContext; + + public AppAutoconfig() { LOG.info("Initializing EntityManagementConfiguration bean as: configuration"); } @PostConstruct - public void init() { + public void init() throws ApplicationInitializationException { if (emConfiguration.isAuthReadEnabled() || emConfiguration.isAuthWriteEnabled()) { String jwtTokenSignatureKey = emConfiguration.getApiKeyPublicKey(); if (jwtTokenSignatureKey == null || jwtTokenSignatureKey.isBlank()) { throw new IllegalStateException("The jwt token signature key cannot be null or empty."); } } + //ensure data + ensureDatabaseInitialization(); + } + public void ensureDatabaseInitialization() throws ApplicationInitializationException { + if(vocabRepository.countRecords() < 1) { + List> xmlEntities; + try { + xmlEntities = MetisDereferenceUtils.parseMetisResponseMany( + jaxbContext.createUnmarshaller(), emConfiguration.loadRoleVocabulary()); + } catch (Exception e) { + throw new ApplicationInitializationException("Cannot load vocabulary from resources!", e); + } + + List roles = new ArrayList<>(xmlEntities.size()); + for(XmlBaseEntityImpl xmlEntity : xmlEntities) { + XmlConceptImpl xmlConcept = (XmlConceptImpl) xmlEntity; + Vocabulary vocab = new Vocabulary(); + vocab.setUri(xmlConcept.getAbout()); + vocab.setInScheme(RdfXmlUtils.toStringList(xmlConcept.getInScheme())); + vocab.setPrefLabel(RdfXmlUtils.toLanguageMap(xmlConcept.getPrefLabel())); + roles.add(vocab); + } + vocabRepository.saveBulk(roles); + } + } + @Bean(name = BEAN_EM_DATA_SOURCES) public DataSources getDataSources() throws IOException { String datasourcesXMLConfigFile = emConfiguration.getDatasourcesXMLConfig(); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/exception/ApplicationInitializationException.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/exception/ApplicationInitializationException.java new file mode 100644 index 000000000..ecf68800b --- /dev/null +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/exception/ApplicationInitializationException.java @@ -0,0 +1,12 @@ +package eu.europeana.entitymanagement.exception; + +public class ApplicationInitializationException extends Exception { + + public ApplicationInitializationException(String message) { + super(message); + } + + public ApplicationInitializationException(String message, Throwable th) { + super(message, th); + } +} diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java index 859f71fbe..76d29112f 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java @@ -7,7 +7,7 @@ import org.springframework.stereotype.Service; import eu.europeana.api.commons.error.EuropeanaApiException; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.ConceptScheme; import eu.europeana.entitymanagement.exception.EntityNotFoundException; import eu.europeana.entitymanagement.exception.EntityRemovedException; @@ -18,7 +18,7 @@ import eu.europeana.entitymanagement.utils.EntityUtils; import eu.europeana.entitymanagement.vocabulary.EntityTypes; -@Service(AppConfig.BEAN_CONCEPT_SCHEME_SERVICE) +@Service(AppAutoconfig.BEAN_CONCEPT_SCHEME_SERVICE) public class ConceptSchemeService { private final ConceptSchemeRepository emConceptSchemeRepo; diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index c0f1d9f66..eb642ba58 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -33,7 +33,7 @@ import eu.europeana.api.commons.error.EuropeanaApiException; import eu.europeana.entitymanagement.common.config.DataSource; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; import eu.europeana.entitymanagement.config.DataSources; import eu.europeana.entitymanagement.definitions.exceptions.EntityModelCreationException; import eu.europeana.entitymanagement.definitions.exceptions.UnsupportedEntityTypeException; @@ -77,7 +77,7 @@ import eu.europeana.entitymanagement.zoho.utils.ZohoException; import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; -@Service(AppConfig.BEAN_ENTITY_RECORD_SERVICE) +@Service(AppConfigConstants.BEAN_ENTITY_RECORD_SERVICE) public class EntityRecordService { private final EntityRecordRepository entityRecordRepository; @@ -176,7 +176,7 @@ private void dereferenceLinkedEntities(Organization org) { } //dereference role if(org.getEuropeanaRoleIds()!=null && !org.getEuropeanaRoleIds().isEmpty()) { - org.setEuropeanaRole(vocabRepository.findByVocabularyUris(org.getEuropeanaRoleIds())); + org.setEuropeanaRole(vocabRepository.findByUri(org.getEuropeanaRoleIds())); } } @@ -1288,7 +1288,6 @@ && isStringOrPrimitive(mapFirstValue.getClass())) { public void dropRepository() { this.entityRecordRepository.dropCollection(); - this.vocabRepository.dropCollection(); } /** @@ -1485,7 +1484,7 @@ public void processReferenceFields(Entity entity) { } //role reference if(org.getEuropeanaRoleIds()!=null && !org.getEuropeanaRoleIds().isEmpty()) { - List vocabs=vocabRepository.findByVocabularyUris(org.getEuropeanaRoleIds()); + List vocabs=vocabRepository.findByUri(org.getEuropeanaRoleIds()); if (vocabs.isEmpty()) { logger.info( "No vocabularies with the uris: {} were found in the database. Cannot assign role reference to organization with id {}", diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/MetisDereferenceService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/MetisDereferenceService.java index c6333a529..d65632a78 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/MetisDereferenceService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/MetisDereferenceService.java @@ -4,7 +4,7 @@ import static eu.europeana.entitymanagement.web.MetisDereferenceUtils.parseMetisResponse; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.Entity; import eu.europeana.entitymanagement.dereference.Dereferencer; import eu.europeana.entitymanagement.exception.DatasourceNotReachableException; @@ -37,7 +37,7 @@ import reactor.netty.http.client.HttpClient; /** Handles de-referencing entities from Metis. */ -@Service(AppConfig.BEAN_METIS_DEREF_SERVICE) +@Service(AppAutoconfig.BEAN_METIS_DEREF_SERVICE) public class MetisDereferenceService implements InitializingBean, Dereferencer { private static final Logger logger = LogManager.getLogger(MetisDereferenceService.class); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java index 682774478..7ac877a0d 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java @@ -19,7 +19,7 @@ import eu.europeana.api.commons.web.model.vocabulary.Operations; import eu.europeana.entitymanagement.batch.service.EntityUpdateService; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.config.DataSources; import eu.europeana.entitymanagement.definitions.model.EntityRecord; import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; @@ -36,7 +36,7 @@ import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; import eu.europeana.entitymanagement.zoho.utils.ZohoException; -@Service(AppConfig.BEAN_ZOHO_SYNC_SERVICE) +@Service(AppAutoconfig.BEAN_ZOHO_SYNC_SERVICE) public class ZohoSyncService extends BaseZohoAccess { @Autowired diff --git a/entity-management-web/src/main/resources/entitymanagement.user.properties.template b/entity-management-web/src/main/resources/entitymanagement.user.properties.template index 15c8a4fe3..11a60ed86 100644 --- a/entity-management-web/src/main/resources/entitymanagement.user.properties.template +++ b/entity-management-web/src/main/resources/entitymanagement.user.properties.template @@ -48,5 +48,7 @@ europeana.searchapi.urlPrefix=https:///record/v2/search.json?wskey #enable/disable generation of entity ids for organizations (should be enabled only in productive environment) #zoho.generate.organization.europeanaid=false -zoho.country.mapping= -zoho.role.mapping= \ No newline at end of file +#configuration files for zoho country and role mappings +#zoho.country.mapping=zoho_country_mapping.json +#zoho.role.mapping=zoho_role_mapping.json +#europeana.role.vocabulary=role_vocabulary.xml \ No newline at end of file diff --git a/entity-management-web/src/test/resources/metis-deref-unittest/roles.xml b/entity-management-web/src/main/resources/role_vocabulary.xml similarity index 100% rename from entity-management-web/src/test/resources/metis-deref-unittest/roles.xml rename to entity-management-web/src/main/resources/role_vocabulary.xml diff --git a/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java index 893be8f31..3de59b956 100644 --- a/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java +++ b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java @@ -7,7 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.Concept; import eu.europeana.entitymanagement.definitions.model.EntityRecord; import eu.europeana.entitymanagement.definitions.model.Vocabulary; @@ -24,7 +24,7 @@ @Disabled("Excluded from automated runs") public class UtilityTests { - @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) + @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE) @Autowired private SolrService emSolrService; @@ -54,6 +54,7 @@ public void reindexSolrFromMongo() throws EntityUpdateException { } //@Test + @Disabled("not needed, the vocabulary is automatically loaded at start time") public void saveVocabulariesToMongo() throws Exception { List> xmlEntities = MetisDereferenceUtils.parseMetisResponseMany( jaxbContext.createUnmarshaller(), UnitTestUtils.loadFile("/metis-deref-unittest/roles.xml")); @@ -61,7 +62,7 @@ public void saveVocabulariesToMongo() throws Exception { XmlConceptImpl xmlConcept = (XmlConceptImpl) xmlEntity; Concept concept = xmlConcept.toEntityModel(); Vocabulary vocab = new Vocabulary(); - vocab.setVocabularyUri(concept.getEntityId()); + vocab.setUri(concept.getEntityId()); vocab.setInScheme(concept.getInScheme()); vocab.setPrefLabel(concept.getPrefLabel()); vocabularyRepo.save(vocab); diff --git a/entity-management-web/src/test/resources/zoho_country_mapping.json b/entity-management-web/src/test/resources/zoho_country_mapping.json deleted file mode 100644 index f8831b757..000000000 --- a/entity-management-web/src/test/resources/zoho_country_mapping.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - { - } -] \ No newline at end of file diff --git a/entity-management-web/src/test/resources/zoho_role_mapping.json b/entity-management-web/src/test/resources/zoho_role_mapping.json deleted file mode 100644 index da1a058a9..000000000 --- a/entity-management-web/src/test/resources/zoho_role_mapping.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - { - } -]