load vocabulary at startup, revert solr schema, #EA-3641 #EA-3702

entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java

@@ -5,6 +5,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -21,6 +22,7 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.PropertySource;
 import org.springframework.context.annotation.PropertySources;
+import org.springframework.core.io.ClassPathResource;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.europeana.entitymanagement.definitions.model.ZohoLabelUriMapping;
@@ -165,11 +167,14 @@ public class EntityManagementConfiguration implements InitializingBean {
   @Value("${spring.profiles.active:}")
   private String activeProfileString;
 
-  @Value("${zoho.country.mapping:null}")
+  @Value("${zoho.country.mapping:zoho_country_mapping.json}")
   private String zohoCountryMappingFilename;
 
-  @Value("${zoho.role.mapping:null}")
+  @Value("${zoho.role.mapping:zoho_role_mapping.json}")
   private String zohoRoleMappingFilename;
+
+  @Value("${europeana.role.vocabulary:role_vocabulary.xml}")
+  private String roleVocabularyFilename;
 
   Map<String, ZohoLabelUriMapping> countryMappings = new HashMap<>();
   Map<String, String> roleMappings = new HashMap<>();
@@ -224,7 +229,9 @@ private void verifyRequiredProperties() {
   }
 
   private void initCountryMappings() throws IOException {
-    try (InputStream inputStream = getClass().getResourceAsStream(getZohoCountryMappingFilename())) {
+    ClassPathResource resource = new ClassPathResource(getZohoCountryMappingFilename());
+
+    try (InputStream inputStream = resource.getInputStream()) {
       assert inputStream != null;
       try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
         String contents = reader.lines().collect(Collectors.joining(System.lineSeparator()));
@@ -237,7 +244,10 @@ private void initCountryMappings() throws IOException {
   }
 
   private void initRoleMappings() throws IOException {
-    try (InputStream inputStream = getClass().getResourceAsStream(getZohoRoleMappingFilename())) {
+
+    ClassPathResource resource = new ClassPathResource(getZohoRoleMappingFilename());
+
+    try (InputStream inputStream =  resource.getInputStream()) {
       assert inputStream != null;
       try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
         String contents = reader.lines().collect(Collectors.joining(System.lineSeparator()));
@@ -249,6 +259,17 @@ private void initRoleMappings() throws IOException {
     }
   }
 
+  public String loadRoleVocabulary() throws IOException {
+    ClassPathResource resource = new ClassPathResource(getRoleVocabularyFilename());
+
+    try (InputStream inputStream = resource.getInputStream()) {
+      assert inputStream != null;
+      try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
+        return reader.lines().collect(Collectors.joining(System.lineSeparator()));
+      }
+    }
+  }
+
   public String getPrSolrUrl() {
     return prSolrUrl;
   }
@@ -436,4 +457,8 @@ public String getZohoRoleMappingFilename() {
   public Map<String, String> getRoleMappings() {
     return roleMappings;
   }
+
+  public String getRoleVocabularyFilename() {
+    return roleVocabularyFilename;
+  }
 }

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/VocabularyFields.java

@@ -7,6 +7,6 @@ private VocabularyFields() {
     // private constructor to prevent instantiation
   }
 
-  public static final String VOCABULARY_URI = "vocabularyUri";
+  public static final String URI = "uri";
 
 }

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java

@@ -95,7 +95,6 @@ public Organization(Organization copy) {
     if (copy.getPhone() != null) this.phone = new ArrayList<>(copy.getPhone());
     if (copy.getMbox() != null) this.mbox = new ArrayList<>(copy.getMbox());
     //because the europeanaRoleRef is a reference to the object we keep it the same (therefore also for europeanaRole)
-    this.europeanaRole = copy.getEuropeanaRole();
     this.europeanaRoleRefs = copy.getEuropeanaRoleRefs();
     if(copy.getEuropeanaRoleIds()!=null) this.europeanaRoleIds=new ArrayList<>(copy.getEuropeanaRoleIds());
     //because the countryRef is a reference to the object we keep it the same (therefore also for country)
@@ -272,7 +271,7 @@ public List<Vocabulary> getEuropeanaRole() {
       europeanaRole=new ArrayList<>();
       for(String roleId : europeanaRoleIds) {
         Vocabulary vocab = new Vocabulary();
-        vocab.setVocabularyUri(roleId);
+        vocab.setUri(roleId);
         europeanaRole.add(vocab);
       }
     }

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Vocabulary.java

@@ -21,14 +21,17 @@
 @JsonInclude(value = JsonInclude.Include.NON_EMPTY)
 @dev.morphia.annotations.Entity("Vocabulary")
 @EntityListeners(VocabularyWatcher.class)
+/**
+ * class used for storing static vocabularies (e.g. europeana roles)
+ */
 public class Vocabulary {
 
   private String type = "Vocabulary";
 
   @Id @JsonIgnore private ObjectId dbId;
 
   @Indexed(options = @IndexOptions(unique = true))
-  private String vocabularyUri;
+  private String uri;
 
   protected List<String> inScheme;
 
@@ -40,6 +43,9 @@ public class Vocabulary {
   @JsonIgnore 
   private Date modified;
 
+  /**
+   * default constructor
+   */
   public Vocabulary() {
   }
 
@@ -69,17 +75,17 @@ public Date getModified() {
 
   @Override
   public String toString() {
-    return String.format("Vocabulary.vocabularyUri: %s", getVocabularyUri());
+    return String.format("Vocabulary.uri: %s", getUri());
   }
 
   @JsonGetter
-  public String getVocabularyUri() {
-    return vocabularyUri;
+  public String getUri() {
+    return uri;
   }
 
   @JsonSetter
-  public void setVocabularyUri(String vocabularyUri) {
-    this.vocabularyUri = vocabularyUri;
+  public void setUri(String vocabularyUri) {
+    this.uri = vocabularyUri;
   }
 
   @JsonGetter(IN_SCHEME)

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java

@@ -113,7 +113,7 @@ public XmlOrganizationImpl(Organization organization) {
       List<XmlConceptImpl> orgXmlRole= new ArrayList<>();
       for(Vocabulary vocab : orgRole) {
         XmlConceptImpl xmlConcept = new XmlConceptImpl();
-        xmlConcept.setAbout(vocab.getVocabularyUri());
+        xmlConcept.setAbout(vocab.getUri());
         xmlConcept.setPrefLabel(RdfXmlUtils.convertMapToXmlMultilingualString(vocab.getPrefLabel()));
         xmlConcept.setInScheme(RdfXmlUtils.convertToRdfResource(vocab.getInScheme()));
         orgXmlRole.add(xmlConcept);
@@ -149,30 +149,18 @@ public Organization toEntityModel() throws EntityModelCreationException {
     entity.setAcronym(RdfXmlUtils.toLanguageMapList(getAcronym()));
     entity.setDescription(RdfXmlUtils.toLanguageMap(getDescription()));
     entity.setLogo(XmlWebResourceWrapper.toWebResource(getLogo()));
-    //set europeanaRole
+    //set europeanaRole id (external dereferencers deliver only the ids, not transitive data)
     if(getEuropeanaRole()!=null && !getEuropeanaRole().isEmpty()) {
-      List<String> roleIds=new ArrayList<>();
-      List<Vocabulary> role=new ArrayList<>();
-      for(XmlConceptImpl xmlConcept : getEuropeanaRole()) {
-        Vocabulary vocab=new Vocabulary();
-        vocab.setVocabularyUri(xmlConcept.getAbout());
-        vocab.setPrefLabel(RdfXmlUtils.toLanguageMap(xmlConcept.getPrefLabel()));
-        vocab.setInScheme(RdfXmlUtils.toStringList(xmlConcept.getInScheme()));
-        role.add(vocab);
-        roleIds.add(xmlConcept.getAbout());
-      }
-      entity.setEuropeanaRole(role);
+      List<String> roleIds = getEuropeanaRole().stream().map(e -> e.getAbout()).toList();
       entity.setEuropeanaRoleIds(roleIds);
     }
 
+    //set country id (external dereferencers deliver only the ids, not transitive data)
     if(getCountry() != null) {
-      //the country is not saved in the database 
-      entity.setCountry(getCountry().toEntityModel());
       //we need to extract the countryID as well (xml about holds the entityId)
       entity.setCountryId(getCountry().getAbout());
     }
 
-
     if (getHomepage() != null) {
       entity.setHomepage(getHomepage().getResource());
     }

entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java

@@ -1,7 +1,7 @@
 package eu.europeana.entitymanagement.mongo.repository;
 
 import static dev.morphia.query.experimental.filters.Filters.in;
-import static eu.europeana.entitymanagement.definitions.VocabularyFields.VOCABULARY_URI;
+import static eu.europeana.entitymanagement.definitions.VocabularyFields.URI;
 import java.util.ArrayList;
 import java.util.List;
 import javax.annotation.Resource;
@@ -18,24 +18,50 @@ public class VocabularyRepository  {
   @Resource(name = AppConfigConstants.BEAN_EM_DATA_STORE)
   Datastore datastore;
 
-  public List<Vocabulary> findByVocabularyUris(List<String> vocabularyUris) {
+  /**
+   * retrieve records by their URI/id
+   * @param vocabularyUris
+   * @return
+   */
+  public List<Vocabulary> findByUri(List<String> vocabularyUris) {
     List<Filter> filters = new ArrayList<>();
-    filters.add(in(VOCABULARY_URI, vocabularyUris));
+    filters.add(in(URI, vocabularyUris));
     return datastore.find(Vocabulary.class)
         .filter(filters.toArray(Filter[]::new))
         .iterator()
         .toList();
   }
 
+  /**
+   * save to database 
+   * @param vocab record to save
+   * @return saved record
+   */
   public Vocabulary save(Vocabulary vocab) {
     return datastore.save(vocab);
   }
 
+  /**
+   * save list of records to database
+   * @param vocabs list of records to save
+   * @return saved records
+   */
   public List<Vocabulary> saveBulk(List<Vocabulary> vocabs) {
     return datastore.save(vocabs);
   }
 
+  /**
+   * clear database collection
+   */
   public void dropCollection() {
     datastore.getMapper().getCollection(Vocabulary.class).drop();
   }
+
+  /**
+   * count the records available in the database
+   * @return number of database records
+   */
+  public long countRecords() {
+    return datastore.find(Vocabulary.class).count();
+  }
 }

entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java

@@ -36,7 +36,8 @@ public class SolrOrganization extends SolrEntity<Organization> {
   @Field(OrganizationSolrFields.FOAF_MBOX)
   private List<String> mbox;
 
-  @Field(OrganizationSolrFields.EUROPEANA_ROLE)
+  //MAPPING TO BE ENABLED BACK IN THE NEXT VERSION - requires schema and ENtity API UPDATE
+  //@Field(OrganizationSolrFields.EUROPEANA_ROLE)
   private List<String> europeanaRole;
 
   @Field(OrganizationSolrFields.COUNTRY)

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java

@@ -151,8 +151,7 @@ static void setProperties(DynamicPropertyRegistry registry) {
     //generate Europeana ID can be set to true when using the mock service, see TextConfig class 
     registry.add("zoho.generate.organization.europeanaid", () -> true);
 
-    registry.add("zoho.country.mapping", () -> "/zoho_country_mapping_test.json");
-    registry.add("zoho.role.mapping", () -> "/zoho_role_mapping_test.json");
+    registry.add("zoho.country.mapping", () -> "zoho_country_mapping_test.json");
 
     // could be used to fix eclipse issues
     registry.add("scmBranch", () -> "dev");

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java

@@ -14,7 +14,7 @@
 import eu.europeana.entitymanagement.batch.repository.FailedTaskRepository;
 import eu.europeana.entitymanagement.batch.service.ScheduledTaskService;
 import eu.europeana.entitymanagement.batch.utils.BatchUtils;
-import eu.europeana.entitymanagement.config.AppConfig;
+import eu.europeana.entitymanagement.config.AppAutoconfig;
 import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord;
 import eu.europeana.entitymanagement.definitions.batch.model.FailedTask;
 import eu.europeana.entitymanagement.definitions.batch.model.FailedTask.Builder;
@@ -63,7 +63,7 @@ class ScheduledTaskServiceIT extends AbstractIntegrationTest {
 
   @Autowired EntityUpdateJobConfig updateJobConfig;
 
-  @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE)
+  @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE)
   @Autowired
   private SolrService emSolrService;
 

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScoringServiceIT.java

@@ -5,7 +5,7 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import eu.europeana.entitymanagement.AbstractIntegrationTest;
-import eu.europeana.entitymanagement.config.AppConfig;
+import eu.europeana.entitymanagement.config.AppAutoconfig;
 import eu.europeana.entitymanagement.definitions.model.Agent;
 import eu.europeana.entitymanagement.definitions.model.Place;
 import eu.europeana.entitymanagement.definitions.model.TimeSpan;
@@ -26,7 +26,7 @@
 @AutoConfigureMockMvc
 class ScoringServiceIT extends AbstractIntegrationTest {
 
-  @Resource(name = AppConfig.BEAN_EM_SCORING_SERVICE)
+  @Resource(name = AppAutoconfig.BEAN_EM_SCORING_SERVICE)
   ScoringService scoringService;
 
   // @Test

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java

@@ -22,7 +22,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.europeana.entitymanagement.AbstractIntegrationTest;
 import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants;
-import eu.europeana.entitymanagement.config.AppConfig;
+import eu.europeana.entitymanagement.config.AppAutoconfig;
 import eu.europeana.entitymanagement.definitions.model.Agent;
 import eu.europeana.entitymanagement.definitions.model.Aggregation;
 import eu.europeana.entitymanagement.definitions.model.Concept;
@@ -48,7 +48,7 @@ public class SolrServiceIT extends AbstractIntegrationTest {
 
   //  private static final String RIGHTS_PD = "https://creativecommons.org/publicdomain/zero/1.0/";
 
-  @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE)
+  @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE)
   @Autowired
   private SolrService emSolrService;
 

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java

@@ -23,7 +23,7 @@
 import eu.europeana.entitymanagement.batch.service.EntityUpdateService;
 import eu.europeana.entitymanagement.batch.service.ScheduledTaskService;
 import eu.europeana.entitymanagement.common.config.DataSource;
-import eu.europeana.entitymanagement.config.AppConfig;
+import eu.europeana.entitymanagement.config.AppAutoconfig;
 import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask;
 import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType;
 import eu.europeana.entitymanagement.definitions.model.Entity;
@@ -49,7 +49,7 @@ abstract class BaseWebControllerTest extends AbstractIntegrationTest {
 
   @Autowired private EntityUpdateService entityUpdateService;
 
-  @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE)
+  @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE)
   @Autowired
   protected SolrService emSolrService;
 

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java

@@ -330,6 +330,8 @@ void registerZohoOrganizationBnfWithNewFieldsShouldBeSuccessful() throws Excepti
         .andExpect(jsonPath("$.language", everyItem(matchesRegex("[a-z]+"))))
         .andExpect(jsonPath("$.hiddenLabel", hasSize(3)))
         .andExpect(jsonPath("$.id", any(String.class)));
+        // excluded as the wikidata hasGeo is not consolidated  
+        //.andExpect(jsonPath("$.hasAddress.hasGeo").isNotEmpty())
   }
 
   @Test