Merge pull request #303 from europeana/EA-3720_zoho_eu_migration

address code quality issues #EA-3720

entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java

@@ -7,9 +7,9 @@
 import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
@@ -175,9 +175,8 @@ public class EntityManagementConfiguration implements InitializingBean {
   @Value("${europeana.role.vocabulary:role_vocabulary.xml}")
   private String roleVocabularyFilename;
 
-  Map<String, ZohoLabelUriMapping> countryMappings = new HashMap<>();
-  Map<String, String> wikidataCountryMappings = new HashMap<>();
-  Map<String, String> roleMappings = new HashMap<>();
+  private final Map<String, ZohoLabelUriMapping> countryMappings = new ConcurrentHashMap<>();
+  private final Map<String, String> roleMappings = new ConcurrentHashMap<>();
 
   @Autowired
   @Qualifier(BEAN_JSON_MAPPER)
@@ -236,17 +235,17 @@ private void initCountryMappings() throws IOException {
       try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
         String contents = reader.lines().collect(Collectors.joining(System.lineSeparator()));
         List<ZohoLabelUriMapping> countryMappingList = emJsonMapper.readValue(contents, new TypeReference<List<ZohoLabelUriMapping>>(){});
-        for (ZohoLabelUriMapping countryMapping : countryMappingList) {
-          //init zoho country mapping
-          countryMappings.put(countryMapping.getZohoLabel(), countryMapping);
-          //init wikidata country mapping 
-          if(StringUtils.isNotEmpty(countryMapping.getWikidataUri())){
-            wikidataCountryMappings.put(countryMapping.getWikidataUri(), countryMapping.getEntityUri()); 
-          }
-        }
+        addToCountryMappings(countryMappingList);
       }
     }
   }
+
+  void addToCountryMappings(List<ZohoLabelUriMapping> countryMappingList) {
+    for (ZohoLabelUriMapping countryMapping : countryMappingList) {
+      //init zoho country mapping
+      countryMappings.put(countryMapping.getZohoLabel(), countryMapping);
+    }
+  }
 
   private void initRoleMappings() throws IOException {
 
@@ -466,8 +465,5 @@ public Map<String, String> getRoleMappings() {
   public String getRoleVocabularyFilename() {
     return roleVocabularyFilename;
   }
-
-  public Map<String, String> getWikidataCountryMappings() {
-    return wikidataCountryMappings;
-  }
+
 }

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Address.java

@@ -17,6 +17,8 @@
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 import com.fasterxml.jackson.annotation.JsonSetter;
 import dev.morphia.annotations.Embedded;
+import dev.morphia.annotations.Transient;
+import eu.europeana.entitymanagement.vocabulary.WebEntityFields;
 
 @Embedded
 @JsonInclude(value = JsonInclude.Include.NON_EMPTY)
@@ -49,6 +51,8 @@ public Address(Address copy) {
     this.hasGeo = copy.getVcardHasGeo();
   }
 
+  @Transient
+  private String type = WebEntityFields.ADDRESS_TYPE;
   private String about;
   private String streetAddress;
   private String postalCode;
@@ -127,6 +131,11 @@ public void setVcardHasGeo(String hasGeo) {
     this.hasGeo = hasGeo;
   }
 
+  @JsonGetter(TYPE)
+  public String getType() {
+    return type;
+  }
+
   /** Checks that this Address metadata properties set. 'about' field not included in this check. */
   public boolean hasMetadataProperties() {
     return StringUtils.isNotEmpty(streetAddress)

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java

@@ -29,6 +29,7 @@
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.fasterxml.jackson.annotation.JsonTypeInfo.As;
+import dev.morphia.annotations.Transient;
 import eu.europeana.entitymanagement.normalization.EntityFieldsCompleteValidationGroup;
 import eu.europeana.entitymanagement.normalization.EntityFieldsCompleteValidationInterface;
 import eu.europeana.entitymanagement.normalization.EntityFieldsDataSourceProxyValidationGroup;
@@ -56,6 +57,9 @@
     groups = {EntityFieldsDataSourceProxyValidationGroup.class})
 public abstract class Entity implements ValidationObject {
 
+
+  @Transient
+  protected String context = ENTITY_CONTEXT;
   protected String entityId;
   // ID of entityRecord in database
 
@@ -83,6 +87,7 @@ protected Entity() {}
   protected <T extends Entity> Entity(T copy) {
     this.entityId = copy.getEntityId();
     this.depiction = copy.getDepiction();
+    this.context = copy.getContext();
     if (copy.getNote() != null) this.note = new HashMap<>(copy.getNote());
     if (copy.getPrefLabel() != null) this.prefLabel = new HashMap<>(copy.getPrefLabel());
     if (copy.getAltLabel() != null) this.altLabel = new HashMap<>(copy.getAltLabel());
@@ -254,8 +259,13 @@ public Aggregation getIsAggregatedBy() {
   /** Not included in XML responses */
   @JsonGetter(CONTEXT)
   public String getContext() {
-    return ENTITY_CONTEXT;
+    return context;
   }
+
+  public void setContext(String context) {
+    this.context = context;
+  }
+
 
   @JsonSetter(IS_AGGREGATED_BY)
   public void setIsAggregatedBy(Aggregation isAggregatedBy) {

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java

@@ -229,6 +229,9 @@ public Place getCountry() {
     if(country == null && getCountryId() != null) {
       //set country if not dereferenced during retrieval from database
       country = new Place(getCountryId());
+    }else if(country != null) {
+      //reset context to remove it from serialization
+      country.setContext(null);
     }
     return country;
   }

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Place.java

@@ -76,6 +76,8 @@ public Place() {
   public Place(String entityId) {
     super();
     setEntityId(entityId);
+    //reset default value for context 
+    setContext(null);
   }
 
   @JsonGetter(IS_NEXT_IN_SEQUENCE)

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/normalization/EntityFieldsCleaner.java

@@ -213,7 +213,12 @@ private Map<String, List<String>> normalizeMultipleValueMap(
 
   private List<String> normalizeValues(String fieldName, List<String> values) {
     List<String> normalized;
-    if (EntityFieldsTypes.getFieldType(fieldName).equals(FIELD_TYPE_DATE)) {
+    //process only functional fields defined in the field types
+    if(!EntityFieldsTypes.hasTypeDefinition(fieldName)) {
+      return values;
+    }
+
+    if (FIELD_TYPE_DATE.equals(EntityFieldsTypes.getFieldType(fieldName))) {
       normalized = new ArrayList<>();
       for (String value : values) {
         String normalizedValue = normalizeTextValue(fieldName, value);
@@ -264,7 +269,8 @@ private boolean isSingleValueStringMap(Field field) {
 
   private void normalizeTextField(Field field, String fieldValue, Entity entity)
       throws IllegalArgumentException, IllegalAccessException {
-    if (fieldValue != null) {
+    //process only the functional fields defined in the fieldTypes
+    if (fieldValue != null && EntityFieldsTypes.hasTypeDefinition(field.getName())) {
       String normalizedValue = normalizeTextValue(field.getName(), fieldValue);
       if (!normalizedValue.equals(fieldValue)) {
         entity.setFieldValue(field, normalizedValue);
@@ -273,11 +279,12 @@ private void normalizeTextField(Field field, String fieldValue, Entity entity)
   }
 
   String normalizeTextValue(String fieldName, String fieldValue) {
-    if (fieldValue != null) {
+    //process only the functional fields defined in the fieldTypes
+    if (fieldValue != null && EntityFieldsTypes.hasTypeDefinition(fieldName)) {
       // remove trailing spaces and capitalise Text fields
       String normalizedValue = capitaliseTextFields(fieldName, fieldValue);
 
-      if (EntityFieldsTypes.getFieldType(fieldName).equals(FIELD_TYPE_DATE)) {
+      if (FIELD_TYPE_DATE.equals(EntityFieldsTypes.getFieldType(fieldName))) {
         normalizedValue = convertDatetimeToDate(normalizedValue);
       }
       if (fieldValue != normalizedValue) {
@@ -309,17 +316,19 @@ String convertDatetimeToDate(String fieldValue) {
    * @return
    */
   private String capitaliseTextFields(String fieldName, String fieldValue) {
+
     // do not capitalize language code strings (e.g. en, de, fr)
-    if (!ISO_LANGUAGES.contains(fieldValue)) {
-      if (EntityFieldsTypes.getFieldType(fieldName).equals(FIELD_TYPE_TEXT)) {
+    // process only functional fields
+    if (EntityFieldsTypes.hasTypeDefinition(fieldName) && !ISO_LANGUAGES.contains(fieldValue)) {
+      if (FIELD_TYPE_TEXT.equals(EntityFieldsTypes.getFieldType(fieldName))) {
         return StringUtils.capitalize(fieldValue.trim());
       }
-      if (EntityFieldsTypes.getFieldType(fieldName).equals(FIELD_TYPE_TEXT_OR_URI)
+      if (FIELD_TYPE_TEXT_OR_URI.equals(EntityFieldsTypes.getFieldType(fieldName))
           && !(StringUtils.startsWithAny(fieldValue, "https://", "http://"))) {
         return StringUtils.capitalize(fieldValue.trim());
       }
       // for keyword field type leave it as it is
-      if (StringUtils.equals(EntityFieldsTypes.getFieldType(fieldName), FIELD_TYPE_KEYWORD)) {
+      if (StringUtils.equals(FIELD_TYPE_KEYWORD, EntityFieldsTypes.getFieldType(fieldName))) {
         return fieldValue.trim();
       }
     }

entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java

@@ -21,7 +21,6 @@
 import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants;
 import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException;
 import eu.europeana.entitymanagement.web.model.ZohoSyncReport;
-import eu.europeana.entitymanagement.web.service.EntityRecordService;
 import eu.europeana.entitymanagement.web.service.ZohoSyncService;
 
 /**
@@ -41,9 +40,7 @@ public class EntityManagementApp implements CommandLineRunner {
   private BatchEntityUpdateExecutor batchUpdateExecutor;
   @Autowired
   private ZohoSyncService zohoSyncService;
-  @Autowired
-  private EntityRecordService entityRecordService;
-
+
   /**
    * Main entry point of this application
    *
@@ -82,9 +79,9 @@ public static void main(String[] args) {
         //failed tasks will not complete, therefore not all scheduled tasks are marked as completed in the database
         //untill we have a better mechanism to reschedule failed tasks we wait for the next executions to mark them as complete
         if (currentRunningTasks == 0 || currentRunningTasks == notCompletedTasks){
-          //if the open tasks is the same after waiting interval, than the processing is considered complete  
+          //if the open tasks is the same after waiting interval, than the processing is considered complete
+          //reseting currentRunningTasks is not needed anymore
           processingComplete = true;
-          currentRunningTasks = 0;
         } else {
           processingComplete = false;
           notCompletedTasks = currentRunningTasks;
@@ -97,7 +94,7 @@ public static void main(String[] args) {
           SpringApplication.exit(context);
           System.exit(-2);
         }
-      } while (notCompletedTasks > 0);
+      } while (!processingComplete);
 
       // failed application execution should be indicated with negative codes
       LOG.info("Stoping application after processing all Schdeduled Tasks!");