metrics updates must used dereferenced organizations #EA-3721

entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java

@@ -175,7 +175,14 @@ public class EntityManagementConfiguration implements InitializingBean {
   @Value("${europeana.role.vocabulary:role_vocabulary.xml}")
   private String roleVocabularyFilename;
 
+  /**
+   * Map of <"Zoho Label", ZohoLabelUriMapping>  
+   */
   private final Map<String, ZohoLabelUriMapping> countryMappings = new ConcurrentHashMap<>();
+  /**
+   * Map of <"EntityId", ZohoLabelUriMapping>  
+   */
+  private final Map<String, ZohoLabelUriMapping> countryIdMappings = new ConcurrentHashMap<>();
   private final Map<String, String> roleMappings = new ConcurrentHashMap<>();
 
   @Autowired
@@ -236,6 +243,7 @@ private void initCountryMappings() throws IOException {
         String contents = reader.lines().collect(Collectors.joining(System.lineSeparator()));
         List<ZohoLabelUriMapping> countryMappingList = emJsonMapper.readValue(contents, new TypeReference<List<ZohoLabelUriMapping>>(){});
         addToCountryMappings(countryMappingList);
+        addToCountryIdMappings(countryMappingList);
       }
     }
   }
@@ -247,6 +255,13 @@ void addToCountryMappings(List<ZohoLabelUriMapping> countryMappingList) {
     }
   }
 
+  void addToCountryIdMappings(List<ZohoLabelUriMapping> countryMappingList) {
+    for (ZohoLabelUriMapping countryMapping : countryMappingList) {
+      //init entityID - to ZohoCountry mapping 
+      countryIdMappings.put(countryMapping.getZohoLabel(), countryMapping);
+    }
+  }
+
   private void initRoleMappings() throws IOException {
 
     ClassPathResource resource = new ClassPathResource(getZohoRoleMappingFilename());
@@ -465,5 +480,9 @@ public Map<String, String> getRoleMappings() {
   public String getRoleVocabularyFilename() {
     return roleVocabularyFilename;
   }
+
+  public Map<String, ZohoLabelUriMapping> getCountryIdMappings() {
+    return countryIdMappings;
+  }
 
 }

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java

@@ -3,6 +3,7 @@
 import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_EXACT_MATCH;
 import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_SAME_AS;
 import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_AGGREGATED_VIA;
+import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_TYPE;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.BASE_DATA_EUROPEANA_URI;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ID;
 import java.util.ArrayList;
@@ -26,7 +27,10 @@
 
 @JsonInclude(value = JsonInclude.Include.NON_EMPTY)
 @dev.morphia.annotations.Entity("EntityRecord")
-@Indexes({@Index(fields = {@Field(ENTITY_EXACT_MATCH)}), @Index(fields = {@Field(ENTITY_SAME_AS)}), 
+@Indexes({
+  @Index(fields = {@Field(ENTITY_TYPE)}),
+  @Index(fields = {@Field(ENTITY_EXACT_MATCH)}), 
+  @Index(fields = {@Field(ENTITY_SAME_AS)}),
   @Index(fields = {@Field(ENTITY_AGGREGATED_VIA)})})
 @EntityListeners(EntityRecordWatcher.class)
 public class EntityRecord {

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/ZohoLabelUriMapping.java

@@ -3,6 +3,7 @@
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ENTITY_URI;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.WIKIDATA_URI;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ZOHO_LABEL;
+import org.apache.commons.lang3.StringUtils;
 import com.fasterxml.jackson.annotation.JsonGetter;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
@@ -49,4 +50,12 @@ public String getWikidataUri() {
   public void setWikidataUri(String wikidataUri) {
     this.wikidataUri = wikidataUri;
   }
+
+  public String getCountryISOCode() {
+    if (getZohoLabel() != null) {
+      return StringUtils.substringAfterLast(getZohoLabel(), ",").trim();
+    }
+    return null;
+  }
+
 }

entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java

@@ -13,6 +13,7 @@
 import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord;
 import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType;
 import eu.europeana.entitymanagement.definitions.model.EntityRecord;
+import eu.europeana.entitymanagement.vocabulary.EntityProfile;
 import eu.europeana.entitymanagement.web.service.EntityRecordService;
 
 /** {@link ItemReader} that reads documents from MongoDB via a paging technique. */
@@ -42,7 +43,7 @@ protected Iterator<BatchEntityRecord> doPageRead() {
     int start = page * pageSize;
     //the dereference profile is not needed here as the consolidation is taking care of processing references
     List<EntityRecord> result =
-        entityRecordService.findEntitiesWithFilter(start, pageSize, queryFilters, null);
+        entityRecordService.findEntitiesWithFilter(start, pageSize, queryFilters, EntityProfile.dereference.name());
 
     List<BatchEntityRecord> batchEntityRecords = toBatchEntityRecords(result, scheduledTaskType);
 

entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java

@@ -10,6 +10,7 @@
 import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord;
 import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask;
 import eu.europeana.entitymanagement.definitions.model.EntityRecord;
+import eu.europeana.entitymanagement.vocabulary.EntityProfile;
 import eu.europeana.entitymanagement.web.service.EntityRecordService;
 
 /**
@@ -42,7 +43,7 @@ protected Iterator<BatchEntityRecord> doPageRead() {
 
     List<String> entityIds = scheduledTasks.stream().map(st -> st.getEntityId()).toList();
     //no need to use the dereference profile here as the consolidation takes care of processing references
-    List<EntityRecord> records = entityRecordService.retrieveMultipleByEntityIds(entityIds, false, true, null);
+    List<EntityRecord> records = entityRecordService.retrieveMultipleByEntityIds(entityIds, false, true, EntityProfile.dereference.name());
     List<BatchEntityRecord> batchRecords = toBatchEntityRecords(records, scheduledTasks);
 
     if (logger.isDebugEnabled()) {

entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java

@@ -39,6 +39,7 @@
 import eu.europeana.entitymanagement.definitions.model.Organization;
 import eu.europeana.entitymanagement.definitions.model.Place;
 import eu.europeana.entitymanagement.definitions.model.TimeSpan;
+import eu.europeana.entitymanagement.definitions.model.ZohoLabelUriMapping;
 import eu.europeana.entitymanagement.definitions.web.EntityIdDisabledStatus;
 import eu.europeana.entitymanagement.exception.EntityAlreadyExistsException;
 import eu.europeana.entitymanagement.exception.EntityCreationException;
@@ -182,6 +183,12 @@ public void dereferenceLinkedEntities(Organization org) {
       EntityRecord countryRecord = entityRecordRepository.findByEntityId(org.getCountryId(),
           new String[] {EntityRecordFields.ENTITY});
       setDereferencedCountry(org, countryRecord);
+
+      ZohoLabelUriMapping mapping = emConfiguration.getCountryIdMappings().get(org.getEntityId());
+      if(mapping != null) {
+        //extract ISO code from ZohoCountry
+        org.setCountryISO(mapping.getCountryISOCode());  
+      }
     }
     // dereference role
     if (org.getEuropeanaRoleIds() != null && !org.getEuropeanaRoleIds().isEmpty()) {

entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java

@@ -80,12 +80,12 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String
       //update address country
       address.setVcardCountryName(extractCountryName(zohoCountryLabel));
 
-      org.setCountryISO(StringUtils.substringAfterLast(zohoCountryLabel, ",").trim());
       //update organization country id
       if(countryMappings.containsKey(zohoCountryLabel)) {
         //get country ID from mappings
-        String entityUri = countryMappings.get(zohoCountryLabel).getEntityUri();
-        org.setCountryId(entityUri);
+        ZohoLabelUriMapping zohoLabelUriMapping = countryMappings.get(zohoCountryLabel);
+        org.setCountryId(zohoLabelUriMapping.getEntityUri());
+        org.setCountryISO(zohoLabelUriMapping.getCountryISOCode());
       } else if(logger.isInfoEnabled()){
         logger.info("The mapping for the zoho country label: {}, to the europeana uri does not exist.", zohoCountryLabel);
       }