diff --git a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java index 368a3018..cfc40b59 100644 --- a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java +++ b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java @@ -7,9 +7,9 @@ import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -175,9 +175,8 @@ public class EntityManagementConfiguration implements InitializingBean { @Value("${europeana.role.vocabulary:role_vocabulary.xml}") private String roleVocabularyFilename; - Map countryMappings = new HashMap<>(); - Map wikidataCountryMappings = new HashMap<>(); - Map roleMappings = new HashMap<>(); + private final Map countryMappings = new ConcurrentHashMap<>(); + private final Map roleMappings = new ConcurrentHashMap<>(); @Autowired @Qualifier(BEAN_JSON_MAPPER) @@ -236,17 +235,17 @@ private void initCountryMappings() throws IOException { try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { String contents = reader.lines().collect(Collectors.joining(System.lineSeparator())); List countryMappingList = emJsonMapper.readValue(contents, new TypeReference>(){}); - for (ZohoLabelUriMapping countryMapping : countryMappingList) { - //init zoho country mapping - countryMappings.put(countryMapping.getZohoLabel(), countryMapping); - //init wikidata country mapping - if(StringUtils.isNotEmpty(countryMapping.getWikidataUri())){ - wikidataCountryMappings.put(countryMapping.getWikidataUri(), countryMapping.getEntityUri()); - } - } + addToCountryMappings(countryMappingList); } } } + + void addToCountryMappings(List countryMappingList) { + for (ZohoLabelUriMapping countryMapping : countryMappingList) { + //init zoho country mapping + countryMappings.put(countryMapping.getZohoLabel(), countryMapping); + } + } private void initRoleMappings() throws IOException { @@ -466,8 +465,5 @@ public Map getRoleMappings() { public String getRoleVocabularyFilename() { return roleVocabularyFilename; } - - public Map getWikidataCountryMappings() { - return wikidataCountryMappings; - } + } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Address.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Address.java index 66f73eeb..e4f46c60 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Address.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Address.java @@ -17,6 +17,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonSetter; import dev.morphia.annotations.Embedded; +import dev.morphia.annotations.Transient; +import eu.europeana.entitymanagement.vocabulary.WebEntityFields; @Embedded @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @@ -49,6 +51,8 @@ public Address(Address copy) { this.hasGeo = copy.getVcardHasGeo(); } + @Transient + private String type = WebEntityFields.ADDRESS_TYPE; private String about; private String streetAddress; private String postalCode; @@ -127,6 +131,11 @@ public void setVcardHasGeo(String hasGeo) { this.hasGeo = hasGeo; } + @JsonGetter(TYPE) + public String getType() { + return type; + } + /** Checks that this Address metadata properties set. 'about' field not included in this check. */ public boolean hasMetadataProperties() { return StringUtils.isNotEmpty(streetAddress) diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java index 79177bba..c733595b 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java @@ -29,6 +29,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import dev.morphia.annotations.Transient; import eu.europeana.entitymanagement.normalization.EntityFieldsCompleteValidationGroup; import eu.europeana.entitymanagement.normalization.EntityFieldsCompleteValidationInterface; import eu.europeana.entitymanagement.normalization.EntityFieldsDataSourceProxyValidationGroup; @@ -56,6 +57,9 @@ groups = {EntityFieldsDataSourceProxyValidationGroup.class}) public abstract class Entity implements ValidationObject { + + @Transient + protected String context = ENTITY_CONTEXT; protected String entityId; // ID of entityRecord in database @@ -83,6 +87,7 @@ protected Entity() {} protected Entity(T copy) { this.entityId = copy.getEntityId(); this.depiction = copy.getDepiction(); + this.context = copy.getContext(); if (copy.getNote() != null) this.note = new HashMap<>(copy.getNote()); if (copy.getPrefLabel() != null) this.prefLabel = new HashMap<>(copy.getPrefLabel()); if (copy.getAltLabel() != null) this.altLabel = new HashMap<>(copy.getAltLabel()); @@ -254,8 +259,13 @@ public Aggregation getIsAggregatedBy() { /** Not included in XML responses */ @JsonGetter(CONTEXT) public String getContext() { - return ENTITY_CONTEXT; + return context; } + + public void setContext(String context) { + this.context = context; + } + @JsonSetter(IS_AGGREGATED_BY) public void setIsAggregatedBy(Aggregation isAggregatedBy) { diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java index 335f267f..da32e1a8 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java @@ -229,6 +229,9 @@ public Place getCountry() { if(country == null && getCountryId() != null) { //set country if not dereferenced during retrieval from database country = new Place(getCountryId()); + }else if(country != null) { + //reset context to remove it from serialization + country.setContext(null); } return country; } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Place.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Place.java index 4fa2a145..af1e086e 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Place.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Place.java @@ -76,6 +76,8 @@ public Place() { public Place(String entityId) { super(); setEntityId(entityId); + //reset default value for context + setContext(null); } @JsonGetter(IS_NEXT_IN_SEQUENCE) diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/normalization/EntityFieldsCleaner.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/normalization/EntityFieldsCleaner.java index a5b38276..fcdc9d40 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/normalization/EntityFieldsCleaner.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/normalization/EntityFieldsCleaner.java @@ -213,7 +213,12 @@ private Map> normalizeMultipleValueMap( private List normalizeValues(String fieldName, List values) { List normalized; - if (EntityFieldsTypes.getFieldType(fieldName).equals(FIELD_TYPE_DATE)) { + //process only functional fields defined in the field types + if(!EntityFieldsTypes.hasTypeDefinition(fieldName)) { + return values; + } + + if (FIELD_TYPE_DATE.equals(EntityFieldsTypes.getFieldType(fieldName))) { normalized = new ArrayList<>(); for (String value : values) { String normalizedValue = normalizeTextValue(fieldName, value); @@ -264,7 +269,8 @@ private boolean isSingleValueStringMap(Field field) { private void normalizeTextField(Field field, String fieldValue, Entity entity) throws IllegalArgumentException, IllegalAccessException { - if (fieldValue != null) { + //process only the functional fields defined in the fieldTypes + if (fieldValue != null && EntityFieldsTypes.hasTypeDefinition(field.getName())) { String normalizedValue = normalizeTextValue(field.getName(), fieldValue); if (!normalizedValue.equals(fieldValue)) { entity.setFieldValue(field, normalizedValue); @@ -273,11 +279,12 @@ private void normalizeTextField(Field field, String fieldValue, Entity entity) } String normalizeTextValue(String fieldName, String fieldValue) { - if (fieldValue != null) { + //process only the functional fields defined in the fieldTypes + if (fieldValue != null && EntityFieldsTypes.hasTypeDefinition(fieldName)) { // remove trailing spaces and capitalise Text fields String normalizedValue = capitaliseTextFields(fieldName, fieldValue); - if (EntityFieldsTypes.getFieldType(fieldName).equals(FIELD_TYPE_DATE)) { + if (FIELD_TYPE_DATE.equals(EntityFieldsTypes.getFieldType(fieldName))) { normalizedValue = convertDatetimeToDate(normalizedValue); } if (fieldValue != normalizedValue) { @@ -309,17 +316,19 @@ String convertDatetimeToDate(String fieldValue) { * @return */ private String capitaliseTextFields(String fieldName, String fieldValue) { + // do not capitalize language code strings (e.g. en, de, fr) - if (!ISO_LANGUAGES.contains(fieldValue)) { - if (EntityFieldsTypes.getFieldType(fieldName).equals(FIELD_TYPE_TEXT)) { + // process only functional fields + if (EntityFieldsTypes.hasTypeDefinition(fieldName) && !ISO_LANGUAGES.contains(fieldValue)) { + if (FIELD_TYPE_TEXT.equals(EntityFieldsTypes.getFieldType(fieldName))) { return StringUtils.capitalize(fieldValue.trim()); } - if (EntityFieldsTypes.getFieldType(fieldName).equals(FIELD_TYPE_TEXT_OR_URI) + if (FIELD_TYPE_TEXT_OR_URI.equals(EntityFieldsTypes.getFieldType(fieldName)) && !(StringUtils.startsWithAny(fieldValue, "https://", "http://"))) { return StringUtils.capitalize(fieldValue.trim()); } // for keyword field type leave it as it is - if (StringUtils.equals(EntityFieldsTypes.getFieldType(fieldName), FIELD_TYPE_KEYWORD)) { + if (StringUtils.equals(FIELD_TYPE_KEYWORD, EntityFieldsTypes.getFieldType(fieldName))) { return fieldValue.trim(); } } diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java index 8879b896..056bc71b 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java @@ -21,7 +21,6 @@ import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; import eu.europeana.entitymanagement.web.model.ZohoSyncReport; -import eu.europeana.entitymanagement.web.service.EntityRecordService; import eu.europeana.entitymanagement.web.service.ZohoSyncService; /** @@ -41,9 +40,7 @@ public class EntityManagementApp implements CommandLineRunner { private BatchEntityUpdateExecutor batchUpdateExecutor; @Autowired private ZohoSyncService zohoSyncService; - @Autowired - private EntityRecordService entityRecordService; - + /** * Main entry point of this application * @@ -82,9 +79,9 @@ public static void main(String[] args) { //failed tasks will not complete, therefore not all scheduled tasks are marked as completed in the database //untill we have a better mechanism to reschedule failed tasks we wait for the next executions to mark them as complete if (currentRunningTasks == 0 || currentRunningTasks == notCompletedTasks){ - //if the open tasks is the same after waiting interval, than the processing is considered complete + //if the open tasks is the same after waiting interval, than the processing is considered complete + //reseting currentRunningTasks is not needed anymore processingComplete = true; - currentRunningTasks = 0; } else { processingComplete = false; notCompletedTasks = currentRunningTasks; @@ -97,7 +94,7 @@ public static void main(String[] args) { SpringApplication.exit(context); System.exit(-2); } - } while (notCompletedTasks > 0); + } while (!processingComplete); // failed application execution should be indicated with negative codes LOG.info("Stoping application after processing all Schdeduled Tasks!"); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseEntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseEntityRecordService.java new file mode 100644 index 00000000..cb6c7dbf --- /dev/null +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseEntityRecordService.java @@ -0,0 +1,699 @@ +package eu.europeana.entitymanagement.web.service; + +import static eu.europeana.entitymanagement.utils.EntityRecordUtils.getDatasourceAggregationId; +import static eu.europeana.entitymanagement.utils.EntityRecordUtils.getEuropeanaAggregationId; +import static eu.europeana.entitymanagement.utils.EntityRecordUtils.getEuropeanaProxyId; +import static eu.europeana.entitymanagement.utils.EntityRecordUtils.getIsAggregatedById; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.ClassUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.util.CollectionUtils; +import eu.europeana.api.commons.error.EuropeanaApiException; +import eu.europeana.entitymanagement.common.config.DataSource; +import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; +import eu.europeana.entitymanagement.config.DataSources; +import eu.europeana.entitymanagement.definitions.exceptions.EntityModelCreationException; +import eu.europeana.entitymanagement.definitions.model.Aggregation; +import eu.europeana.entitymanagement.definitions.model.Entity; +import eu.europeana.entitymanagement.definitions.model.EntityProxy; +import eu.europeana.entitymanagement.definitions.model.EntityRecord; +import eu.europeana.entitymanagement.definitions.model.Organization; +import eu.europeana.entitymanagement.definitions.model.Vocabulary; +import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; +import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; +import eu.europeana.entitymanagement.mongo.repository.VocabularyRepository; +import eu.europeana.entitymanagement.solr.service.SolrService; +import eu.europeana.entitymanagement.utils.EMCollectionUtils; +import eu.europeana.entitymanagement.utils.EntityObjectFactory; +import eu.europeana.entitymanagement.utils.EntityRecordUtils; +import eu.europeana.entitymanagement.utils.UriValidator; +import eu.europeana.entitymanagement.vocabulary.EntityFieldsTypes; +import eu.europeana.entitymanagement.vocabulary.WebEntityFields; +import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; + +public class BaseEntityRecordService { + + final EntityRecordRepository entityRecordRepository; + + final VocabularyRepository vocabRepository; + + final EntityManagementConfiguration emConfiguration; + + final DataSources datasources; + + final SolrService solrService; + + final ZohoConfiguration zohoConfiguration; + + protected final Logger logger = LogManager.getLogger(getClass()); + + // Fields to be ignored during consolidation ("type" is final) + static final Set ignoredMergeFields = Set.of(WebEntityFields.TYPE); + + + protected BaseEntityRecordService(EntityRecordRepository entityRecordRepository, + VocabularyRepository vocabRepository, EntityManagementConfiguration emConfiguration, + ZohoConfiguration zohoConfiguration, DataSources datasources, SolrService solrService) { + this.entityRecordRepository = entityRecordRepository; + this.vocabRepository = vocabRepository; + this.emConfiguration = emConfiguration; + this.zohoConfiguration = zohoConfiguration; + this.datasources = datasources; + this.solrService = solrService; + } + + /** + * Gets that have at list one of the provided URIs present in co-references (sameAs or exactMatch + * value in the Consolidated version). The current entity identified by the entityId, if not null + * or empty is excluded. Exclude disabled indicates if the disabled entities should be included in + * the response or not + * + * @param uris co-reference uris + * @param entityId URI indicating the the record for the given entityId should not be retrieved as + * matchingCoreference + * @param excludeDisabled indicated if the disabled entities should be filtered out or not + * @return List of dupplicated entities or empty. + */ + public List findEntitiesByCoreference(List uris, String entityId, + boolean excludeDisabled) { + return entityRecordRepository.findEntitiesByCoreference(uris, entityId, excludeDisabled); + } + + protected void addValueOrInternalReference(List updatedReferences, String value) { + if (value.startsWith(WebEntityFields.BASE_DATA_EUROPEANA_URI) || !UriValidator.isUri(value)) { + // value is internal reference or string literal + updatedReferences.add(value); + } else { + // value is external URI, replace it with internal reference if they are accessible + // do not use disabled entities as they are not accessible anymore + List records = + findEntitiesByCoreference(Collections.singletonList(value), null, true); + if (!records.isEmpty()) { + // if the prevention of dupplication worked propertly, that we should find only one active + // entry in the database + updatedReferences.add(records.get(0).getEntityId()); + } + } + } + + + /** + * Merges metadata between two entities. This method performs a deep copy of the objects, for the + * mutable (custom) field types. + * + * @param primary Primary entity. Metadata from this entity takes precedence + * @param secondary Secondary entity. Metadata from this entity is only used if no matching field + * is contained within the primary entity. + * @param fieldsToCombine metadata fields to reconcile + * @param accumulate if true, metadata from the secondary entity are added to the matching + * collection (eg. maps, lists and arrays) within the primary . If accumulate is false, the + * "primary" content overwrites the "secondary" + */ + protected Entity combineEntities(Entity primary, Entity secondary, List fieldsToCombine, + boolean accumulate) throws EuropeanaApiException, EntityModelCreationException { + Entity consolidatedEntity = + EntityObjectFactory.createConsolidatedEntityObject(primary.getType()); + + try { + + /* + * store the preferred label in the secondary entity that is different from the preferred + * label in the primary entity to the alternative labels of the consolidated entity + */ + Map prefLabelsForAltLabels = new HashMap<>(); + + for (Field field : fieldsToCombine) { + Class fieldType = field.getType(); + String fieldName = field.getName(); + + if (isStringOrPrimitive(fieldType)) { + mergePrimitiveField(field, primary, secondary, consolidatedEntity); + } else if (Date.class.isAssignableFrom(fieldType)) { + mergeDateField(field, primary, secondary, consolidatedEntity); + } else if (fieldType.isArray()) { + Object[] mergedArray = mergeArrays(primary, secondary, field, accumulate); + consolidatedEntity.setFieldValue(field, mergedArray); + } else if (List.class.isAssignableFrom(fieldType)) { + mergeListField(field, primary, secondary, consolidatedEntity, accumulate); + } else if (Map.class.isAssignableFrom(fieldType)) { + combineEntities(consolidatedEntity, primary, secondary, prefLabelsForAltLabels, field, + fieldName, accumulate); + } else { + mergeCustomObjects(primary, secondary, field, consolidatedEntity); + } + } + + mergeSkippedPrefLabels(consolidatedEntity, prefLabelsForAltLabels, fieldsToCombine); + + } catch (IllegalAccessException e) { + throw new EntityUpdateException( + "Metadata consolidation failed to access required properties!", e); + } + + return consolidatedEntity; + } + + @SuppressWarnings("unchecked") + void mergeListField(Field field, Entity primary, Entity secondary, Entity consolidatedEntity, + boolean accumulate) throws IllegalAccessException, EntityUpdateException { + List fieldValuePrimaryObjectList = (List) primary.getFieldValue(field); + List fieldValueSecondaryObjectList = (List) secondary.getFieldValue(field); + mergeList(consolidatedEntity, fieldValuePrimaryObjectList, fieldValueSecondaryObjectList, field, + accumulate); + } + + void mergeDateField(Field field, Entity primary, Entity secondary, Entity consolidatedEntity) + throws IllegalAccessException { + Object fieldValuePrimaryObjectDate = primary.getFieldValue(field); + Object fieldValueSecondaryObjectDate = secondary.getFieldValue(field); + + if (fieldValuePrimaryObjectDate != null) { + consolidatedEntity.setFieldValue(field, + new Date(((Date) fieldValuePrimaryObjectDate).getTime())); + } else if (fieldValueSecondaryObjectDate != null) { + consolidatedEntity.setFieldValue(field, + new Date(((Date) fieldValueSecondaryObjectDate).getTime())); + } + } + + void mergePrimitiveField(Field field, Entity primary, Entity secondary, Entity consolidatedEntity) + throws IllegalAccessException { + Object fieldValuePrimaryObjectPrimitiveOrString = primary.getFieldValue(field); + Object fieldValueSecondaryObjectPrimitiveOrString = secondary.getFieldValue(field); + + if (fieldValuePrimaryObjectPrimitiveOrString != null) { + consolidatedEntity.setFieldValue(field, fieldValuePrimaryObjectPrimitiveOrString); + } else if (fieldValueSecondaryObjectPrimitiveOrString != null) { + consolidatedEntity.setFieldValue(field, fieldValueSecondaryObjectPrimitiveOrString); + } + } + + + private void mergeCustomObjects(Entity primary, Entity secondary, Field field, + Entity consolidatedEntity) throws IllegalAccessException, EntityUpdateException { + Object primaryObj = primary.getFieldValue(field); + Object secondaryObj = secondary.getFieldValue(field); + if (primaryObj != null) { + consolidatedEntity.setFieldValue(field, deepCopyOfObject(primaryObj)); + } else if (secondaryObj != null) { + consolidatedEntity.setFieldValue(field, deepCopyOfObject(secondaryObj)); + } + } + + + /** + * Merges the Primary and secondary list without duplicates + * + * @param fieldValuePrimaryObject + * @param key + * @param elemSecondary + * @param fieldName + * @param prefLabelsForAltLabels + * @throws EntityUpdateException + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + protected void mergePrimarySecondaryListWitoutDuplicates( + Map fieldValuePrimaryObject, Object key, Map.Entry elemSecondary, + String fieldName, Map prefLabelsForAltLabels) throws EntityUpdateException { + if (fieldValuePrimaryObject.containsKey(key) + && List.class.isAssignableFrom(elemSecondary.getValue().getClass())) { + List listSecondaryObject = (List) elemSecondary.getValue(); + List listPrimaryObject = + deepCopyOfList((List) fieldValuePrimaryObject.get(key)); + boolean listPrimaryObjectChanged = false; + for (Object elemSecondaryList : listSecondaryObject) { + // check if value already exists in the primary list. + if (!EMCollectionUtils.ifValueAlreadyExistsInList(listPrimaryObject, elemSecondaryList, + doSloppyMatch(fieldName))) { + listPrimaryObject.add(elemSecondaryList); + if (listPrimaryObjectChanged == false) { + listPrimaryObjectChanged = true; + } + } + } + + if (listPrimaryObjectChanged) { + fieldValuePrimaryObject.put(key, listPrimaryObject); + } + } + // keep the different preferred labels in the secondary object for the + // alternative label in the consolidated object + else if (fieldValuePrimaryObject.containsKey(key) && fieldName.toLowerCase().contains("pref") + && fieldName.toLowerCase().contains("label")) { + Object primaryObjectPrefLabel = fieldValuePrimaryObject.get(key); + if (!primaryObjectPrefLabel.equals(elemSecondary.getValue())) { + prefLabelsForAltLabels.put(key, elemSecondary.getValue()); + } + } else if (!fieldValuePrimaryObject.containsKey(key)) { + fieldValuePrimaryObject.put(key, elemSecondary.getValue()); + } + } + + + @SuppressWarnings("unchecked") + protected boolean addValuesToAltLabel(Map prefLabelsForAltLabels, + Map altLabelPrimaryObject, boolean altLabelPrimaryValueChanged) + throws EntityUpdateException { + for (Map.Entry prefLabel : prefLabelsForAltLabels.entrySet()) { + String keyPrefLabel = (String) prefLabel.getKey(); + List altLabelPrimaryObjectList = + (List) altLabelPrimaryObject.get(keyPrefLabel); + List altLabelPrimaryValue = deepCopyOfList(altLabelPrimaryObjectList); + if (shouldValuesBeAddedToAltLabel(altLabelPrimaryValue, prefLabel)) { + altLabelPrimaryValue.add(prefLabel.getValue()); + if (altLabelPrimaryValueChanged == false) { + altLabelPrimaryValueChanged = true; + } + altLabelPrimaryObject.put(keyPrefLabel, altLabelPrimaryValue); + } + } + return altLabelPrimaryValueChanged; + } + + + protected boolean isFieldAltLabel(String fieldName) { + return fieldName.toLowerCase().contains("alt") && fieldName.toLowerCase().contains("label"); + } + + + private boolean shouldValuesBeAddedToAltLabel(List altLabelPrimaryValue, + Map.Entry prefLabel) { + return altLabelPrimaryValue.isEmpty() || (!altLabelPrimaryValue.isEmpty() && !EMCollectionUtils + .ifValueAlreadyExistsInList(altLabelPrimaryValue, prefLabel.getValue(), true)); + } + + + void mergeList(Entity consolidatedEntity, List fieldValuePrimaryObjectList, + List fieldValueSecondaryObjectList, Field field, boolean accumulate) + throws IllegalAccessException, EntityUpdateException { + List fieldValuePrimaryObject = deepCopyOfList(fieldValuePrimaryObjectList); + List fieldValueSecondaryObject = deepCopyOfList(fieldValueSecondaryObjectList); + + if (!CollectionUtils.isEmpty(fieldValuePrimaryObject) + && !CollectionUtils.isEmpty(fieldValueSecondaryObject) && accumulate) { + for (Object secondaryObjectListObject : fieldValueSecondaryObject) { + addToPrimaryList(field, fieldValuePrimaryObject, secondaryObjectListObject); + } + consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); + } else if (!CollectionUtils.isEmpty(fieldValuePrimaryObject)) { + consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); + } else if (!CollectionUtils.isEmpty(fieldValueSecondaryObject)) { + consolidatedEntity.setFieldValue(field, fieldValueSecondaryObject); + } + } + + + /** + * Add the secondary value in the primary list (if not already present) + * + * @param field + * @param fieldValuePrimaryObject + * @param secondaryObjectListObject + */ + private void addToPrimaryList(Field field, List fieldValuePrimaryObject, + Object secondaryObjectListObject) { + // check if the secondary value already exists in primary List + if (!EMCollectionUtils.ifValueAlreadyExistsInList(fieldValuePrimaryObject, + secondaryObjectListObject, doSloppyMatch(field.getName()))) { + fieldValuePrimaryObject.add(secondaryObjectListObject); + } + } + + + Object[] mergeArrays(Entity primary, Entity secondary, Field field, boolean append) + throws IllegalAccessException, EntityUpdateException { + Object[] primaryArray = (Object[]) primary.getFieldValue(field); + Object[] secondaryArray = (Object[]) secondary.getFieldValue(field); + + Object[] deepCopyPrimaryArray = deepCopyOfArray(primaryArray); + Object[] deepCopySecondaryArray = deepCopyOfArray(secondaryArray); + + if (deepCopyPrimaryArray.length == 0 && deepCopySecondaryArray.length == 0) { + return deepCopyPrimaryArray; + } else if (deepCopyPrimaryArray.length == 0) { + return deepCopySecondaryArray; + } else if (secondaryArray.length == 0 || !append) { + return deepCopyPrimaryArray; + } + // merge arrays + Set mergedAndOrdered = new TreeSet<>(Arrays.asList(deepCopyPrimaryArray)); + for (Object second : deepCopySecondaryArray) { + if (!EMCollectionUtils.ifValueAlreadyExistsInList(Arrays.asList(deepCopyPrimaryArray), second, + doSloppyMatch(field.getName()))) { + mergedAndOrdered.add(second); + } + } + return mergedAndOrdered.toArray(Arrays.copyOf(deepCopyPrimaryArray, 0)); + } + + + /** + * Deep copy of an object. + * + * @param obj + * @param isReference if the object is a reference to another object (in which case we keep the + * reference without deep copying) + * @return + * @throws EntityUpdateException + */ + private Object deepCopyOfObject(Object obj) throws EntityUpdateException { + if (obj == null || isStringOrPrimitive(obj.getClass())) { + return obj; + } + + try { + return obj.getClass().getConstructor(obj.getClass()).newInstance(obj); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException | NoSuchMethodException | SecurityException e) { + throw new EntityUpdateException( + "Metadata consolidation failed due to illegal creation of the object copy by calling newInstance.", + e); + } + + } + + + private Object[] deepCopyOfArray(Object[] input) throws EntityUpdateException { + if (input == null || input.length == 0) { + return new Object[0]; + } + Object[] copy; + if (isStringOrPrimitive(input[0].getClass())) { + copy = input.clone(); + } else { + copy = new Object[input.length]; + for (int i = 0; i < input.length; i++) { + try { + copy[i] = + input[i].getClass().getDeclaredConstructor(input[i].getClass()).newInstance(input[i]); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException | NoSuchMethodException | SecurityException e) { + throw new EntityUpdateException( + "Metadata consolidation failed due to illegal creation of the object " + + "copy within an array by calling newInstance.", + e); + } + } + } + return copy; + } + + + private List deepCopyOfList(List input) throws EntityUpdateException { + if (input == null || input.isEmpty()) { + return new ArrayList<>(); + } + + List copy; + if (isStringOrPrimitive(input.get(0).getClass())) { + copy = new ArrayList(input); + } else { + copy = new ArrayList<>(input.size()); + for (Object obj : input) { + try { + copy.add(obj.getClass().getDeclaredConstructor(obj.getClass()).newInstance(obj)); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException | NoSuchMethodException | SecurityException e) { + throw new EntityUpdateException( + "Metadata consolidation failed due to illegal creation of the object " + + "copy within a list by calling newInstance.", + e); + } + } + } + return copy; + } + + + @SuppressWarnings("unchecked") + protected Map deepCopyOfMap(Map input) + throws EntityUpdateException { + if (input == null || input.isEmpty()) { + return new HashMap<>(); + } + + Map copy; + Object mapFirstKey = input.entrySet().iterator().next().getKey(); + Object mapFirstValue = input.entrySet().iterator().next().getValue(); + // if both keys and values are of primitive type, no need for deep copy + if (isStringOrPrimitive(mapFirstKey.getClass()) + && isStringOrPrimitive(mapFirstValue.getClass())) { + copy = new HashMap<>(input); + } else { + copy = new HashMap<>(input.size()); + for (Map.Entry entry : input.entrySet()) { + Object keyDeepCopy = null; + Object valueDeepCopy = null; + if (List.class.isAssignableFrom(mapFirstKey.getClass())) { + keyDeepCopy = deepCopyOfList((List) entry.getKey()); + } else { + keyDeepCopy = deepCopyOfObject(entry.getKey()); + } + + if (List.class.isAssignableFrom(mapFirstValue.getClass())) { + valueDeepCopy = deepCopyOfList((List) entry.getValue()); + } else { + valueDeepCopy = deepCopyOfObject(entry.getValue()); + } + copy.put(keyDeepCopy, valueDeepCopy); + } + } + + return copy; + + } + + + protected Aggregation createNewAggregation(String entityId, Date timestamp) { + Aggregation isAggregatedBy = new Aggregation(); + isAggregatedBy.setId(getIsAggregatedById(entityId)); + isAggregatedBy.setCreated(timestamp); + isAggregatedBy.setModified(timestamp); + return isAggregatedBy; + } + + + protected void setEuropeanaMetadata(Entity europeanaProxyMetadata, String entityId, + List corefs, EntityRecord entityRecord, Date timestamp) { + Aggregation europeanaAggr = new Aggregation(); + Optional europeanaDataSource = datasources.getEuropeanaDatasource(); + + + europeanaAggr.setId(getEuropeanaAggregationId(entityId)); + // europeana datasource is checked on startup, so it cannot be empty here + if (europeanaDataSource.isPresent()) { + europeanaAggr.setRights(europeanaDataSource.get().getRights()); + europeanaAggr.setSource(europeanaDataSource.get().getUrl()); + } + europeanaAggr.setCreated(timestamp); + europeanaAggr.setModified(timestamp); + + EntityProxy europeanaProxy = new EntityProxy(); + europeanaProxy.setProxyId(getEuropeanaProxyId(entityId)); + europeanaProxy.setProxyFor(entityId); + europeanaProxy.setProxyIn(europeanaAggr); + // update co-references + addSameReferenceLinks(europeanaProxyMetadata, corefs); + europeanaProxy.setEntity(europeanaProxyMetadata); + + entityRecord.addProxy(europeanaProxy); + } + + /** + * Adds the specified uris to the entity's exactMatch / sameAs + * + * @param entity entity to update + * @param uris uris to add to entity's sameAs / exactMatch + */ + public void addSameReferenceLinks(Entity entity, List uris) { + List entitySameReferenceLinks = entity.getSameReferenceLinks(); + + if (entitySameReferenceLinks == null) { + // sameAs is mutable here as we might need to add more values to it later + entity.setSameReferenceLinks(new ArrayList<>(uris)); + return; + } + + // combine uris with existing sameReferenceLinks, minus duplicates + entity.setSameReferenceLinks(Stream.concat(entitySameReferenceLinks.stream(), uris.stream()) + .distinct().collect(Collectors.toList())); + } + + protected EntityProxy setExternalProxy(Entity metisResponse, String proxyId, String entityId, + DataSource externalDatasource, EntityRecord entityRecord, Date timestamp, int aggregationId) { + Aggregation datasourceAggr = new Aggregation(); + datasourceAggr.setId(getDatasourceAggregationId(entityId, aggregationId)); + datasourceAggr.setCreated(timestamp); + datasourceAggr.setModified(timestamp); + datasourceAggr.setRights(externalDatasource.getRights()); + datasourceAggr.setSource(externalDatasource.getUrl()); + + EntityProxy datasourceProxy = new EntityProxy(); + datasourceProxy.setProxyId(proxyId); + datasourceProxy.setProxyFor(entityId); + datasourceProxy.setProxyIn(datasourceAggr); + datasourceProxy.setEntity(metisResponse); + + entityRecord.addProxy(datasourceProxy); + return datasourceProxy; + } + + + protected void processRoleReference(Organization org) { + if (org.getEuropeanaRoleIds() != null && !org.getEuropeanaRoleIds().isEmpty()) { + List vocabs = vocabRepository.findByUri(org.getEuropeanaRoleIds()); + if (vocabs.isEmpty()) { + if (logger.isWarnEnabled()) { + logger.warn( + "No vocabularies with the uris: {} were found in the database. Cannot assign role reference to organization with id {}", + org.getEuropeanaRoleIds(), org.getEntityId()); + } + } else { + org.setEuropeanaRoleRefs(vocabs); + } + } + } + + + protected void processCountryReference(Organization org) { + // country reference + if (StringUtils.isEmpty(org.getCountryId())) { + return; + } + String europeanaCountryId = getEuropeanaCountryId(org); + if (europeanaCountryId == null) { + if (logger.isWarnEnabled()) { + logger.warn("Dropping unsupported country id in consolidated entity version: {} -- {} ", + org.getEntityId(), org.getCountryId()); + } + org.setCountryId(null); + } else { + // replace wikidata country ids + org.setCountryId(europeanaCountryId); + // search reference + EntityRecord orgCountry = entityRecordRepository.findByEntityId(europeanaCountryId); + if (orgCountry != null) { + org.setCountryRef(orgCountry); + } else if (logger.isWarnEnabled()) { + logger.warn( + "No country found in database for the entity id: {}. Cannot assign country reference to organization with id {}", + europeanaCountryId, org.getEntityId()); + } + } + } + + + String getEuropeanaCountryId(Organization org) { + if (EntityRecordUtils.isEuropeanaEntity(org.getCountryId())) { + // country id is already europeana entity + return org.getCountryId(); + } + // drop all country ids except for the Europeana entities + return null; + } + + boolean isStringOrPrimitive(Class fieldType) { + return String.class.isAssignableFrom(fieldType) || ClassUtils.isPrimitiveOrWrapper(fieldType); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + void combineEntities(Entity consolidatedEntity, Entity primary, Entity secondary, + Map prefLabelsForAltLabels, Field field, String fieldName, boolean accumulate) + throws IllegalAccessException, EntityUpdateException { + // TODO: refactor implemetation + + Map fieldValuePrimaryObjectMap = + (Map) primary.getFieldValue(field); + Map fieldValueSecondaryObjectMap = + (Map) secondary.getFieldValue(field); + Map fieldValuePrimaryObject = deepCopyOfMap(fieldValuePrimaryObjectMap); + Map fieldValueSecondaryObject = deepCopyOfMap(fieldValueSecondaryObjectMap); + + if (!CollectionUtils.isEmpty(fieldValuePrimaryObject) + && !CollectionUtils.isEmpty(fieldValueSecondaryObject) && accumulate) { + for (Map.Entry elemSecondary : fieldValueSecondaryObject.entrySet()) { + Object key = elemSecondary.getKey(); + /* + * if the map value is a list, merge the lists of the primary and the secondary object + * without duplicates + */ + mergePrimarySecondaryListWitoutDuplicates(fieldValuePrimaryObject, key, elemSecondary, + fieldName, prefLabelsForAltLabels); + } + consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); + } else if (!CollectionUtils.isEmpty(fieldValuePrimaryObject)) { + consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); + } else if (!CollectionUtils.isEmpty(fieldValueSecondaryObject)) { + consolidatedEntity.setFieldValue(field, fieldValueSecondaryObject); + } + } + + @SuppressWarnings("unchecked") + void mergeSkippedPrefLabels(Entity consolidatedEntity, Map prefLabelsForAltLabels, + List allEntityFields) throws IllegalAccessException, EntityUpdateException { + /* + * adding the preferred labels from the secondary object to the alternative labels of + * consolidated object + */ + if (prefLabelsForAltLabels.isEmpty()) { + // nothing to merge + return; + } + + // get the alt label field + Optional altLabelField = + allEntityFields.stream().filter(field -> isFieldAltLabel(field.getName())).findFirst(); + + if (altLabelField.isEmpty()) { + // altLabel field not found + if (logger.isWarnEnabled()) { + logger.warn("altLabel field not found in list: {}", allEntityFields); + } + + // skip + return; + } + + Map altLabelConsolidatedMap = + (Map) consolidatedEntity.getFieldValue(altLabelField.get()); + Map altLabelPrimaryObject = deepCopyOfMap(altLabelConsolidatedMap); + boolean altLabelPrimaryValueChanged = false; + altLabelPrimaryValueChanged = addValuesToAltLabel(prefLabelsForAltLabels, altLabelPrimaryObject, + altLabelPrimaryValueChanged); + if (altLabelPrimaryValueChanged) { + consolidatedEntity.setFieldValue(altLabelField.get(), altLabelPrimaryObject); + } + } + + static boolean doSloppyMatch(String fieldName) { + if (EntityFieldsTypes.hasTypeDefinition(fieldName)) { + String type = EntityFieldsTypes.getFieldType(fieldName); + // for text do a sloppy match + if (StringUtils.equals(type, EntityFieldsTypes.FIELD_TYPE_TEXT)) { + return true; + } + // for uri or keywords do an exact match + else if (StringUtils.equals(type, EntityFieldsTypes.FIELD_TYPE_URI) + || StringUtils.equals(type, EntityFieldsTypes.FIELD_TYPE_KEYWORD)) { + return false; + } + } + // for all other cases + return false; + } +} diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index e40b49a1..c2758bd9 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -3,31 +3,20 @@ import static eu.europeana.entitymanagement.solr.SolrUtils.createSolrEntity; import static eu.europeana.entitymanagement.utils.EntityRecordUtils.getDatasourceAggregationId; import static eu.europeana.entitymanagement.utils.EntityRecordUtils.getEuropeanaAggregationId; -import static eu.europeana.entitymanagement.utils.EntityRecordUtils.getEuropeanaProxyId; -import static eu.europeana.entitymanagement.utils.EntityRecordUtils.getIsAggregatedById; import static java.time.Instant.now; import java.io.IOException; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; import com.mongodb.client.result.UpdateResult; import dev.morphia.query.experimental.filters.Filter; import eu.europeana.api.commons.error.EuropeanaApiException; @@ -47,7 +36,6 @@ import eu.europeana.entitymanagement.definitions.model.Organization; import eu.europeana.entitymanagement.definitions.model.Place; import eu.europeana.entitymanagement.definitions.model.TimeSpan; -import eu.europeana.entitymanagement.definitions.model.Vocabulary; import eu.europeana.entitymanagement.definitions.web.EntityIdDisabledStatus; import eu.europeana.entitymanagement.exception.EntityAlreadyExistsException; import eu.europeana.entitymanagement.exception.EntityCreationException; @@ -61,12 +49,9 @@ import eu.europeana.entitymanagement.mongo.repository.VocabularyRepository; import eu.europeana.entitymanagement.solr.exception.SolrServiceException; import eu.europeana.entitymanagement.solr.service.SolrService; -import eu.europeana.entitymanagement.utils.EMCollectionUtils; import eu.europeana.entitymanagement.utils.EntityObjectFactory; import eu.europeana.entitymanagement.utils.EntityRecordUtils; import eu.europeana.entitymanagement.utils.EntityUtils; -import eu.europeana.entitymanagement.utils.UriValidator; -import eu.europeana.entitymanagement.vocabulary.EntityFieldsTypes; import eu.europeana.entitymanagement.vocabulary.EntityProfile; import eu.europeana.entitymanagement.vocabulary.EntityTypes; import eu.europeana.entitymanagement.vocabulary.WebEntityConstants; @@ -78,35 +63,15 @@ import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; @Service(AppConfigConstants.BEAN_ENTITY_RECORD_SERVICE) -public class EntityRecordService { - - private final EntityRecordRepository entityRecordRepository; - - private final VocabularyRepository vocabRepository; - - final EntityManagementConfiguration emConfiguration; - - private final DataSources datasources; - - private final SolrService solrService; - - private final ZohoConfiguration zohoConfiguration; - - private static final Logger logger = LogManager.getLogger(EntityRecordService.class); - - // Fields to be ignored during consolidation ("type" is final) - private static final Set ignoredMergeFields = Set.of(WebEntityFields.TYPE); +public class EntityRecordService extends BaseEntityRecordService { @Autowired - public EntityRecordService(EntityRecordRepository entityRecordRepository, VocabularyRepository vocabRepository, - EntityManagementConfiguration emConfiguration, ZohoConfiguration zohoConfiguration, - DataSources datasources, SolrService solrService) throws IOException { - this.entityRecordRepository = entityRecordRepository; - this.vocabRepository=vocabRepository; - this.emConfiguration = emConfiguration; - this.zohoConfiguration = zohoConfiguration; - this.datasources = datasources; - this.solrService = solrService; + public EntityRecordService(EntityRecordRepository entityRecordRepository, + VocabularyRepository vocabRepository, EntityManagementConfiguration emConfiguration, + ZohoConfiguration zohoConfiguration, DataSources datasources, SolrService solrService){ + + super(entityRecordRepository, vocabRepository, emConfiguration, zohoConfiguration, datasources, + solrService); } public boolean existsByEntityId(String entityId) { @@ -158,14 +123,14 @@ public EntityRecord retrieveEntityRecord(String entityUri, String profiles, } void dereferenceLinkedEntities(EntityRecord entityRecord) { - //dereference links for organizations - if(EntityTypes.isOrganization(entityRecord.getEntity().getType())) { + // dereference links for organizations + if (EntityTypes.isOrganization(entityRecord.getEntity().getType())) { dereferenceLinkedEntities((Organization) entityRecord.getEntity()); } } private void dereferenceLinkedEntities(Organization org) { - //dereference country + // dereference country if (org.getCountryId() != null) { EntityRecord countryRecord = entityRecordRepository.findByEntityId(org.getCountryId(), true); if (countryRecord != null) { @@ -174,9 +139,9 @@ private void dereferenceLinkedEntities(Organization org) { org.setCountry(country); } } - //dereference role - if(org.getEuropeanaRoleIds()!=null && !org.getEuropeanaRoleIds().isEmpty()) { - org.setEuropeanaRole(vocabRepository.findByUri(org.getEuropeanaRoleIds())); + // dereference role + if (org.getEuropeanaRoleIds() != null && !org.getEuropeanaRoleIds().isEmpty()) { + org.setEuropeanaRole(vocabRepository.findByUri(org.getEuropeanaRoleIds())); } } @@ -253,23 +218,6 @@ public List retrieveEntityDeprecationStatus(List return entityRecordRepository.getEntityIds(entityIds, excludeDisabled); } - /** - * Gets that have at list one of the provided URIs present in co-references (sameAs or exactMatch - * value in the Consolidated version). The current entity identified by the entityId, if not null - * or empty is excluded. Exclude disabled indicates if the disabled entities should be included in - * the response or not - * - * @param uris co-reference uris - * @param entityId URI indicating the the record for the given entityId should not be retrieved as - * matchingCoreference - * @param excludeDisabled indicated if the disabled entities should be filtered out or not - * @return List of dupplicated entities or empty. - */ - public List findEntitiesByCoreference(List uris, String entityId, - boolean excludeDisabled) { - return entityRecordRepository.findEntitiesByCoreference(uris, entityId, excludeDisabled); - } - public EntityRecord saveEntityRecord(EntityRecord er) { return entityRecordRepository.save(er); } @@ -820,23 +768,6 @@ private List replaceWithInternalReferences(List originalReferenc return updatedReferences; } - private void addValueOrInternalReference(List updatedReferences, String value) { - if (value.startsWith(WebEntityFields.BASE_DATA_EUROPEANA_URI) || !UriValidator.isUri(value)) { - // value is internal reference or string literal - updatedReferences.add(value); - } else { - // value is external URI, replace it with internal reference if they are accessible - // do not use disabled entities as they are not accessible anymore - List records = - findEntitiesByCoreference(Collections.singletonList(value), null, true); - if (!records.isEmpty()) { - // if the prevention of dupplication worked propertly, that we should find only one active - // entry in the database - updatedReferences.add(records.get(0).getEntityId()); - } - } - } - /** * This function merges the metadata data from the provided entities and returns the consolidated * version @@ -884,411 +815,7 @@ public void replaceEuropeanaProxy(final Entity updateRequestEntity, EntityRecord europeanaProxy.getProxyIn().setModified(Date.from(now())); } - /** - * Merges metadata between two entities. This method performs a deep copy of the objects, for the - * mutable (custom) field types. - * - * @param primary Primary entity. Metadata from this entity takes precedence - * @param secondary Secondary entity. Metadata from this entity is only used if no matching field - * is contained within the primary entity. - * @param fieldsToCombine metadata fields to reconcile - * @param accumulate if true, metadata from the secondary entity are added to the matching - * collection (eg. maps, lists and arrays) within the primary . If accumulate is false, the - * "primary" content overwrites the "secondary" - */ - @SuppressWarnings("unchecked") - private Entity combineEntities(Entity primary, Entity secondary, List fieldsToCombine, - boolean accumulate) throws EuropeanaApiException, EntityModelCreationException { - Entity consolidatedEntity = - EntityObjectFactory.createConsolidatedEntityObject(primary.getType()); - - try { - - /* - * store the preferred label in the secondary entity that is different from the preferred - * label in the primary entity to the alternative labels of the consolidated entity - */ - Map prefLabelsForAltLabels = new HashMap<>(); - - for (Field field : fieldsToCombine) { - Class fieldType = field.getType(); - String fieldName = field.getName(); - - if (isStringOrPrimitive(fieldType)) { - Object fieldValuePrimaryObjectPrimitiveOrString = primary.getFieldValue(field); - Object fieldValueSecondaryObjectPrimitiveOrString = secondary.getFieldValue(field); - - if (fieldValuePrimaryObjectPrimitiveOrString != null) { - consolidatedEntity.setFieldValue(field, fieldValuePrimaryObjectPrimitiveOrString); - } else if (fieldValueSecondaryObjectPrimitiveOrString != null) { - consolidatedEntity.setFieldValue(field, fieldValueSecondaryObjectPrimitiveOrString); - } - - } else if (Date.class.isAssignableFrom(fieldType)) { - Object fieldValuePrimaryObjectDate = primary.getFieldValue(field); - Object fieldValueSecondaryObjectDate = secondary.getFieldValue(field); - - if (fieldValuePrimaryObjectDate != null) { - consolidatedEntity.setFieldValue(field, - new Date(((Date) fieldValuePrimaryObjectDate).getTime())); - } else if (fieldValueSecondaryObjectDate != null) { - consolidatedEntity.setFieldValue(field, - new Date(((Date) fieldValueSecondaryObjectDate).getTime())); - } - } else if (fieldType.isArray()) { - Object[] mergedArray = mergeArrays(primary, secondary, field, accumulate); - consolidatedEntity.setFieldValue(field, mergedArray); - } else if (List.class.isAssignableFrom(fieldType)) { - List fieldValuePrimaryObjectList = (List) primary.getFieldValue(field); - List fieldValueSecondaryObjectList = - (List) secondary.getFieldValue(field); - mergeList(consolidatedEntity, fieldValuePrimaryObjectList, fieldValueSecondaryObjectList, - field, accumulate); - } else if (Map.class.isAssignableFrom(fieldType)) { - combineEntities(consolidatedEntity, primary, secondary, prefLabelsForAltLabels, field, - fieldName, accumulate); - } else { - mergeCustomObjects(primary, secondary, field, consolidatedEntity); - } - } - - mergeSkippedPrefLabels(consolidatedEntity, prefLabelsForAltLabels, fieldsToCombine); - - } catch (IllegalAccessException e) { - throw new EntityUpdateException( - "Metadata consolidation failed to access required properties!", e); - } - - return consolidatedEntity; - } - - private void mergeCustomObjects(Entity primary, Entity secondary, Field field, - Entity consolidatedEntity) throws IllegalAccessException, EntityUpdateException { - Object primaryObj = primary.getFieldValue(field); - Object secondaryObj = secondary.getFieldValue(field); - if (primaryObj != null) { - consolidatedEntity.setFieldValue(field, deepCopyOfObject(primaryObj)); - } else if (secondaryObj != null) { - consolidatedEntity.setFieldValue(field, deepCopyOfObject(secondaryObj)); - } - } - - boolean isStringOrPrimitive(Class fieldType) { - return String.class.isAssignableFrom(fieldType) || fieldType.isPrimitive() - || Float.class.isAssignableFrom(fieldType) || Integer.class.isAssignableFrom(fieldType) - || Double.class.isAssignableFrom(fieldType) || Short.class.isAssignableFrom(fieldType) - || Byte.class.isAssignableFrom(fieldType) || Boolean.class.isAssignableFrom(fieldType) - || Long.class.isAssignableFrom(fieldType); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - void combineEntities(Entity consolidatedEntity, Entity primary, Entity secondary, - Map prefLabelsForAltLabels, Field field, String fieldName, boolean accumulate) - throws IllegalAccessException, EntityUpdateException { - // TODO: refactor implemetation - - Map fieldValuePrimaryObjectMap = - (Map) primary.getFieldValue(field); - Map fieldValueSecondaryObjectMap = - (Map) secondary.getFieldValue(field); - Map fieldValuePrimaryObject = deepCopyOfMap(fieldValuePrimaryObjectMap); - Map fieldValueSecondaryObject = deepCopyOfMap(fieldValueSecondaryObjectMap); - - if (!CollectionUtils.isEmpty(fieldValuePrimaryObject) - && !CollectionUtils.isEmpty(fieldValueSecondaryObject) && accumulate) { - for (Map.Entry elemSecondary : fieldValueSecondaryObject.entrySet()) { - Object key = elemSecondary.getKey(); - /* - * if the map value is a list, merge the lists of the primary and the secondary object - * without duplicates - */ - mergePrimarySecondaryListWitoutDuplicates(fieldValuePrimaryObject, key, elemSecondary, - fieldName, prefLabelsForAltLabels); - } - consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); - } else if (!CollectionUtils.isEmpty(fieldValuePrimaryObject)) { - consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); - } else if (!CollectionUtils.isEmpty(fieldValueSecondaryObject)) { - consolidatedEntity.setFieldValue(field, fieldValueSecondaryObject); - } - } - - /** - * Merges the Primary and secondary list without duplicates - * - * @param fieldValuePrimaryObject - * @param key - * @param elemSecondary - * @param fieldName - * @param prefLabelsForAltLabels - * @throws EntityUpdateException - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - private void mergePrimarySecondaryListWitoutDuplicates( - Map fieldValuePrimaryObject, Object key, Map.Entry elemSecondary, - String fieldName, Map prefLabelsForAltLabels) throws EntityUpdateException { - if (fieldValuePrimaryObject.containsKey(key) - && List.class.isAssignableFrom(elemSecondary.getValue().getClass())) { - List listSecondaryObject = (List) elemSecondary.getValue(); - List listPrimaryObject = - deepCopyOfList((List) fieldValuePrimaryObject.get(key)); - boolean listPrimaryObjectChanged = false; - for (Object elemSecondaryList : listSecondaryObject) { - // check if value already exists in the primary list. - if (!EMCollectionUtils.ifValueAlreadyExistsInList(listPrimaryObject, elemSecondaryList, - doSloppyMatch(fieldName))) { - listPrimaryObject.add(elemSecondaryList); - if (listPrimaryObjectChanged == false) { - listPrimaryObjectChanged = true; - } - } - } - - if (listPrimaryObjectChanged) { - fieldValuePrimaryObject.put(key, listPrimaryObject); - } - } - // keep the different preferred labels in the secondary object for the - // alternative label in the consolidated object - else if (fieldValuePrimaryObject.containsKey(key) && fieldName.toLowerCase().contains("pref") - && fieldName.toLowerCase().contains("label")) { - Object primaryObjectPrefLabel = fieldValuePrimaryObject.get(key); - if (!primaryObjectPrefLabel.equals(elemSecondary.getValue())) { - prefLabelsForAltLabels.put(key, elemSecondary.getValue()); - } - } else if (!fieldValuePrimaryObject.containsKey(key)) { - fieldValuePrimaryObject.put(key, elemSecondary.getValue()); - } - } - - @SuppressWarnings("unchecked") - void mergeSkippedPrefLabels(Entity consilidatedEntity, Map prefLabelsForAltLabels, - List allEntityFields) throws IllegalAccessException, EntityUpdateException { - /* - * adding the preferred labels from the secondary object to the alternative labels of - * consolidated object - */ - if (prefLabelsForAltLabels.size() > 0) { - for (Field field : allEntityFields) { - String fieldName = field.getName(); - if (isFieldAltLabel(fieldName)) { - Map altLabelConsolidatedMap = - (Map) consilidatedEntity.getFieldValue(field); - Map altLabelPrimaryObject = deepCopyOfMap(altLabelConsolidatedMap); - boolean altLabelPrimaryValueChanged = false; - altLabelPrimaryValueChanged = addValuesToAltLabel(prefLabelsForAltLabels, - altLabelPrimaryObject, altLabelPrimaryValueChanged); - if (altLabelPrimaryValueChanged) { - consilidatedEntity.setFieldValue(field, altLabelPrimaryObject); - } - break; - } - } - } - } - - @SuppressWarnings("unchecked") - private boolean addValuesToAltLabel(Map prefLabelsForAltLabels, - Map altLabelPrimaryObject, boolean altLabelPrimaryValueChanged) - throws EntityUpdateException { - for (Map.Entry prefLabel : prefLabelsForAltLabels.entrySet()) { - String keyPrefLabel = (String) prefLabel.getKey(); - List altLabelPrimaryObjectList = - (List) altLabelPrimaryObject.get(keyPrefLabel); - List altLabelPrimaryValue = deepCopyOfList(altLabelPrimaryObjectList); - if (shouldValuesBeAddedToAltLabel(altLabelPrimaryValue, prefLabel)) { - altLabelPrimaryValue.add(prefLabel.getValue()); - if (altLabelPrimaryValueChanged == false) { - altLabelPrimaryValueChanged = true; - } - altLabelPrimaryObject.put(keyPrefLabel, altLabelPrimaryValue); - } - } - return altLabelPrimaryValueChanged; - } - - private boolean isFieldAltLabel(String fieldName) { - return fieldName.toLowerCase().contains("alt") && fieldName.toLowerCase().contains("label"); - } - - private boolean shouldValuesBeAddedToAltLabel(List altLabelPrimaryValue, - Map.Entry prefLabel) { - return altLabelPrimaryValue.isEmpty() || (!altLabelPrimaryValue.isEmpty() && !EMCollectionUtils - .ifValueAlreadyExistsInList(altLabelPrimaryValue, prefLabel.getValue(), true)); - } - - void mergeList(Entity consolidatedEntity, List fieldValuePrimaryObjectList, - List fieldValueSecondaryObjectList, Field field, boolean accumulate) - throws IllegalAccessException, EntityUpdateException { - List fieldValuePrimaryObject = deepCopyOfList(fieldValuePrimaryObjectList); - List fieldValueSecondaryObject = deepCopyOfList(fieldValueSecondaryObjectList); - - if (!CollectionUtils.isEmpty(fieldValuePrimaryObject) - && !CollectionUtils.isEmpty(fieldValueSecondaryObject) && accumulate) { - for (Object secondaryObjectListObject : fieldValueSecondaryObject) { - addToPrimaryList(field, fieldValuePrimaryObject, secondaryObjectListObject); - } - consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); - } else if (!CollectionUtils.isEmpty(fieldValuePrimaryObject)) { - consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); - } else if (!CollectionUtils.isEmpty(fieldValueSecondaryObject)) { - consolidatedEntity.setFieldValue(field, fieldValueSecondaryObject); - } - } - - /** - * Add the secondary value in the primary list (if not already present) - * - * @param field - * @param fieldValuePrimaryObject - * @param secondaryObjectListObject - */ - private void addToPrimaryList(Field field, List fieldValuePrimaryObject, - Object secondaryObjectListObject) { - // check if the secondary value already exists in primary List - if (!EMCollectionUtils.ifValueAlreadyExistsInList(fieldValuePrimaryObject, - secondaryObjectListObject, doSloppyMatch(field.getName()))) { - fieldValuePrimaryObject.add(secondaryObjectListObject); - } - } - - Object[] mergeArrays(Entity primary, Entity secondary, Field field, boolean append) - throws IllegalAccessException, EntityUpdateException { - Object[] primaryArray = (Object[]) primary.getFieldValue(field); - Object[] secondaryArray = (Object[]) secondary.getFieldValue(field); - - Object[] deepCopyPrimaryArray = deepCopyOfArray(primaryArray); - Object[] deepCopySecondaryArray = deepCopyOfArray(secondaryArray); - - if (deepCopyPrimaryArray.length == 0 && deepCopySecondaryArray.length == 0) { - return deepCopyPrimaryArray; - } else if (deepCopyPrimaryArray.length == 0) { - return deepCopySecondaryArray; - } else if (secondaryArray.length == 0 || !append) { - return deepCopyPrimaryArray; - } - // merge arrays - Set mergedAndOrdered = new TreeSet<>(Arrays.asList(deepCopyPrimaryArray)); - for (Object second : deepCopySecondaryArray) { - if (!EMCollectionUtils.ifValueAlreadyExistsInList(Arrays.asList(deepCopyPrimaryArray), second, - doSloppyMatch(field.getName()))) { - mergedAndOrdered.add(second); - } - } - return mergedAndOrdered.toArray(Arrays.copyOf(deepCopyPrimaryArray, 0)); - } - - /** - * Deep copy of an object. - * - * @param obj - * @param isReference if the object is a reference to another object (in which case we keep the - * reference without deep copying) - * @return - * @throws EntityUpdateException - */ - private Object deepCopyOfObject(Object obj) throws EntityUpdateException { - if (obj == null || isStringOrPrimitive(obj.getClass())) { - return obj; - } - - try { - return obj.getClass().getConstructor(obj.getClass()).newInstance(obj); - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException - | InvocationTargetException | NoSuchMethodException | SecurityException e) { - throw new EntityUpdateException( - "Metadata consolidation failed due to illegal creation of the object copy by calling newInstance.", - e); - } - - } - - private Object[] deepCopyOfArray(Object[] input) throws EntityUpdateException { - if (input == null || input.length == 0) { - return new Object[0]; - } - Object[] copy; - if (isStringOrPrimitive(input[0].getClass())) { - copy = input.clone(); - } else { - copy = new Object[input.length]; - for (int i = 0; i < input.length; i++) { - try { - copy[i] = - input[i].getClass().getDeclaredConstructor(input[i].getClass()).newInstance(input[i]); - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException - | InvocationTargetException | NoSuchMethodException | SecurityException e) { - throw new EntityUpdateException( - "Metadata consolidation failed due to illegal creation of the object " - + "copy within an array by calling newInstance.", - e); - } - } - } - return copy; - } - - private List deepCopyOfList(List input) throws EntityUpdateException { - if (input == null || input.isEmpty()) { - return new ArrayList<>(); - } - - List copy; - if (isStringOrPrimitive(input.get(0).getClass())) { - copy = new ArrayList(input); - } else { - copy = new ArrayList<>(input.size()); - for (Object obj : input) { - try { - copy.add(obj.getClass().getDeclaredConstructor(obj.getClass()).newInstance(obj)); - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException - | InvocationTargetException | NoSuchMethodException | SecurityException e) { - throw new EntityUpdateException( - "Metadata consolidation failed due to illegal creation of the object " - + "copy within a list by calling newInstance.", - e); - } - } - } - return copy; - } - - private Map deepCopyOfMap(Map input) - throws EntityUpdateException { - if (input == null || input.isEmpty()) { - return new HashMap<>(); - } - - Map copy; - Object mapFirstKey = input.entrySet().iterator().next().getKey(); - Object mapFirstValue = input.entrySet().iterator().next().getValue(); - // if both keys and values are of primitive type, no need for deep copy - if (isStringOrPrimitive(mapFirstKey.getClass()) - && isStringOrPrimitive(mapFirstValue.getClass())) { - copy = new HashMap<>(input); - } else { - copy = new HashMap<>(input.size()); - for (Map.Entry entry : input.entrySet()) { - Object keyDeepCopy = null; - Object valueDeepCopy = null; - if (List.class.isAssignableFrom(mapFirstKey.getClass())) { - keyDeepCopy = deepCopyOfList((List) entry.getKey()); - } else { - keyDeepCopy = deepCopyOfObject(entry.getKey()); - } - - if (List.class.isAssignableFrom(mapFirstValue.getClass())) { - valueDeepCopy = deepCopyOfList((List) entry.getValue()); - } else { - valueDeepCopy = deepCopyOfObject(entry.getValue()); - } - copy.put(keyDeepCopy, valueDeepCopy); - } - } - - return copy; - - } + public void dropRepository() { this.entityRecordRepository.dropCollection(); @@ -1318,14 +845,6 @@ private void updateEntityAggregation(EntityRecord entityRecord, String entityId, updateEntityAggregatesList(aggregation, entityRecord, entityId); } - private Aggregation createNewAggregation(String entityId, Date timestamp) { - Aggregation isAggregatedBy = new Aggregation(); - isAggregatedBy.setId(getIsAggregatedById(entityId)); - isAggregatedBy.setCreated(timestamp); - isAggregatedBy.setModified(timestamp); - return isAggregatedBy; - } - private void updateEntityAggregatesList(Aggregation aggregation, EntityRecord entityRecord, String entityId) { // aggregates is mutable in case we need to append to it later @@ -1339,47 +858,6 @@ private void updateEntityAggregatesList(Aggregation aggregation, EntityRecord en aggregation.setAggregates(aggregates); } - private void setEuropeanaMetadata(Entity europeanaProxyMetadata, String entityId, - List corefs, EntityRecord entityRecord, Date timestamp) { - Aggregation europeanaAggr = new Aggregation(); - Optional europeanaDataSource = datasources.getEuropeanaDatasource(); - europeanaAggr.setId(getEuropeanaAggregationId(entityId)); - // europeana datasource is checked on startup, so it cannot be empty here - europeanaAggr.setRights(europeanaDataSource.get().getRights()); - europeanaAggr.setSource(europeanaDataSource.get().getUrl()); - europeanaAggr.setCreated(timestamp); - europeanaAggr.setModified(timestamp); - - EntityProxy europeanaProxy = new EntityProxy(); - europeanaProxy.setProxyId(getEuropeanaProxyId(entityId)); - europeanaProxy.setProxyFor(entityId); - europeanaProxy.setProxyIn(europeanaAggr); - // update co-references - addSameReferenceLinks(europeanaProxyMetadata, corefs); - europeanaProxy.setEntity(europeanaProxyMetadata); - - entityRecord.addProxy(europeanaProxy); - } - - private EntityProxy setExternalProxy(Entity metisResponse, String proxyId, String entityId, - DataSource externalDatasource, EntityRecord entityRecord, Date timestamp, int aggregationId) { - Aggregation datasourceAggr = new Aggregation(); - datasourceAggr.setId(getDatasourceAggregationId(entityId, aggregationId)); - datasourceAggr.setCreated(timestamp); - datasourceAggr.setModified(timestamp); - datasourceAggr.setRights(externalDatasource.getRights()); - datasourceAggr.setSource(externalDatasource.getUrl()); - - EntityProxy datasourceProxy = new EntityProxy(); - datasourceProxy.setProxyId(proxyId); - datasourceProxy.setProxyFor(entityId); - datasourceProxy.setProxyIn(datasourceAggr); - datasourceProxy.setEntity(metisResponse); - - entityRecord.addProxy(datasourceProxy); - return datasourceProxy; - } - /** * Recreates the external proxy on an Entity, using the newProxyId value as its proxyId * @@ -1408,26 +886,6 @@ public void changeExternalProxy(EntityRecord entityRecord, String newProxyId) entityRecord.getEntityId(), dataSource, entityRecord, new Date(), 1); } - /** - * Adds the specified uris to the entity's exactMatch / sameAs - * - * @param entity entity to update - * @param uris uris to add to entity's sameAs / exactMatch - */ - public void addSameReferenceLinks(Entity entity, List uris) { - List entitySameReferenceLinks = entity.getSameReferenceLinks(); - - if (entitySameReferenceLinks == null) { - // sameAs is mutable here as we might need to add more values to it later - entity.setSameReferenceLinks(new ArrayList<>(uris)); - return; - } - - // combine uris with existing sameReferenceLinks, minus duplicates - entity.setSameReferenceLinks(Stream.concat(entitySameReferenceLinks.stream(), uris.stream()) - .distinct().collect(Collectors.toList())); - } - public EntityRecord updateUsedForEnrichment(EntityTypes type, String identifier, String profile, String action) throws EuropeanaApiException { @@ -1445,28 +903,14 @@ public EntityRecord updateUsedForEnrichment(EntityTypes type, String identifier, return update(entityRecord); } - public static boolean doSloppyMatch(String fieldName) { - String type = EntityFieldsTypes.getFieldType(fieldName); - // for text do a sloppy match - if (StringUtils.equals(type, EntityFieldsTypes.FIELD_TYPE_TEXT)) { - return true; - } - // for uri or keywords do an exact match - else if (StringUtils.equals(type, EntityFieldsTypes.FIELD_TYPE_URI) - || StringUtils.equals(type, EntityFieldsTypes.FIELD_TYPE_KEYWORD)) { - return false; - } - // for all other cases - return false; - } - void updateEuropeanaIDFieldInZoho(String zohoOrganizationUrl, String europeanaId) throws EntityCreationException { try { zohoConfiguration.getZohoAccessClient().updateZohoRecordOrganizationStringField( zohoOrganizationUrl, ZohoConstants.EUROPEANA_ID_FIELD, europeanaId); - if(logger.isDebugEnabled()) { - logger.debug("Updated organization id in Zoho got organization: {} - {}", zohoOrganizationUrl, europeanaId); + if (logger.isDebugEnabled()) { + logger.debug("Updated organization id in Zoho got organization: {} - {}", + zohoOrganizationUrl, europeanaId); } } catch (ZohoException e) { String message = @@ -1478,58 +922,11 @@ void updateEuropeanaIDFieldInZoho(String zohoOrganizationUrl, String europeanaId public void processReferenceFields(Entity entity) { if (EntityTypes.isOrganization(entity.getType())) { Organization org = (Organization) entity; - //update country reference + // update country reference processCountryReference(org); - - //update role reference - processRoleReference(org); - } - } - - void processRoleReference(Organization org) { - if(org.getEuropeanaRoleIds()!=null && !org.getEuropeanaRoleIds().isEmpty()) { - List vocabs=vocabRepository.findByUri(org.getEuropeanaRoleIds()); - if (vocabs.isEmpty()) { - logger.warn( - "No vocabularies with the uris: {} were found in the database. Cannot assign role reference to organization with id {}", - org.getEuropeanaRoleIds(), org.getEntityId()); - } else { - org.setEuropeanaRoleRefs(vocabs); - } - } - } - void processCountryReference(Organization org) { - //country reference - if (StringUtils.isNotEmpty(org.getCountryId())) { - String europeanaCountryId = getEuropeanaCountryId(org); - if(europeanaCountryId == null) { - logger.warn("Dropping unsupported country id in consolidated entity version: {} -- {} ", org.getEntityId(), org.getCountryId()); - org.setCountryId(null); - } else { - //replace wikidata country ids - org.setCountryId(europeanaCountryId); - //search reference - EntityRecord orgCountry = entityRecordRepository.findByEntityId(europeanaCountryId); - if (orgCountry == null) { - logger.warn( - "No country found in database for the entity id: {}. Cannot assign country reference to organization with id {}", - europeanaCountryId, org.getEntityId()); - } else { - org.setCountryRef(orgCountry); - } - } + // update role reference + processRoleReference(org); } } - - String getEuropeanaCountryId(Organization org) { - if(EntityRecordUtils.isEuropeanaEntity(org.getCountryId())) { - // country id is already europeana entity - return org.getCountryId(); - }else if(WikidataUtils.isWikidataEntity(org.getCountryId())) { - //get europeana country id by wikidata id - return emConfiguration.getWikidataCountryMappings().getOrDefault(org.getCountryId(), null); - } - return null; - } }