diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/batch/model/FailedTask.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/batch/model/FailedTask.java index 737ce70fb..16ca2de97 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/batch/model/FailedTask.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/batch/model/FailedTask.java @@ -42,8 +42,8 @@ public class FailedTask { private Instant modified; - private FailedTask() { - // default constructor + FailedTask() { + // default constructor used by morphia } public FailedTask( diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java index 7768ca733..aefd8fb13 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java @@ -77,6 +77,9 @@ public class Organization extends Entity { private String countryId; @Transient private Place country; + @Transient + private String countryISO; + private List europeanaRoleIds; @Reference(lazy = true) @@ -280,10 +283,8 @@ public Place getCountry() { if(country == null && getCountryId() != null) { //set country if not dereferenced during retrieval from database country = new Place(getCountryId()); - }else if(country != null) { - //reset context to remove it from serialization - country.setContext(null); - } + } + return country; } @@ -336,4 +337,13 @@ public List getEuropeanaRole() { public void setEuropeanaRole(List europeanaRole) { this.europeanaRole = europeanaRole; } + + @JsonIgnore + public String getCountryISO() { + return countryISO; + } + + public void setCountryISO(String countryISO) { + this.countryISO = countryISO; + } } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/EntityComparator.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/EntityComparator.java index e178904d6..f231ecd25 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/EntityComparator.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/EntityComparator.java @@ -58,7 +58,6 @@ int compareClass(Object e1, Object e2) { return 0; } - @SuppressWarnings("unchecked") int compareContent(Entity e1, Entity e2) throws IllegalAccessException { List allObjectFieldsE1 = EntityUtils.getAllFields(e1.getClass()); @@ -74,6 +73,7 @@ int compareContent(Entity e1, Entity e2) throws IllegalAccessException { return 0; } + @SuppressWarnings("unchecked") private int compareEntityField(Entity e1, Entity e2, Field field) throws IllegalAccessException { Class fieldType = field.getType(); @@ -148,7 +148,6 @@ private int compareLists(List l1, List l2) { return 0; } - @SuppressWarnings("unchecked") private int compareMaps(Map m1, Map m2) { if (MapUtils.isEmpty(m1) && MapUtils.isEmpty(m2)) { // if both null or empty @@ -166,6 +165,7 @@ private int compareMaps(Map m1, Map m2) { return getMapResults(m1, m2); } + @SuppressWarnings("unchecked") private int getMapResults(Map m1, Map m2) { for (Map.Entry m1Elem : m1.entrySet()) { if (m2.containsKey(m1Elem.getKey())) { diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/SchemaOrgUtils.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/SchemaOrgUtils.java index 79044b3d3..9d606ee91 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/SchemaOrgUtils.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/SchemaOrgUtils.java @@ -1,5 +1,13 @@ package eu.europeana.entitymanagement.utils; +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import eu.europeana.corelib.edm.model.schemaorg.ContextualEntity; import eu.europeana.corelib.edm.model.schemaorg.EdmOrganization; import eu.europeana.corelib.edm.model.schemaorg.GeoCoordinates; @@ -17,29 +25,12 @@ import eu.europeana.entitymanagement.definitions.model.Organization; import eu.europeana.entitymanagement.definitions.model.Place; import eu.europeana.entitymanagement.definitions.model.TimeSpan; -import java.time.format.DateTimeFormatter; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; public final class SchemaOrgUtils { private static final Logger LOG = LogManager.getLogger(SchemaOrgUtils.class); private static final String URL_PREFIX = "http://data.europeana.eu"; - private static final String PLACE_PREFIX = "http://data.europeana.eu/place"; - private static final String TIMESPAN_PREFIX = "http://semium.org"; - private static final String UNIT_CODE_E37 = "E37"; - - private static DateTimeFormatter formatter = - DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss zzz yyyy", Locale.ENGLISH); - private static DateTimeFormatter dateFormatter = - DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.ENGLISH); private SchemaOrgUtils() { // empty constructor prevent initialization @@ -71,26 +62,26 @@ public static void processEntity(Entity entity, ContextualEntity contextualEntit * * @param concept source EDM concept * @param conceptObject Schema.Org Contextual Entity object in which the properties will be filled - * in + * in */ public static void processConcept(Concept concept, ContextualEntity conceptObject) { // @id conceptObject.setId(concept.getAbout()); // name - addMultilingualProperties( - conceptObject, SchemaOrgConstants.PROPERTY_NAME, concept.getPrefLabel()); + addMultilingualProperties(conceptObject, SchemaOrgConstants.PROPERTY_NAME, + concept.getPrefLabel()); // alternateName - addMultilingualProperties( - conceptObject, concept.getAltLabel(), SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); + addMultilingualProperties(conceptObject, concept.getAltLabel(), + SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); // description - addMultilingualProperties( - conceptObject, concept.getNote(), SchemaOrgConstants.PROPERTY_DESCRIPTION); + addMultilingualProperties(conceptObject, concept.getNote(), + SchemaOrgConstants.PROPERTY_DESCRIPTION); // sameAs - addTextProperties( - conceptObject, concept.getSameReferenceLinks(), SchemaOrgConstants.PROPERTY_SAME_AS); + addTextProperties(conceptObject, concept.getSameReferenceLinks(), + SchemaOrgConstants.PROPERTY_SAME_AS); // url addEntityPageUrl(concept, conceptObject, SchemaOrgConstants.ENTITY_PAGE_URL_CONCEPT_TYPE); @@ -103,16 +94,12 @@ public static void processConcept(Concept concept, ContextualEntity conceptObjec } } - private static void addEntityPageUrl( - Entity entity, + private static void addEntityPageUrl(Entity entity, eu.europeana.corelib.edm.model.schemaorg.ContextualEntity conceptObject, String entityPageType) { if (StringUtils.startsWithIgnoreCase(entity.getAbout(), URL_PREFIX)) { - String entityPageUrl = - String.format( - SchemaOrgConstants.ENTITY_PAGE_URL_PATTERN, - entityPageType, - EntityRecordUtils.getIdentifierFromUrl(entity.getEntityId())); + String entityPageUrl = String.format(SchemaOrgConstants.ENTITY_PAGE_URL_PATTERN, + entityPageType, EntityRecordUtils.getIdentifierFromUrl(entity.getEntityId())); conceptObject.setEntityPageUrl(entityPageUrl); } } @@ -132,43 +119,36 @@ public static void processPlace(Place edmPlace, ContextualEntity placeObject) { placeObject.setId(edmPlace.getAbout()); // name - addMultilingualProperties( - placeObject, SchemaOrgConstants.PROPERTY_NAME, edmPlace.getPrefLabel()); + addMultilingualProperties(placeObject, SchemaOrgConstants.PROPERTY_NAME, + edmPlace.getPrefLabel()); // alternateName - addMultilingualProperties( - placeObject, edmPlace.getAltLabel(), SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); + addMultilingualProperties(placeObject, edmPlace.getAltLabel(), + SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); // geo createGeoCoordinates(edmPlace, (eu.europeana.corelib.edm.model.schemaorg.Place) placeObject); // description - addMultilingualProperties( - placeObject, edmPlace.getNote(), SchemaOrgConstants.PROPERTY_DESCRIPTION); + addMultilingualProperties(placeObject, edmPlace.getNote(), + SchemaOrgConstants.PROPERTY_DESCRIPTION); // containsPlace - addReferences( - placeObject, - edmPlace.getHasPart(), - SchemaOrgConstants.PROPERTY_CONTAINS_PLACE, - eu.europeana.corelib.edm.model.schemaorg.Place.class, - null); + addReferences(placeObject, edmPlace.getHasPart(), SchemaOrgConstants.PROPERTY_CONTAINS_PLACE, + eu.europeana.corelib.edm.model.schemaorg.Place.class, null); // containedInPlace - addReferences( - placeObject, - edmPlace.getIsPartOfArray(), + addReferences(placeObject, edmPlace.getIsPartOfArray(), SchemaOrgConstants.PROPERTY_CONTAINED_IN_PLACE, - eu.europeana.corelib.edm.model.schemaorg.Place.class, - null); + eu.europeana.corelib.edm.model.schemaorg.Place.class, null); // sameAs - addTextProperties( - placeObject, edmPlace.getSameReferenceLinks(), SchemaOrgConstants.PROPERTY_SAME_AS); + addTextProperties(placeObject, edmPlace.getSameReferenceLinks(), + SchemaOrgConstants.PROPERTY_SAME_AS); // url // not available yet - // addEntityPageUrl(edmPlace, placeObject, + // addEntityPageUrl(edmPlace, placeObject, // SchemaOrgConstants.ENTITY_PAGE_URL_PLACE_TYPE); // image @@ -186,8 +166,8 @@ public static void processPlace(Place edmPlace, ContextualEntity placeObject) { * @param edmPlace EDM Place with necessary data * @param placeObject Schema.Org Place object to update */ - private static void createGeoCoordinates( - Place edmPlace, eu.europeana.corelib.edm.model.schemaorg.Place placeObject) { + private static void createGeoCoordinates(Place edmPlace, + eu.europeana.corelib.edm.model.schemaorg.Place placeObject) { GeoCoordinates geoCoordinates = new GeoCoordinates(); // latitude @@ -227,83 +207,73 @@ public static void processAgent(Agent agent, ContextualEntity agentObject) { addMultilingualProperties(agentObject, SchemaOrgConstants.PROPERTY_NAME, agent.getName()); // alternateName - addMultilingualProperties( - agentObject, agent.getAltLabel(), SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); + addMultilingualProperties(agentObject, agent.getAltLabel(), + SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); // description - addMultilingualProperties( - agentObject, agent.getNote(), SchemaOrgConstants.PROPERTY_DESCRIPTION); - addMultilingualProperties( - agentObject, agent.getBiographicalInformation(), SchemaOrgConstants.PROPERTY_DESCRIPTION); + addMultilingualProperties(agentObject, agent.getNote(), + SchemaOrgConstants.PROPERTY_DESCRIPTION); + addMultilingualProperties(agentObject, agent.getBiographicalInformation(), + SchemaOrgConstants.PROPERTY_DESCRIPTION); if (agentObject instanceof Person) { // birthDate if (agent.getDateOfBirth() != null) { - addStringProperties( - agentObject, agent.getDateOfBirth(), SchemaOrgConstants.PROPERTY_BIRTH_DATE); + addStringProperties(agentObject, agent.getDateOfBirth(), + SchemaOrgConstants.PROPERTY_BIRTH_DATE); } else if (agent.getBegin() != null) { addStringProperties(agentObject, agent.getBegin(), SchemaOrgConstants.PROPERTY_BIRTH_DATE); } // deathDate if (agent.getDateOfDeath() != null) { - addStringProperties( - agentObject, agent.getDateOfDeath(), SchemaOrgConstants.PROPERTY_DEATH_DATE); + addStringProperties(agentObject, agent.getDateOfDeath(), + SchemaOrgConstants.PROPERTY_DEATH_DATE); } else if (agent.getEnd() != null) { addStringProperties(agentObject, agent.getEnd(), SchemaOrgConstants.PROPERTY_DEATH_DATE); } // gender if (agent.getGender() != null) { - addStringProperty( - agentObject, - String.valueOf(CollectionUtils.get(agent.getGender(), 0)), + addStringProperty(agentObject, String.valueOf(CollectionUtils.get(agent.getGender(), 0)), SchemaOrgConstants.PROPERTY_GENDER); } // jobTitle - addStringProperties( - agentObject, agent.getProfessionOrOccupation(), SchemaOrgConstants.PROPERTY_JOB_TITLE); + addStringProperties(agentObject, agent.getProfessionOrOccupation(), + SchemaOrgConstants.PROPERTY_JOB_TITLE); // birthPlace - addResourceOrReferenceProperties( - agentObject, - agent.getPlaceOfBirth(), + addResourceOrReferenceProperties(agentObject, agent.getPlaceOfBirth(), SchemaOrgConstants.PROPERTY_BIRTH_PLACE, - eu.europeana.corelib.edm.model.schemaorg.Place.class, - null); + eu.europeana.corelib.edm.model.schemaorg.Place.class, null); // deathPlace - addResourceOrReferenceProperties( - agentObject, - agent.getPlaceOfDeath(), + addResourceOrReferenceProperties(agentObject, agent.getPlaceOfDeath(), SchemaOrgConstants.PROPERTY_DEATH_PLACE, - eu.europeana.corelib.edm.model.schemaorg.Place.class, - null); + eu.europeana.corelib.edm.model.schemaorg.Place.class, null); } if (agentObject instanceof eu.europeana.corelib.edm.model.schemaorg.Organization) { // foundingDate if (agent.getDateOfEstablishment() != null) { - addStringProperty( - agentObject, + addStringProperty(agentObject, String.valueOf(CollectionUtils.get(agent.getDateOfEstablishment(), 0)), SchemaOrgConstants.PROPERTY_FOUNDING_DATE); } // dissolutionDate if (agent.getDateOfTermination() != null) { - addStringProperty( - agentObject, + addStringProperty(agentObject, String.valueOf(CollectionUtils.get(agent.getDateOfTermination(), 0)), SchemaOrgConstants.PROPERTY_DISSOLUTION_DATE); } } // sameAs - addTextProperties( - agentObject, agent.getSameReferenceLinks(), SchemaOrgConstants.PROPERTY_SAME_AS); + addTextProperties(agentObject, agent.getSameReferenceLinks(), + SchemaOrgConstants.PROPERTY_SAME_AS); // url addEntityPageUrl(agent, agentObject, SchemaOrgConstants.ENTITY_PAGE_URL_AGENT_TYPE); @@ -328,30 +298,26 @@ public static void processOrganization(Organization organization, ContextualEnti entityObject.setId(organization.getAbout()); // name - addMultilingualProperties( - entityObject, SchemaOrgConstants.PROPERTY_NAME, organization.getPrefLabel()); + addMultilingualProperties(entityObject, SchemaOrgConstants.PROPERTY_NAME, + organization.getPrefLabel()); // alternateName - addMultilingualProperties( - entityObject, organization.getAltLabel(), SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); - addMultilingualProperties( - entityObject, organization.getAcronym(), SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); + addMultilingualProperties(entityObject, organization.getAltLabel(), + SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); + addMultilingualProperties(entityObject, organization.getAcronym(), + SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); // description - addMultilingualProperties( - entityObject, SchemaOrgConstants.PROPERTY_DESCRIPTION, organization.getDescription()); + addMultilingualProperties(entityObject, SchemaOrgConstants.PROPERTY_DESCRIPTION, + organization.getDescription()); // mainEntityOfPage - addTextProperties( - entityObject, - Arrays.asList(organization.getHomepage()), + addTextProperties(entityObject, Arrays.asList(organization.getHomepage()), SchemaOrgConstants.PROPERTY_MAIN_ENTITY_OF_PAGE); // logo if (organization.getLogo() != null) { - addTextProperties( - entityObject, - Arrays.asList(organization.getLogo().getId()), + addTextProperties(entityObject, Arrays.asList(organization.getLogo().getId()), SchemaOrgConstants.PROPERTY_LOGO); } @@ -362,16 +328,16 @@ public static void processOrganization(Organization organization, ContextualEnti createPostalAddress(organization, (EdmOrganization) entityObject); // identifier - addTextProperties( - entityObject, organization.getIdentifier(), SchemaOrgConstants.PROPERTY_IDENTIFIER); + addTextProperties(entityObject, organization.getIdentifier(), + SchemaOrgConstants.PROPERTY_IDENTIFIER); // sameAs - addTextProperties( - entityObject, organization.getSameReferenceLinks(), SchemaOrgConstants.PROPERTY_SAME_AS); + addTextProperties(entityObject, organization.getSameReferenceLinks(), + SchemaOrgConstants.PROPERTY_SAME_AS); // url - addEntityPageUrl( - organization, entityObject, SchemaOrgConstants.ENTITY_PAGE_URL_ORGANIZATION_TYPE); + addEntityPageUrl(organization, entityObject, + SchemaOrgConstants.ENTITY_PAGE_URL_ORGANIZATION_TYPE); // image if (organization.getDepiction() != null) { @@ -392,23 +358,23 @@ public static void processTimespan(TimeSpan time, ContextualEntity timespanObjec timespanObject.setId(time.getAbout()); // name - addMultilingualProperties( - timespanObject, SchemaOrgConstants.PROPERTY_NAME, time.getPrefLabel()); + addMultilingualProperties(timespanObject, SchemaOrgConstants.PROPERTY_NAME, + time.getPrefLabel()); // alternateName - addMultilingualProperties( - timespanObject, time.getAltLabel(), SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); + addMultilingualProperties(timespanObject, time.getAltLabel(), + SchemaOrgConstants.PROPERTY_ALTERNATE_NAME); // description - addMultilingualProperties( - timespanObject, time.getNote(), SchemaOrgConstants.PROPERTY_DESCRIPTION); + addMultilingualProperties(timespanObject, time.getNote(), + SchemaOrgConstants.PROPERTY_DESCRIPTION); // url addEntityPageUrl(time, timespanObject, SchemaOrgConstants.ENTITY_PAGE_URL_TIMESPAN_TYPE); // sameAs - addTextProperties( - timespanObject, time.getSameReferenceLinks(), SchemaOrgConstants.PROPERTY_SAME_AS); + addTextProperties(timespanObject, time.getSameReferenceLinks(), + SchemaOrgConstants.PROPERTY_SAME_AS); // image if (time.getDepiction() != null) { @@ -425,44 +391,35 @@ public static void processTimespan(TimeSpan time, ContextualEntity timespanObjec * @param organization EDM Organization with necessary data * @param organizationObject Schema.Org EdmOrganization object to update */ - private static void createPostalAddress( - Organization organization, EdmOrganization organizationObject) { + private static void createPostalAddress(Organization organization, + EdmOrganization organizationObject) { PostalAddress postalAddress = new PostalAddress(); Address address = organization.getAddress(); - if (address == null) return; + if (address == null) + return; // id postalAddress.setId(address.getAbout()); // streetAddress - addTextProperties( - postalAddress, - Arrays.asList(address.getVcardStreetAddress()), + addTextProperties(postalAddress, Arrays.asList(address.getVcardStreetAddress()), SchemaOrgConstants.PROPERTY_STREET_ADDRESS); // postalCode - addTextProperties( - postalAddress, - Arrays.asList(address.getVcardPostalCode()), + addTextProperties(postalAddress, Arrays.asList(address.getVcardPostalCode()), SchemaOrgConstants.PROPERTY_POSTAL_CODE); // postOfficeBoxNumber - addTextProperties( - postalAddress, - Arrays.asList(address.getVcardPostOfficeBox()), + addTextProperties(postalAddress, Arrays.asList(address.getVcardPostOfficeBox()), SchemaOrgConstants.PROPERTY_POST_OFFICE_BOX_NUMBER); // addressLocality - addTextProperties( - postalAddress, - Arrays.asList(address.getVcardLocality()), + addTextProperties(postalAddress, Arrays.asList(address.getVcardLocality()), SchemaOrgConstants.PROPERTY_ADDRESS_LOCALITY); // addressCountry - addTextProperties( - postalAddress, - Arrays.asList(address.getVcardCountryName()), + addTextProperties(postalAddress, Arrays.asList(address.getVcardCountryName()), SchemaOrgConstants.PROPERTY_ADDRESS_COUNTRY); // postalAddress @@ -476,22 +433,20 @@ private static void createPostalAddress( * @param map map of language to list of values to be added * @param propertyName name of property */ - private static void addMultilingualProperties( - Thing object, Map> map, String propertyName) { + private static void addMultilingualProperties(Thing object, Map> map, + String propertyName) { if (map == null) { return; } for (Map.Entry> entry : map.entrySet()) { - addMultilingualProperties( - object, - entry.getValue(), + addMultilingualProperties(object, entry.getValue(), SchemaOrgConstants.DEFAULT_LANGUAGE.equals(entry.getKey()) ? "" : entry.getKey(), propertyName); } } - private static void addMultilingualProperties( - Thing object, String propertyName, Map map) { + private static void addMultilingualProperties(Thing object, String propertyName, + Map map) { if (map == null) { return; } @@ -511,8 +466,8 @@ private static void addMultilingualProperties( * @param language language used for each value * @param propertyName name of property */ - private static void addMultilingualProperties( - Thing object, List values, String language, String propertyName) { + private static void addMultilingualProperties(Thing object, List values, String language, + String propertyName) { if (values == null) { return; } @@ -523,8 +478,8 @@ private static void addMultilingualProperties( } } - private static void addMultilingualProperty( - Thing object, String value, String language, String propertyName) { + private static void addMultilingualProperty(Thing object, String value, String language, + String propertyName) { MultilingualString property = new MultilingualString(); property.setLanguage(language); property.setValue(value); @@ -554,11 +509,8 @@ private static void addStringProperty(Thing object, String value, String propert object.addProperty(propertyName, new Text(value)); } - private static void addResourceOrReferenceProperties( - Thing object, - List entry, - String propertyName, - Class referenceClass, + private static void addResourceOrReferenceProperties(Thing object, List entry, + String propertyName, Class referenceClass, List linkedContextualEntities) { if (entry == null) { return; @@ -594,12 +546,8 @@ private static void addResourceOrReferenceProperties( * @param referenceClass class of reference * @param linkedContextualEntities list of all the references in the object */ - private static void addReferences( - Thing object, - List values, - String propertyName, - Class referenceClass, - List linkedContextualEntities) { + private static void addReferences(Thing object, List values, String propertyName, + Class referenceClass, List linkedContextualEntities) { if (values == null) { return; } @@ -619,23 +567,15 @@ private static void addReferences( * @param value value to add * @param linkedContextualEntities list of all the references in the object */ - private static void addProperty( - Thing object, - String value, - String language, - String propertyName, - Class referenceClass, - List linkedContextualEntities) { + private static void addProperty(Thing object, String value, String language, String propertyName, + Class referenceClass, List linkedContextualEntities) { if (notNullNorEmpty(value)) { if (UriValidator.isUri(value)) { addLinkedContextualEntities(value, linkedContextualEntities); addReference(object, value, propertyName, referenceClass); } else { - addMultilingualProperty( - object, - value, - SchemaOrgConstants.DEFAULT_LANGUAGE.equals(language) ? "" : language, - propertyName); + addMultilingualProperty(object, value, + SchemaOrgConstants.DEFAULT_LANGUAGE.equals(language) ? "" : language, propertyName); } } } @@ -649,8 +589,9 @@ private static void addProperty( * @param propertyName name of property * @param referenceClass class of reference that should be used for Reference object */ - private static void addReference( - Thing object, String id, String propertyName, Class referenceClass) { + @SuppressWarnings("rawtypes") + private static void addReference(Thing object, String id, String propertyName, + Class referenceClass) { Reference reference = new Reference(referenceClass); reference.setId(id); object.addProperty(propertyName, reference); @@ -666,12 +607,8 @@ private static void addReference( * @param value value to add * @param referenceClass class of reference that should be used for Reference object */ - private static void addResourceProperty( - Thing object, - String value, - String language, - String propertyName, - Class referenceClass) { + private static void addResourceProperty(Thing object, String value, String language, + String propertyName, Class referenceClass) { if (notNullNorEmpty(value)) { Thing resource = instantiateResourceObject(referenceClass); if (StringUtils.equals(language, SchemaOrgConstants.DEFAULT_LANGUAGE)) { @@ -702,21 +639,23 @@ private static void addTextProperties(Thing object, List values, String } /* - * Instantiates the reference class object, otherwise a Thing object will be - * instantiate. + * Instantiates the reference class object, otherwise a Thing object will be instantiate. * * @param referenceClass class to instantiate */ private static Thing instantiateResourceObject(Class referenceClass) { - if (referenceClass == null) return new Thing(); + if (referenceClass == null) + return new Thing(); Thing resource = null; try { - resource = referenceClass.newInstance(); - } catch (InstantiationException | IllegalAccessException e) { + resource = referenceClass.getDeclaredConstructor().newInstance(); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException | NoSuchMethodException | SecurityException e) { resource = new Thing(); - LOG.debug( - "Cannot instantiate object of class {} . Instance of Thing is used instead!", - referenceClass.getCanonicalName()); + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot instantiate object of class {} . Instance of Thing is used instead!", + referenceClass.getCanonicalName(), e); + } } return resource; } @@ -779,8 +718,8 @@ private static Class getReferenceClass(String propertyValue) { * * @param value value to be added */ - private static void addLinkedContextualEntities( - String value, List linkedContextualEntities) { + private static void addLinkedContextualEntities(String value, + List linkedContextualEntities) { if (linkedContextualEntities != null) { linkedContextualEntities.add(value); } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java index 5d663792f..cba2e627f 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java @@ -106,6 +106,7 @@ public enum EntityFieldsTypes { europeanaRoleRefs( EntityFieldsTypes.FIELD_TYPE_VOCABULARY, false, EntityFieldsTypes.FIELD_CARDINALITY_0_INFINITE), countryId(EntityFieldsTypes.FIELD_TYPE_URI, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), + countryISO(EntityFieldsTypes.FIELD_TYPE_TEXT, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), countryRef(EntityFieldsTypes.FIELD_TYPE_ENTITY_RECORD, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), homepage(EntityFieldsTypes.FIELD_TYPE_URI, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), phone( diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java index 241aeabdb..cd5f8f895 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java @@ -1,27 +1,29 @@ package eu.europeana.entitymanagement.vocabulary; -/* - * TODO: define the constant for the fields countryMap and address +/** + * Constants for solr field names */ public interface OrganizationSolrFields extends EntitySolrFields { - public static final String EXTENSION_ALL = DYNAMIC_FIELD_SEPARATOR + "*"; - public static final String DC_DESCRIPTION = "dc_description"; - public static final String DC_DESCRIPTION_ALL = "dc_description" + EXTENSION_ALL; - public static final String EDM_ACRONYM = "edm_acronym"; - public static final String EDM_ACRONYM_ALL = EDM_ACRONYM + EXTENSION_ALL; - public static final String FOAF_LOGO = "foaf_logo"; - public static final String FOAF_HOMEPAGE = "foaf_homepage"; - public static final String FOAF_PHONE = "foaf_phone"; - public static final String FOAF_MBOX = "foaf_mbox"; - public static final String EUROPEANA_ROLE = "europeanaRole"; - public static final String COUNTRY = "country"; - public static final String VCARD_HAS_ADDRESS = "vcard_hasAddress.1"; - public static final String VCARD_STREET_ADDRESS = "vcard_streetAddress.1"; - public static final String VCARD_LOCALITY = "vcard_locality.1"; - public static final String VCARD_REGION = "vcard_region.1"; - public static final String VCARD_POSTAL_CODE = "vcard_postalCode.1"; - public static final String VCARD_COUNTRYNAME = "vcard_countryName.1"; - public static final String VCARD_POST_OFFICE_BOX = "vcard_postOfficeBox.1"; - public static final String VCARD_HAS_GEO = "hasGeo"; + String EXTENSION_ALL = DYNAMIC_FIELD_SEPARATOR + "*"; + String DC_DESCRIPTION = "dc_description"; + String DC_DESCRIPTION_ALL = "dc_description" + EXTENSION_ALL; + String EDM_ACRONYM = "edm_acronym"; + String EDM_ACRONYM_ALL = EDM_ACRONYM + EXTENSION_ALL; + String FOAF_LOGO = "foaf_logo"; + String FOAF_HOMEPAGE = "foaf_homepage"; + String FOAF_PHONE = "foaf_phone"; + String FOAF_MBOX = "foaf_mbox"; + String EUROPEANA_ROLE = "europeanaRole"; + String COUNTRY = "country"; + String COUNTRY_LABEL = "countryLabel"; + String COUNTRY_LABEL_ALL = "countryLabel" + EXTENSION_ALL; + String VCARD_HAS_ADDRESS = "vcard_hasAddress.1"; + String VCARD_STREET_ADDRESS = "vcard_streetAddress.1"; + String VCARD_LOCALITY = "vcard_locality.1"; + String VCARD_REGION = "vcard_region.1"; + String VCARD_POSTAL_CODE = "vcard_postalCode.1"; + String VCARD_COUNTRYNAME = "vcard_countryName.1"; + String VCARD_POST_OFFICE_BOX = "vcard_postOfficeBox.1"; + String VCARD_HAS_GEO = "hasGeo"; } diff --git a/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/EntityRecordRepository.java b/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/EntityRecordRepository.java index a19570ca3..321334540 100644 --- a/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/EntityRecordRepository.java +++ b/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/EntityRecordRepository.java @@ -242,7 +242,7 @@ public List saveBulk(List entityRecords) { * @param filters Query filters * @return List with results */ - public List findWithCount(int start, int count, Filter[] filters) { + public List find(int start, int count, Filter[] filters) { return getDataStore().find(EntityRecord.class) .filter(filters) .iterator(new FindOptions().skip(start).sort(ascending(ENTITY_MODIFIED)).limit(count)) diff --git a/entity-management-solr/pom.xml b/entity-management-solr/pom.xml index 3f12eb733..6dde09170 100644 --- a/entity-management-solr/pom.xml +++ b/entity-management-solr/pom.xml @@ -25,8 +25,7 @@ org.apache.solr - solr-solrj - ${solrj.version} + solr-solrj diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/SolrSearchCursorIterator.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/SolrSearchCursorIterator.java index bd555d8af..7a6fd64b4 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/SolrSearchCursorIterator.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/SolrSearchCursorIterator.java @@ -1,10 +1,5 @@ package eu.europeana.entitymanagement.solr; -import eu.europeana.entitymanagement.definitions.model.Entity; -import eu.europeana.entitymanagement.solr.exception.InvalidSearchQueryException; -import eu.europeana.entitymanagement.solr.exception.SolrServiceException; -import eu.europeana.entitymanagement.solr.model.SolrEntity; -import eu.europeana.entitymanagement.vocabulary.EntitySolrFields; import java.io.IOException; import java.util.Arrays; import java.util.Collections; @@ -17,7 +12,7 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.beans.DocumentObjectBinder; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -25,6 +20,11 @@ import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; +import eu.europeana.entitymanagement.definitions.model.Entity; +import eu.europeana.entitymanagement.solr.exception.InvalidSearchQueryException; +import eu.europeana.entitymanagement.solr.exception.SolrServiceException; +import eu.europeana.entitymanagement.solr.model.SolrEntity; +import eu.europeana.entitymanagement.vocabulary.EntitySolrFields; /** * Fetches documents from Solr using a cursor. See @@ -69,7 +69,7 @@ public List> next() throws SolrServiceException try { response = client.query(solrQuery); - } catch (HttpSolrClient.RemoteSolrException e) { + } catch (BaseHttpSolrClient.RemoteSolrException e) { throw handleRemoteSolrException(solrQuery, e); } catch (SolrServerException | IOException ex) { throw new SolrServiceException( @@ -137,7 +137,7 @@ private void ensureSortClause(SolrQuery solrQuery) { } private SolrServiceException handleRemoteSolrException( - SolrQuery searchQuery, HttpSolrClient.RemoteSolrException e) { + SolrQuery searchQuery, BaseHttpSolrClient.RemoteSolrException e) { String remoteMessage = e.getMessage(); if (remoteMessage.contains(UNDEFINED_FIELD)) { diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrEntity.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrEntity.java index 44ef195b7..cf94fb2d9 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrEntity.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrEntity.java @@ -101,13 +101,21 @@ public SolrEntity(T entity) { setHiddenLabelMap(entity.getHiddenLabel()); setIsShownBy(entity.getIsShownBy()); - if (entity.getIdentifier() != null) this.identifier = new ArrayList<>(entity.getIdentifier()); - if (entity.getIsRelatedTo() != null) + if (entity.getIdentifier() != null) { + this.identifier = new ArrayList<>(entity.getIdentifier()); + } + if (entity.getIsRelatedTo() != null){ this.isRelatedTo = new ArrayList<>(entity.getIsRelatedTo()); - if (entity.getHasPart() != null) this.hasPart = new ArrayList<>(entity.getHasPart()); - if (entity.getIsPartOfArray() != null) + } + if (entity.getHasPart() != null) { + this.hasPart = new ArrayList<>(entity.getHasPart()); + } + if (entity.getIsPartOfArray() != null) { this.isPartOf = new ArrayList<>(entity.getIsPartOfArray()); - if (entity.getInScheme() != null) this.inScheme = new ArrayList<>(entity.getInScheme()); + } + if (entity.getInScheme() != null) { + this.inScheme = new ArrayList<>(entity.getInScheme()); + } if(entity.getIsAggregatedBy() != null) { this.created = entity.getIsAggregatedBy().getCreated(); diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java index 61044285e..13c83812d 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java @@ -11,7 +11,7 @@ import eu.europeana.entitymanagement.solr.SolrUtils; import eu.europeana.entitymanagement.utils.EntityUtils; import eu.europeana.entitymanagement.vocabulary.EntitySolrFields; -import eu.europeana.entitymanagement.vocabulary.OrganizationSolrFields; +import static eu.europeana.entitymanagement.vocabulary.OrganizationSolrFields.*; public class SolrOrganization extends SolrEntity { @@ -21,53 +21,55 @@ public class SolrOrganization extends SolrEntity { @Field(EntitySolrFields.AGGREGATED_VIA) private List aggregatedVia; - @Field(OrganizationSolrFields.DC_DESCRIPTION_ALL) + @Field(DC_DESCRIPTION_ALL) private Map description; - @Field(OrganizationSolrFields.EDM_ACRONYM_ALL) + @Field(EDM_ACRONYM_ALL) private Map> acronym; - @Field(OrganizationSolrFields.FOAF_LOGO) + @Field(FOAF_LOGO) private String logo; - @Field(OrganizationSolrFields.FOAF_HOMEPAGE) + @Field(FOAF_HOMEPAGE) private String homepage; - @Field(OrganizationSolrFields.FOAF_PHONE) + @Field(FOAF_PHONE) private List phone; - @Field(OrganizationSolrFields.FOAF_MBOX) + @Field(FOAF_MBOX) private List mbox; - //MAPPING TO BE ENABLED BACK IN THE NEXT VERSION - requires schema and ENtity API UPDATE - //@Field(OrganizationSolrFields.EUROPEANA_ROLE) + @Field(EUROPEANA_ROLE) private List europeanaRole; - @Field(OrganizationSolrFields.COUNTRY) - private String country; + @Field(COUNTRY) + private List country; + + @Field(COUNTRY_LABEL_ALL) + private Map countryLabel; - @Field(OrganizationSolrFields.VCARD_HAS_ADDRESS) + @Field(VCARD_HAS_ADDRESS) private String hasAddress; - @Field(OrganizationSolrFields.VCARD_STREET_ADDRESS) + @Field(VCARD_STREET_ADDRESS) private String streetAddress; - @Field(OrganizationSolrFields.VCARD_LOCALITY) + @Field(VCARD_LOCALITY) private String locality; - @Field(OrganizationSolrFields.VCARD_REGION) + @Field(VCARD_REGION) private String region; - @Field(OrganizationSolrFields.VCARD_POSTAL_CODE) + @Field(VCARD_POSTAL_CODE) private String postalCode; - @Field(OrganizationSolrFields.VCARD_COUNTRYNAME) + @Field(VCARD_COUNTRYNAME) private String countryName; - @Field(OrganizationSolrFields.VCARD_POST_OFFICE_BOX) + @Field(VCARD_POST_OFFICE_BOX) private String postBox; - @Field(OrganizationSolrFields.VCARD_HAS_GEO) + @Field(VCARD_HAS_GEO) private String hasGeo; public SolrOrganization() { @@ -88,7 +90,19 @@ public SolrOrganization(Organization organization) { if(organization.getEuropeanaRoleIds()!=null) this.europeanaRole=new ArrayList<>(organization.getEuropeanaRoleIds()); - this.country=organization.getCountryId(); + this.country=new ArrayList<>(); + String orgCountryId=organization.getCountryId(); + String orgCoutryISO=organization.getCountryISO(); + if(orgCountryId!=null) { + this.country.add(orgCountryId); + } + if(orgCoutryISO!=null) { + this.country.add(orgCoutryISO); + } + if(organization.getCountry() != null) { + this.setCountryLabel(organization.getCountry().getPrefLabel()); + } + if (organization.getSameReferenceLinks() != null) { this.sameAs = new ArrayList<>(organization.getSameReferenceLinks()); @@ -113,7 +127,7 @@ private void setDescription(Map dcDescription) { this.description = new HashMap<>( SolrUtils.normalizeStringMapByAddingPrefix( - OrganizationSolrFields.DC_DESCRIPTION + EntitySolrFields.DYNAMIC_FIELD_SEPARATOR, + DC_DESCRIPTION + EntitySolrFields.DYNAMIC_FIELD_SEPARATOR, dcDescription)); } } @@ -123,7 +137,7 @@ private void setAcronym(Map> acronym) { this.acronym = new HashMap<>( SolrUtils.normalizeStringListMapByAddingPrefix( - OrganizationSolrFields.EDM_ACRONYM + EntitySolrFields.DYNAMIC_FIELD_SEPARATOR, + EDM_ACRONYM + EntitySolrFields.DYNAMIC_FIELD_SEPARATOR, acronym)); } } @@ -197,7 +211,21 @@ public List getAggregatedVia() { return aggregatedVia; } - public String getCountry() { + public List getCountry() { return country; } + + public Map getCountryLabel() { + return countryLabel; + } + + public void setCountryLabel(Map countryLabel) { + if (MapUtils.isNotEmpty(countryLabel)) { + this.countryLabel = + new HashMap<>( + SolrUtils.normalizeStringMapByAddingPrefix( + COUNTRY_LABEL + EntitySolrFields.DYNAMIC_FIELD_SEPARATOR, + countryLabel)); + } + } } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java index ec5ea9068..5cc597f9b 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java @@ -29,11 +29,14 @@ import eu.europeana.entitymanagement.config.DataSources; import eu.europeana.entitymanagement.definitions.model.Entity; import eu.europeana.entitymanagement.definitions.model.EntityRecord; +import eu.europeana.entitymanagement.exception.EntityNotFoundException; +import eu.europeana.entitymanagement.exception.EntityRemovedException; import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; import eu.europeana.entitymanagement.testutils.MongoContainer; import eu.europeana.entitymanagement.testutils.SolrContainer; import eu.europeana.entitymanagement.testutils.TestConfig; +import eu.europeana.entitymanagement.vocabulary.EntityProfile; import eu.europeana.entitymanagement.vocabulary.EntityTypes; import eu.europeana.entitymanagement.web.MetisDereferenceUtils; import eu.europeana.entitymanagement.web.service.ConceptSchemeService; @@ -274,7 +277,7 @@ protected EntityRecord createEntity( entityUpdateService.runSynchronousUpdate(savedRecord.getEntityId()); // return entityRecord version with consolidated entity - return entityRecordService.retrieveByEntityId(savedRecord.getEntityId()).orElseThrow(); + return entityRecordService.retrieveEntityRecord(savedRecord.getEntityId(), EntityProfile.dereference.name(), false); } public static String loadFile(String resourcePath) throws IOException { @@ -283,7 +286,19 @@ public static String loadFile(String resourcePath) throws IOException { return IOUtils.toString(is, StandardCharsets.UTF_8).replace("\n", ""); } - public Optional retrieveEntity(String entityId) { - return entityRecordService.retrieveByEntityId(entityId); + /** + * @Deprecated + * @deprecated should use the entityRecordService.retrieveEntityRecord directly + * @param entityId + * @return + * @throws EntityRemovedException + * @throws EntityNotFoundException + */ + public Optional retrieveEntityEvenIfDisabled(String entityId){ + try { + return Optional.of(entityRecordService.retrieveEntityRecord(entityId, EntityProfile.dereference.name(), true)); + } catch (EntityNotFoundException | EntityRemovedException e) { + return Optional.empty(); + } } } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java index eb975acd7..2bbed29a6 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java @@ -5,7 +5,19 @@ import static eu.europeana.entitymanagement.definitions.batch.EMBatchConstants.ENTITY_ID; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; - +import java.time.Instant; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; import com.mongodb.assertions.Assertions; import dev.morphia.query.experimental.filters.Filter; import dev.morphia.query.experimental.filters.Filters; @@ -15,7 +27,6 @@ import eu.europeana.entitymanagement.batch.service.ScheduledTaskService; import eu.europeana.entitymanagement.batch.utils.BatchUtils; import eu.europeana.entitymanagement.config.AppAutoconfig; -import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; import eu.europeana.entitymanagement.definitions.batch.model.FailedTask; import eu.europeana.entitymanagement.definitions.batch.model.FailedTask.Builder; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledRemovalType; @@ -23,36 +34,19 @@ import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledUpdateType; import eu.europeana.entitymanagement.definitions.model.EntityRecord; -import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; import eu.europeana.entitymanagement.solr.model.SolrConcept; import eu.europeana.entitymanagement.solr.service.SolrService; import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; -import java.time.Instant; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; @SpringBootTest @AutoConfigureMockMvc class ScheduledTaskServiceIT extends AbstractIntegrationTest { - @Autowired private ScheduledTaskService service; + @Autowired private ScheduledTaskService scheduledTaskService; /** Used for testing deletion based on failureCount */ @Autowired private FailedTaskRepository failedTaskRepository; - /** Used for testing EntityRecord retrieval via ScheduledTask */ - @Autowired private EntityRecordRepository entityRecordRepository; - @Autowired @Qualifier(ENTITY_UPDATE_JOB_LAUNCHER) JobLauncher entityUpdateJobLauncher; @@ -69,7 +63,7 @@ class ScheduledTaskServiceIT extends AbstractIntegrationTest { @BeforeEach void setUp() throws Exception { - service.dropCollection(); + scheduledTaskService.dropCollection(); // ensure a clean FailedTasks collection failedTaskRepository.dropCollection(); entityRecordRepository.dropCollection(); @@ -89,9 +83,9 @@ void shouldCreateTasksForEntities() { Map.of( entityId1, testUpdateType, entityId2, testUpdateType); - service.scheduleTasksForEntities(map); + scheduledTaskService.scheduleTasksForEntities(map); - List tasks = service.getTasks(entityIds); + List tasks = scheduledTaskService.getTasks(entityIds); assertEquals(2, tasks.size()); } @@ -106,20 +100,22 @@ void shouldFetchRecordsForTasks() throws Exception { createEntity(europeanaMetadata, metisResponse, IntegrationTestUtils.CONCEPT_BATHTUB_URI); String entityId = savedEntityRecord.getEntityId(); Map entityIdUpdateType = Map.of(entityId, testUpdateType); - service.scheduleTasksForEntities(entityIdUpdateType); + scheduledTaskService.scheduleTasksForEntities(entityIdUpdateType); - List retrievedRecords = - service.getEntityRecordsForTasks(0, 1, new Filter[] {Filters.eq(ENTITY_ID, entityId)}); + List tasks = + scheduledTaskService.getTasks(0, 1, new Filter[] {Filters.eq(ENTITY_ID, entityId)}); - assertEquals(1, retrievedRecords.size()); - EntityRecord retrievedEntityRecord = retrievedRecords.get(0).getEntityRecord(); - - assertEquals(entityId, retrievedEntityRecord.getEntityId()); - - // No need doing a deep comparison here. Just ensure that other fields exist besides entityId - assertEquals(savedEntityRecord.getProxies().size(), retrievedEntityRecord.getProxies().size()); - assertEquals( - savedEntityRecord.getCreated().getTime(), retrievedEntityRecord.getCreated().getTime()); + assertEquals(1, tasks.size()); + assertEquals(entityId, tasks.get(0).getEntityId()); + //outdated ... becasue of changes in scheduledTasks service due to dereferencing +// EntityRecord retrievedEntityRecord = retrievedRecords.get(0).getEntityRecord(); +// +// assertEquals(entityId, retrievedEntityRecord.getEntityId()); +// +// // No need doing a deep comparison here. Just ensure that other fields exist besides entityId +// assertEquals(savedEntityRecord.getProxies().size(), retrievedEntityRecord.getProxies().size()); +// assertEquals( +// savedEntityRecord.getCreated().getTime(), retrievedEntityRecord.getCreated().getTime()); } @Test @@ -129,11 +125,11 @@ void shouldMarkAsProcessed() { Map.of( entityId1, testUpdateType, entityId2, testUpdateType); - service.scheduleTasksForEntities(map); + scheduledTaskService.scheduleTasksForEntities(map); - service.markAsProcessed(map); + scheduledTaskService.markAsProcessed(map); - List tasks = service.getTasks(entityIds); + List tasks = scheduledTaskService.getTasks(entityIds); assertEquals(2, tasks.size()); assertTrue( @@ -151,7 +147,7 @@ void shouldRemoveScheduledTasksWithFailures() { entityId1, testUpdateType, entityId2, testUpdateType); - service.scheduleTasksForEntities(map); + scheduledTaskService.scheduleTasksForEntities(map); // create failures for both FailedTask failedTask1 = new Builder(entityId1, testUpdateType).modified(Instant.now()).build(); @@ -163,9 +159,9 @@ void shouldRemoveScheduledTasksWithFailures() { failedTaskRepository.upsert(failedTask1); // remove scheduled tasks with failureCount >= 2 - service.removeScheduledTasksWithFailures(2, List.of(testUpdateType)); + scheduledTaskService.removeScheduledTasksWithFailures(2, List.of(testUpdateType)); - List tasks = service.getTasks(entityIds); + List tasks = scheduledTaskService.getTasks(entityIds); // task for entityId1 should have been deleted assertEquals(1, tasks.size()); @@ -193,10 +189,10 @@ void updateScheduledTasksExecution() throws Exception { Map.of( entityId1, ScheduledUpdateType.FULL_UPDATE, entityId2, ScheduledUpdateType.METRICS_UPDATE); - service.scheduleTasksForEntities(map); + scheduledTaskService.scheduleTasksForEntities(map); //check the count method - long runningTasks = service.getRunningTasksCount(); + long runningTasks = scheduledTaskService.getRunningTasksCount(); assertEquals(map.size(), runningTasks); Date dateBeforeRun = new Date(); @@ -210,8 +206,8 @@ void updateScheduledTasksExecution() throws Exception { List.of(ScheduledUpdateType.FULL_UPDATE, ScheduledUpdateType.METRICS_UPDATE), false)); - Optional entityRecord1Updated = retrieveEntity(entityId1); - Optional entityRecord2Updated = retrieveEntity(entityId2); + Optional entityRecord1Updated = retrieveEntityEvenIfDisabled(entityId1); + Optional entityRecord2Updated = retrieveEntityEvenIfDisabled(entityId2); /* * check the modification date for the external proxies (updated in case of the dereferce processor which is @@ -249,13 +245,13 @@ void updateScheduledTasksExecution() throws Exception { > 0); List entityIds = List.of(entityId1, entityId2); - List tasks = service.getTasks(entityIds); + List tasks = scheduledTaskService.getTasks(entityIds); // check that the scheduled tasks has been processed Assertions.assertFalse( tasks.stream().map(p -> p.hasBeenProcessed()).collect(Collectors.toList()).contains(false)); //check that not running tasks exist in the database - runningTasks = service.getRunningTasksCount(); + runningTasks = scheduledTaskService.getRunningTasksCount(); assertEquals(0, runningTasks); @@ -290,10 +286,10 @@ void deprecationAndDeletionScheduledTasksExecution() throws Exception { Map.of( entityId1, ScheduledRemovalType.DEPRECATION, entityId2, ScheduledRemovalType.PERMANENT_DELETION); - service.scheduleTasksForEntities(map); + scheduledTaskService.scheduleTasksForEntities(map); //check the count method - long runningTasks = service.getRunningTasksCount(); + long runningTasks = scheduledTaskService.getRunningTasksCount(); assertEquals(map.size(), runningTasks); entityDeletionsJobLauncher.run( @@ -305,8 +301,8 @@ void deprecationAndDeletionScheduledTasksExecution() throws Exception { List.of(ScheduledRemovalType.DEPRECATION, ScheduledRemovalType.PERMANENT_DELETION), false)); - Optional entityRecord1DbUpdated = retrieveEntity(entityId1); - Optional entityRecord2DbUpdated = retrieveEntity(entityId2); + Optional entityRecord1DbUpdated = retrieveEntityEvenIfDisabled(entityId1); + Optional entityRecord2DbUpdated = retrieveEntityEvenIfDisabled(entityId2); SolrConcept entity1Solr = emSolrService.searchById(SolrConcept.class, entityId1); SolrConcept entity2Solr = emSolrService.searchById(SolrConcept.class, entityId2); @@ -318,13 +314,13 @@ void deprecationAndDeletionScheduledTasksExecution() throws Exception { Assertions.assertNull(entity2Solr); List entityIds = List.of(entityId1, entityId2); - List tasks = service.getTasks(entityIds); + List tasks = scheduledTaskService.getTasks(entityIds); // check that the scheduled tasks has been processed Assertions.assertFalse( tasks.stream().map(p -> p.hasBeenProcessed()).collect(Collectors.toList()).contains(false)); //check the count method, should not have unprocessed tasks anymore - runningTasks = service.getRunningTasksCount(); + runningTasks = scheduledTaskService.getRunningTasksCount(); assertEquals(0, runningTasks); // check that no failed tasks are created diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java index aa99ec23e..31eb7da1d 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java @@ -35,6 +35,7 @@ import eu.europeana.entitymanagement.solr.service.SolrService; import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; import eu.europeana.entitymanagement.testutils.TestConfig; +import eu.europeana.entitymanagement.vocabulary.EntityProfile; import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; @@ -153,7 +154,7 @@ protected EntityRecord createEntity( entityUpdateService.runSynchronousUpdate(savedRecord.getEntityId()); // return entityRecord version with consolidated entity - return entityRecordService.retrieveByEntityId(savedRecord.getEntityId()).orElseThrow(); + return entityRecordService.retrieveEntityRecord(savedRecord.getEntityId(), EntityProfile.dereference.name(), false); } protected EntityRecord createConcept() throws Exception { @@ -189,7 +190,7 @@ protected EntityRecord createOrganization(String europeanaProxyEntityStr, Record entityUpdateService.runSynchronousUpdate(savedRecord.getEntityId()); // return entityRecord version with consolidated entity - return entityRecordService.retrieveByEntityId(savedRecord.getEntityId()).orElseThrow(); + return entityRecordService.retrieveEntityRecord(savedRecord.getEntityId(), EntityProfile.dereference.name(), false); } protected void deprecateEntity(EntityRecord entityRecord) throws EntityUpdateException { diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityAdminControllerIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityAdminControllerIT.java index ffea771bd..daf03b9ce 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityAdminControllerIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityAdminControllerIT.java @@ -80,7 +80,7 @@ void permanentDeletionWithSyncProfileShouldBeSuccessful() throws Exception { .accept(MediaType.APPLICATION_JSON)) .andExpect(status().isNoContent()); - Optional dbRecordOptional = retrieveEntity(entityRecord.getEntityId()); + Optional dbRecordOptional = retrieveEntityEvenIfDisabled(entityRecord.getEntityId()); Assertions.assertTrue(dbRecordOptional.isEmpty()); // confirm that Solr document no longer exists @@ -139,7 +139,7 @@ void migrationShouldBeSuccessful() throws Exception { .andExpect(jsonPath("$.proxies", hasSize(2))); // check that record is present - Optional dbRecordOptional = retrieveEntity(entityId); + Optional dbRecordOptional = retrieveEntityEvenIfDisabled(entityId); Assertions.assertFalse(dbRecordOptional.isEmpty()); } @@ -177,7 +177,7 @@ void updateForStaticDataSourceShouldBeSuccessful() throws Exception { migrateEntity("Concept", entityId, IntegrationTestUtils.STATIC_ENTITY_EXTERNAL_ID); // check that record is present - Optional dbRecordOptional = retrieveEntity(entityId); + Optional dbRecordOptional = retrieveEntityEvenIfDisabled(entityId); Assertions.assertFalse(dbRecordOptional.isEmpty()); } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityDeprecationIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityDeprecationIT.java index 2f04a5016..0c9b7d798 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityDeprecationIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityDeprecationIT.java @@ -64,7 +64,7 @@ void deprecationWithSyncProfileShouldBeSuccessful() throws Exception { .accept(MediaType.APPLICATION_JSON)) .andExpect(status().isNoContent()); - Optional dbRecordOptional = retrieveEntity(entityRecord.getEntityId()); + Optional dbRecordOptional = retrieveEntityEvenIfDisabled(entityRecord.getEntityId()); Assertions.assertTrue(dbRecordOptional.get().isDisabled()); // confirm that Solr document no longer exists diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java index 25d5dbd6d..cac61d7eb 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java @@ -7,6 +7,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesRegex; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -280,6 +281,8 @@ public void registerZohoOrganizationBergerShouldBeSuccessful() throws Exception // check if indexing is successfull by searching the organization in solr SolrOrganization org = emSolrService.searchById(SolrOrganization.class, WebEntityFields.BASE_DATA_EUROPEANA_URI + "organization/1"); assertNotNull(org.getHasAddress()); + assertNotNull(org.getEuropeanaRole()); + assertTrue(org.getCountry().size()==2); } @Test diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java index 1003c4b2b..78551a9f1 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java @@ -8,6 +8,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assert.assertNotNull; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -476,14 +477,6 @@ public void retrieveOrganizationJsonExternalWithCountryAndRoleDereference() thro String metisResponse = loadFile(IntegrationTestUtils.PLACE_SWEDEN_XML); createEntity(europeanaMetadata, metisResponse, IntegrationTestUtils.PLACE_SWEDEN_URI); -// //forcefully change the country mapping uri to the right one -// List countryMap= entityRecordService.getCountryMapping(); -// for(CountryMapping cm : countryMap) { -// if(cm.getZohoLabel().equals("Sweden, SE")) { -// cm.setEntityUri("http://data.europeana.eu/place/1"); -// } -// } - //2. register zoho GFM org europeanaMetadata = loadFile(IntegrationTestUtils.ORGANIZATION_REGISTER_GFM_ZOHO_JSON); Optional zohoRecord = @@ -504,6 +497,7 @@ public void retrieveOrganizationJsonExternalWithCountryAndRoleDereference() thro .andExpect(jsonPath("$.type", is(EntityTypes.Organization.getEntityType()))) .andExpect(jsonPath("$.sameAs").isNotEmpty()) .andExpect(jsonPath("$.country.prefLabel.en", is("Sweden"))) + .andExpect(jsonPath("$.country.prefLabel.en", notNullValue())) .andExpect(jsonPath("$.country.latitude").doesNotExist()) //only the country prefLabel, id, and type should be present .andExpect(jsonPath("$.europeanaRole[0].prefLabel.en", is("Providing Institution"))); } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityUpdateIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityUpdateIT.java index fdf106876..607320864 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityUpdateIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityUpdateIT.java @@ -119,7 +119,7 @@ void updateTimespanShouldBeSuccessful() throws Exception { .contentType(MediaType.APPLICATION_JSON)) .andExpect(status().isAccepted()); - Optional entityRecordUpdated = retrieveEntity(entityRecord.getEntityId()); + Optional entityRecordUpdated = retrieveEntityEvenIfDisabled(entityRecord.getEntityId()); Assertions.assertTrue(entityRecordUpdated.isPresent()); TimeSpan timespan = (TimeSpan) (entityRecordUpdated.get().getEntity()); Assertions.assertNotNull(timespan.getPrefLabel()); @@ -148,7 +148,7 @@ void updateConceptShouldBeSuccessful() throws Exception { final ObjectNode nodeReference = mapper.readValue( loadFile(IntegrationTestUtils.CONCEPT_UPDATE_BATHTUB_JSON), ObjectNode.class); - Optional entityRecordUpdated = retrieveEntity(entityRecord.getEntityId()); + Optional entityRecordUpdated = retrieveEntityEvenIfDisabled(entityRecord.getEntityId()); Assertions.assertTrue(entityRecordUpdated.isPresent()); Assertions.assertEquals( nodeReference.path("depiction").path("id").asText(), @@ -190,7 +190,7 @@ void updatePUTShouldReplaceEuropeanaProxy() throws Exception { .andExpect(status().isAccepted()); // check that update removed fields from Europeana proxy in original request - Optional updatedRecord = retrieveEntity(savedRecord.getEntityId()); + Optional updatedRecord = retrieveEntityEvenIfDisabled(savedRecord.getEntityId()); Assertions.assertTrue(updatedRecord.isPresent()); europeanaProxyEntity = updatedRecord.get().getEuropeanaProxy().getEntity(); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/OrganizationIndexingIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/OrganizationIndexingIT.java new file mode 100644 index 000000000..d08bdc9e6 --- /dev/null +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/OrganizationIndexingIT.java @@ -0,0 +1,61 @@ +package eu.europeana.entitymanagement.web; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.util.List; +import java.util.Optional; +import org.apache.solr.client.solrj.SolrQuery; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import com.zoho.crm.api.record.Record; +import eu.europeana.entitymanagement.config.AppAutoconfig; +import eu.europeana.entitymanagement.definitions.model.Entity; +import eu.europeana.entitymanagement.solr.model.SolrEntity; +import eu.europeana.entitymanagement.solr.service.SolrService; +import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; + +@SpringBootTest +@AutoConfigureMockMvc +public class OrganizationIndexingIT extends BaseWebControllerTest { + + @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE) + @Autowired + private SolrService emSolrService; + + @Test + public void searchOrganizationByDereferencedProps() throws Exception { + // 1. create a place "Sweden" to be used to dereference zoho country for the zoho GFM org + String europeanaMetadata = loadFile(IntegrationTestUtils.PLACE_REGISTER_SWEDEN_JSON); + String metisResponse = loadFile(IntegrationTestUtils.PLACE_SWEDEN_XML); + createEntity(europeanaMetadata, metisResponse, IntegrationTestUtils.PLACE_SWEDEN_URI); + + // 2. register zoho GFM org + europeanaMetadata = loadFile(IntegrationTestUtils.ORGANIZATION_REGISTER_GFM_ZOHO_JSON); + Optional zohoRecord = IntegrationTestUtils + .getZohoOrganizationRecord(IntegrationTestUtils.ORGANIZATION_GFM_URI_ZOHO); + + assert zohoRecord.isPresent() : "Mocked Zoho response not loaded"; + String entityId = createOrganization(europeanaMetadata, zohoRecord.get()).getEntityId(); + + // search by ISO Code + List> res = emSolrService.searchByQuery(new SolrQuery("country:SE")); + assertEquals(1, res.size()); + assertEquals(entityId, res.get(0).getEntityId()); + + // search by europeanaRole + res = emSolrService.searchByQuery(new SolrQuery( + "europeanaRole:\"http://data.europeana.eu/vocabulary/role/ProvidingInstitution\"")); + assertEquals(1, res.size()); + assertEquals(entityId, res.get(0).getEntityId()); + + // search by country label + res = emSolrService.searchByQuery(new SolrQuery("countryLabel.en:Sweden")); + assertEquals(1, res.size()); + assertEquals(entityId, res.get(0).getEntityId()); + } + + + +} diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java index 7fa028cf2..5459a0820 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java @@ -2,7 +2,6 @@ import static org.junit.Assert.assertNotNull; import java.util.Optional; -import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; diff --git a/entity-management-tests/src/integration-test/resources/content/organization.json b/entity-management-tests/src/integration-test/resources/content/organization.json index 41462810a..b553ac5da 100644 --- a/entity-management-tests/src/integration-test/resources/content/organization.json +++ b/entity-management-tests/src/integration-test/resources/content/organization.json @@ -88,7 +88,16 @@ "en": "Netherlands" } }, - + "europeanaRole": { + "type" : "Vocabulary", + "vocabularyUri" : "http://data.europeana.eu/vocabulary/role/ProvidingInstitution", + "inScheme" : [ + "http://data.europeana.eu/vocabulary/role" + ], + "prefLabel" : { + "en" : "Providing Institution" + } + }, "homepage": "https://www.naturalis.nl/nl/", "phone": [ "+31-71-751-9600" diff --git a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml index 5b64ec731..72311aff1 100644 --- a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml +++ b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml @@ -487,43 +487,20 @@ stored="true" multiValued="true" /> - - - - - - - - - - - - - - + - + + multiValued="true" /> + + - @@ -580,11 +557,7 @@ dest="text.*" /> - - - - - + @@ -595,10 +568,7 @@ dest="text" /> - - - - + diff --git a/entity-management-tests/src/integration-test/resources/zoho_country_mapping_test.json b/entity-management-tests/src/integration-test/resources/zoho_country_mapping_test.json index c518600b2..e793d7462 100644 --- a/entity-management-tests/src/integration-test/resources/zoho_country_mapping_test.json +++ b/entity-management-tests/src/integration-test/resources/zoho_country_mapping_test.json @@ -3,5 +3,10 @@ "Zoho Label": "Sweden, SE", "Entity URI": "http://data.europeana.eu/place/1", "Wikidata URI": "http://www.wikidata.org/entity/Q34" - } + }, + { + "Zoho Label": "Norway, NO", + "Entity URI": "http://data.europeana.eu/place/2", + "Wikidata URI": "http://www.wikidata.org/entity/Q20" + } ] \ No newline at end of file diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/config/EntityUpdateJobConfig.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/config/EntityUpdateJobConfig.java index f343ae2ec..6b7bbb794 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/config/EntityUpdateJobConfig.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/config/EntityUpdateJobConfig.java @@ -203,7 +203,7 @@ private SynchronizedItemStreamReader scheduledTaskReader( ScheduledTaskDatabaseReader reader = new ScheduledTaskDatabaseReader( - scheduledTaskService, + scheduledTaskService, entityRecordService, configuredBatchChunkSize, Filters.lte(EMBatchConstants.CREATED, currentStartTime), Filters.in(UPDATE_TYPE, updateTypeList)); @@ -363,4 +363,8 @@ public Job removeScheduledEntities(List removalType) { removalType, configuredBatchChunkSize, removalsStepExecutor, scheduledTaskReader)) .build(); } + + EntityVerificationLogger getVerificationLogger() { + return verificationLogger; + } } diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/BaseDatabaseReader.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/BaseDatabaseReader.java index 441233c75..cf93d32b9 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/BaseDatabaseReader.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/BaseDatabaseReader.java @@ -1,6 +1,13 @@ package eu.europeana.entitymanagement.batch.reader; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.springframework.batch.item.data.AbstractPaginatedDataItemReader; +import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; +import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask; +import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType; +import eu.europeana.entitymanagement.definitions.model.EntityRecord; public abstract class BaseDatabaseReader extends AbstractPaginatedDataItemReader { @@ -21,4 +28,23 @@ protected void doOpen() throws Exception { setPageSize(readerPageSize); setName(getClassName()); } + + List toBatchEntityRecords(List result, ScheduledTaskType scheduledTaskType) { + return + result.stream() + .map(rec -> new BatchEntityRecord(rec, scheduledTaskType)) + .collect(Collectors.toList()); + } + + List toBatchEntityRecords(List result, List scheduledTasks) { + // Use EntityId - ScheduledTaskType map for quick lookup + Map taskTypeMap = + scheduledTasks.stream() + .collect(Collectors.toMap(ScheduledTask::getEntityId, ScheduledTask::getUpdateType)); + + return result.stream() + .map(rec -> new BatchEntityRecord(rec, taskTypeMap.get(rec.getEntityId()))) + .collect(Collectors.toList()); + + } } diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java index 243b84f6f..e934f58e4 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java @@ -1,20 +1,19 @@ package eu.europeana.entitymanagement.batch.reader; -import dev.morphia.query.experimental.filters.Filter; -import eu.europeana.entitymanagement.batch.utils.BatchUtils; -import eu.europeana.entitymanagement.definitions.batch.ScheduledTaskUtils; -import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; -import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType; -import eu.europeana.entitymanagement.definitions.model.EntityRecord; -import eu.europeana.entitymanagement.web.service.EntityRecordService; import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.batch.item.ItemReader; import org.springframework.lang.NonNull; +import dev.morphia.query.experimental.filters.Filter; +import eu.europeana.entitymanagement.batch.utils.BatchUtils; +import eu.europeana.entitymanagement.definitions.batch.ScheduledTaskUtils; +import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; +import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType; +import eu.europeana.entitymanagement.definitions.model.EntityRecord; +import eu.europeana.entitymanagement.web.service.EntityRecordService; /** {@link ItemReader} that reads documents from MongoDB via a paging technique. */ public class EntityRecordDatabaseReader extends BaseDatabaseReader { @@ -41,13 +40,11 @@ protected Iterator doPageRead() { // number of items to skip when reading. pageSize is incremented in parent class every time // this method is invoked int start = page * pageSize; - List result = - entityRecordService.findEntitiesWithFilter(start, pageSize, queryFilters); + //the dereference profile is not needed here as the consolidation is taking care of processing references + List result = + entityRecordService.findEntitiesWithFilter(start, pageSize, queryFilters, null); - List batchEntityRecords = - result.stream() - .map(p -> new BatchEntityRecord(p, scheduledTaskType)) - .collect(Collectors.toList()); + List batchEntityRecords = toBatchEntityRecords(result, scheduledTaskType); if (logger.isDebugEnabled()) { logger.debug( diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java index cb0621cef..8e7ee9eea 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java @@ -1,15 +1,16 @@ package eu.europeana.entitymanagement.batch.reader; -import dev.morphia.query.experimental.filters.Filter; -import eu.europeana.entitymanagement.batch.service.ScheduledTaskService; -import eu.europeana.entitymanagement.batch.utils.BatchUtils; -import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.lang.NonNull; +import dev.morphia.query.experimental.filters.Filter; +import eu.europeana.entitymanagement.batch.service.ScheduledTaskService; +import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; +import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask; +import eu.europeana.entitymanagement.definitions.model.EntityRecord; +import eu.europeana.entitymanagement.web.service.EntityRecordService; /** * Reads scheduled entities from the ScheduledTask collection, then retrieves the matching @@ -19,13 +20,15 @@ public class ScheduledTaskDatabaseReader extends BaseDatabaseReader doPageRead() { int start = page * pageSize; - List entityRecords = - scheduledTaskService.getEntityRecordsForTasks(start, pageSize, scheduledTaskFilter); + + List scheduledTasks = + scheduledTaskService.getTasks(start, pageSize, scheduledTaskFilter); + + List entityIds = scheduledTasks.stream().map(st -> st.getEntityId()).toList(); + //no need to use the dereference profile here as the consolidation takes care of processing references + List records = entityRecordService.retrieveMultipleByEntityIds(entityIds, false, true, null); + List batchRecords = toBatchEntityRecords(records, scheduledTasks); if (logger.isDebugEnabled()) { logger.debug( "Retrieved {} scheduled entities from database. skip={}, limit={}, entityIds={}", - entityRecords.size(), + batchRecords.size(), start, pageSize, - Arrays.toString(BatchUtils.getEntityIds(entityRecords))); + entityIds); } - return entityRecords.iterator(); + return batchRecords.iterator(); } @Override diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/repository/ScheduledTaskRepository.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/repository/ScheduledTaskRepository.java index d86dd1fa2..ae0ba4abc 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/repository/ScheduledTaskRepository.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/repository/ScheduledTaskRepository.java @@ -16,7 +16,15 @@ import static eu.europeana.entitymanagement.definitions.batch.EMBatchConstants.UPDATE_TYPE; import static eu.europeana.entitymanagement.mongo.utils.MorphiaUtils.MULTI_DELETE_OPTS; import static eu.europeana.entitymanagement.mongo.utils.MorphiaUtils.UPSERT_OPTS; - +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.bson.Document; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Repository; import com.mongodb.bulk.BulkWriteResult; import com.mongodb.client.MongoCollection; import com.mongodb.client.model.UpdateOneModel; @@ -29,21 +37,10 @@ import dev.morphia.query.experimental.filters.Filter; import dev.morphia.query.internal.MorphiaCursor; import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; -import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; import eu.europeana.entitymanagement.definitions.batch.model.FailedTask; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledUpdateType; -import eu.europeana.entitymanagement.definitions.model.EntityRecord; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.bson.Document; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Repository; @Repository public class ScheduledTaskRepository implements InitializingBean { @@ -68,18 +65,15 @@ public void afterPropertiesSet() { * @param tasks list of scheduled tasks * @return BulkWriteResult of db query */ - public BulkWriteResult markAsProcessed(List tasks) { - List> updates = new ArrayList<>(); + public BulkWriteResult markAsProcessed(@NonNull List tasks) { + List> updates = new ArrayList<>(tasks.size()); for (ScheduledTask task : tasks) { - updates.add( - new UpdateOneModel<>( - // query filters on updateType - new Document(ENTITY_ID, task.getEntityId()) - .append(UPDATE_TYPE, task.getUpdateType().getValue()), - new Document( - DOC_SET, - new Document(HAS_BEEN_PROCESSED, task.hasBeenProcessed()) - .append(MODIFIED, task.getModified())))); + updates.add(new UpdateOneModel<>( + // query filters on updateType + new Document(ENTITY_ID, task.getEntityId()).append(UPDATE_TYPE, + task.getUpdateType().getValue()), + new Document(DOC_SET, new Document(HAS_BEEN_PROCESSED, task.hasBeenProcessed()) + .append(MODIFIED, task.getModified())))); } return datastore.getMapper().getCollection(ScheduledTask.class).bulkWrite(updates); } @@ -90,28 +84,24 @@ public BulkWriteResult markAsProcessed(List tasks) { * @param tasks list of tasks * @return BulkWriteResult of db query */ - public BulkWriteResult upsertBulk(List tasks) { + public BulkWriteResult upsertBulk(@NonNull List tasks) { MongoCollection collection = datastore.getMapper().getCollection(ScheduledTask.class); - List> updates = new ArrayList<>(); + List> updates = new ArrayList<>(tasks.size()); for (ScheduledTask task : tasks) { - Document updateDoc = - new Document(ENTITY_ID, task.getEntityId()) - .append(HAS_BEEN_PROCESSED, task.hasBeenProcessed()) - .append(MODIFIED, task.getModified()); + Document updateDoc = new Document(ENTITY_ID, task.getEntityId()) + .append(HAS_BEEN_PROCESSED, task.hasBeenProcessed()).append(MODIFIED, task.getModified()); - Document setOnInsertDoc = - new Document(CREATED, task.getModified()) - // manually set Morphia discriminator as we're bypassing its API for this query - .append(MORPHIA_DISCRIMINATOR, SCHEDULED_TASK_CLASSNAME); + Document setOnInsertDoc = new Document(CREATED, task.getModified()) + // manually set Morphia discriminator as we're bypassing its API for this query + .append(MORPHIA_DISCRIMINATOR, SCHEDULED_TASK_CLASSNAME); boolean shouldChangeUpdateType = task.getUpdateType() == ScheduledUpdateType.FULL_UPDATE; /* - * If entity is being scheduled for a full update, this: - * - changes the current updateType from METRICS to FULL; or - * - leaves current updateType as FULL (no change) otherwise + * If entity is being scheduled for a full update, this: - changes the current updateType from + * METRICS to FULL; or - leaves current updateType as FULL (no change) otherwise */ if (shouldChangeUpdateType) { updateDoc.append(UPDATE_TYPE, task.getUpdateType().getValue()); @@ -120,13 +110,11 @@ public BulkWriteResult upsertBulk(List tasks) { setOnInsertDoc.append(UPDATE_TYPE, task.getUpdateType().getValue()); } - updates.add( - new UpdateOneModel<>( - new Document(ENTITY_ID, task.getEntityId()), - new Document(DOC_SET, updateDoc) - // set "created" and updateType if this is a new document - .append(DOC_SET_ON_INSERT, setOnInsertDoc), - UPSERT_OPTS)); + updates.add(new UpdateOneModel<>(new Document(ENTITY_ID, task.getEntityId()), + new Document(DOC_SET, updateDoc) + // set "created" and updateType if this is a new document + .append(DOC_SET_ON_INSERT, setOnInsertDoc), + UPSERT_OPTS)); } return collection.bulkWrite(updates); @@ -139,60 +127,43 @@ public BulkWriteResult upsertBulk(List tasks) { * @return number of deleted entries */ public long removeProcessedTasks(List updateType) { - return datastore - .find(ScheduledTask.class) - .filter( - eq(HAS_BEEN_PROCESSED, Boolean.TRUE), + return datastore.find(ScheduledTask.class) + .filter(eq(HAS_BEEN_PROCESSED, Boolean.TRUE), or(updateType.stream().map(u -> eq(UPDATE_TYPE, u.getValue())).toArray(Filter[]::new))) - .delete(MULTI_DELETE_OPTS) - .getDeletedCount(); + .delete(MULTI_DELETE_OPTS).getDeletedCount(); } - - public void deleteScheduledTask(String entityId) { + + /** + * Delete scheduled tasks associated to the given entity + * @param entityId the ID of the entity for which to delete the scheduled tasks + */ + public void deleteScheduledTasks(String entityId) { datastore.find(ScheduledTask.class).filter(eq(ENTITY_ID, entityId)).delete(); } /** - * Queries the ScheduledTasks collection for records that match the given filter(s). Then - * retrieves the matching EntityRecords. Results are sorted in ascending order of entityId. + * Queries the ScheduledTasks collection to retrieve a page of results * - * @param filters Query filters + * @param start - the start index for the results page + * @param limit - the size of the current results page + * @param filters Query filters to match the results * @return List with results */ - public List getEntityRecordsForTasks(int start, int count, Filter[] filters) { - List scheduledTasks = - datastore - .find(ScheduledTask.class) - .filter(filters) - .iterator( - new FindOptions() - // we only care about the EntityID and update type - .projection() - .include(ENTITY_ID, UPDATE_TYPE) - .skip(start) - // matches the index sort order defined in ScheduledTask - // sort with _id in case of multiple matching created values - .sort(ascending(CREATED), ascending(ENTITY_ID)) - .limit(count)) - .toList(); - - // Use EntityId - ScheduledTaskType map for quick lookup - Map taskTypeMap = - scheduledTasks.stream() - .collect(Collectors.toMap(ScheduledTask::getEntityId, ScheduledTask::getUpdateType)); - - List entityRecords = - datastore - .find(EntityRecord.class) - .filter(in(ENTITY_ID, taskTypeMap.keySet())) - .iterator() - .toList(); - - return entityRecords.stream() - .map(r -> new BatchEntityRecord(r, taskTypeMap.get(r.getEntityId()))) - .collect(Collectors.toList()); + public List getTasks(int start, int limit, Filter[] filters) { + return datastore.find(ScheduledTask.class).filter(filters).iterator(new FindOptions() + // we only care about the EntityID and update type + .projection().include(ENTITY_ID, UPDATE_TYPE).skip(start) + // matches the index sort order defined in ScheduledTask + // sort with _id in case of multiple matching created values + .sort(ascending(CREATED), ascending(ENTITY_ID)).limit(limit)).toList(); + } + /** + * Retrieve the first scheduled task for the given entity + * @param entityId the id of the entity + * @return the first task found in the database + */ public ScheduledTask getTask(String entityId) { return datastore.find(ScheduledTask.class).filter(eq(ENTITY_ID, entityId)).first(); } @@ -203,9 +174,10 @@ public ScheduledTask getTask(String entityId) { * @return The number of scheduled task which are not marked as processed */ public long getRuningTasksCount() { - return datastore.find(ScheduledTask.class).filter(eq(HAS_BEEN_PROCESSED, Boolean.FALSE)).count(); + return datastore.find(ScheduledTask.class).filter(eq(HAS_BEEN_PROCESSED, Boolean.FALSE)) + .count(); } - + public List getTasks(List entityIds) { return datastore.find(ScheduledTask.class).filter(in(ENTITY_ID, entityIds)).iterator().toList(); } @@ -217,26 +189,22 @@ public void dropCollection() { /** * Gets all ScheduledTasks with failures, whose failureCount is above the maxFailedTaskRetries * value. Note: this method returns a cursor, which callers are responsible for closing + * @param maxFailedTaskRetries maximum number of failed tasks to be used as filter + * @param updateType a list of scheduled tasks types to be used for filtering + * @return the database cursor to access scheduled tasks */ - public MorphiaCursor getTasksWithFailures( - int maxFailedTaskRetries, List updateType) { - return datastore - .aggregate(ScheduledTask.class) - .match( - eq(HAS_BEEN_PROCESSED, Boolean.FALSE), - in( - UPDATE_TYPE, + public MorphiaCursor getTasksWithFailures(int maxFailedTaskRetries, + List updateType) { + return datastore.aggregate(ScheduledTask.class) + .match(eq(HAS_BEEN_PROCESSED, Boolean.FALSE), + in(UPDATE_TYPE, updateType.stream().map(ScheduledTaskType::getValue).collect(Collectors.toList()))) // both collections use the same entityId field name - .lookup( - Lookup.from(FailedTask.class) - .localField(ENTITY_ID) - .foreignField(ENTITY_ID) - .as("failed_tasks_lookup")) + .lookup(Lookup.from(FailedTask.class).localField(ENTITY_ID).foreignField(ENTITY_ID) + .as("failed_tasks_lookup")) .unwind(Unwind.on("failed_tasks_lookup")) .match(gte("failed_tasks_lookup.failureCount", maxFailedTaskRetries)) // we only care about the entityId for this query - .project(Projection.of().include(ENTITY_ID)) - .execute(ScheduledTask.class); + .project(Projection.of().include(ENTITY_ID)).execute(ScheduledTask.class); } } diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/service/ScheduledTaskService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/service/ScheduledTaskService.java index fb2e571d6..2cf14cf7c 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/service/ScheduledTaskService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/service/ScheduledTaskService.java @@ -14,7 +14,6 @@ import dev.morphia.query.internal.MorphiaCursor; import eu.europeana.entitymanagement.batch.repository.ScheduledTaskRepository; import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; -import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType; @@ -91,14 +90,14 @@ public void removeScheduledTasksWithFailures( repository.getTasksWithFailures(maxFailedTaskRetries, updateType)) { while (cursor.hasNext()) { - repository.deleteScheduledTask(cursor.next().getEntityId()); + repository.deleteScheduledTasks(cursor.next().getEntityId()); } } } - public List getEntityRecordsForTasks( + public List getTasks( int start, int count, Filter[] queryFilters) { - return repository.getEntityRecordsForTasks(start, count, queryFilters); + return repository.getTasks(start, count, queryFilters); } /** diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/EMController.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/EMController.java index be1588399..3a6dd3a52 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/EMController.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/EMController.java @@ -595,11 +595,12 @@ public ResponseEntity changeProvenance( produces = {HttpHeaders.CONTENT_TYPE_JSONLD, MediaType.APPLICATION_JSON_VALUE}) public ResponseEntity retrieveEntities( @RequestParam(value = CommonApiConstants.PARAM_WSKEY, required = false) String wskey, + @RequestParam(value = WebEntityConstants.QUERY_PARAM_PROFILE, required = false) String profiles, @RequestBody List urls, HttpServletRequest request) throws Exception { verifyReadAccess(request); - return createResponseMultipleEntities(urls, request); + return createResponseMultipleEntities(urls, profiles, request); } private ResponseEntity createResponseForRetrieve(EntityTypes type, String identifier, @@ -633,10 +634,10 @@ private ResponseEntity createResponseForRetrieve(EntityTypes type, Strin contentType, entityRecord, HttpStatus.OK); } - private ResponseEntity createResponseMultipleEntities(List entityIds, + private ResponseEntity createResponseMultipleEntities(List entityIds, String profiles, HttpServletRequest request) throws EuropeanaApiException { List entityRecords = - entityRecordService.retrieveMultipleByEntityIdsOrCoreference(entityIds); + entityRecordService.retrieveMultipleByEntityIdsOrCoreference(entityIds, profiles); // create response headers String contentType = HttpHeaders.CONTENT_TYPE_JSONLD_UTF8; diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseEntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseEntityRecordService.java index 7e57f78bd..868241562 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseEntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseEntityRecordService.java @@ -33,6 +33,7 @@ import eu.europeana.entitymanagement.definitions.model.EntityProxy; import eu.europeana.entitymanagement.definitions.model.EntityRecord; import eu.europeana.entitymanagement.definitions.model.Organization; +import eu.europeana.entitymanagement.definitions.model.Place; import eu.europeana.entitymanagement.definitions.model.Vocabulary; import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; @@ -43,6 +44,7 @@ import eu.europeana.entitymanagement.utils.EntityRecordUtils; import eu.europeana.entitymanagement.utils.UriValidator; import eu.europeana.entitymanagement.vocabulary.EntityFieldsTypes; +import eu.europeana.entitymanagement.vocabulary.EntityTypes; import eu.europeana.entitymanagement.vocabulary.WebEntityFields; import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; @@ -554,6 +556,10 @@ protected EntityProxy setExternalProxy(Entity metisResponse, String proxyId, Str } + /** + * Method used for entity consolidation + * @param org consolidated Organization to process + */ protected void processRoleReference(Organization org) { if (org.getEuropeanaRoleIds() != null && !org.getEuropeanaRoleIds().isEmpty()) { List vocabs = vocabRepository.findByUri(org.getEuropeanaRoleIds()); @@ -569,8 +575,27 @@ protected void processRoleReference(Organization org) { } } - - protected void processCountryReference(Organization org) { + /** + * Method used for entity consolidation + * @param entity consolidated Entities to process reference fields + */ + public void processReferenceFields(Entity entity) { + if (EntityTypes.isOrganization(entity.getType())) { + Organization org = (Organization) entity; + // update country reference + processCountryReference(org); + + // update role reference + processRoleReference(org); + } + } + + + /** + * Method used for entity consolidation + * @param org consolidated Organization to process + */ + void processCountryReference(Organization org) { // country reference if (StringUtils.isEmpty(org.getCountryId())) { return; @@ -589,6 +614,8 @@ protected void processCountryReference(Organization org) { EntityRecord orgCountry = entityRecordRepository.findByEntityId(europeanaCountryId, null); if (orgCountry != null) { org.setCountryRef(orgCountry); + //need to set the country during the consolidation so that it is available for indexing + setDereferencedCountry(org, orgCountry); } else if (logger.isWarnEnabled()) { logger.warn( "No country found in database for the entity id: {}. Cannot assign country reference to organization with id {}", @@ -597,6 +624,16 @@ protected void processCountryReference(Organization org) { } } + protected void setDereferencedCountry(Organization org, EntityRecord countryRecord) { + if (countryRecord != null) { + Place country = new Place(); + country.setEntityId(countryRecord.getEntity().getEntityId()); + country.setPrefLabel(countryRecord.getEntity().getPrefLabel()); + country.setContext(null); + org.setCountry(country); + } + } + String getEuropeanaCountryId(Organization org) { if (EntityRecordUtils.isEuropeanaEntity(org.getCountryId())) { diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java index 8dd9aa028..ca242bfba 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java @@ -28,7 +28,6 @@ import eu.europeana.entitymanagement.exception.EntityCreationException; import eu.europeana.entitymanagement.exception.FunctionalRuntimeException; import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; -import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; import eu.europeana.entitymanagement.mongo.repository.ZohoSyncRepository; import eu.europeana.entitymanagement.solr.exception.SolrServiceException; import eu.europeana.entitymanagement.solr.service.SolrService; @@ -48,8 +47,6 @@ public class BaseZohoAccess { final EntityRecordService entityRecordService; - final EntityRecordRepository entityRecordRepository; - final EntityUpdateService entityUpdateService; final EntityManagementConfiguration emConfiguration; @@ -65,7 +62,6 @@ public class BaseZohoAccess { public BaseZohoAccess( EntityRecordService entityRecordService, EntityUpdateService entityUpdateService, - EntityRecordRepository entityRecordRepository, EntityManagementConfiguration emConfiguration, DataSources datasources, ZohoConfiguration zohoConfiguration, @@ -73,7 +69,6 @@ public BaseZohoAccess( ZohoSyncRepository zohoSyncRepo) { this.entityRecordService = entityRecordService; this.entityUpdateService = entityUpdateService; - this.entityRecordRepository = entityRecordRepository; this.emConfiguration = emConfiguration; this.datasources = datasources; this.zohoConfiguration = zohoConfiguration; diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java index 76d29112f..6ef0cb641 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java @@ -140,4 +140,8 @@ public void dropRepository() { logger.warn("The concept scheme collection was deleted!"); this.emConceptSchemeRepo.dropCollection(); } + + SolrService getSolrService() { + return solrService; + } } diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index c60443c56..36116c74d 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -71,7 +71,7 @@ public class EntityRecordService extends BaseEntityRecordService { @Autowired public EntityRecordService(EntityRecordRepository entityRecordRepository, VocabularyRepository vocabRepository, EntityManagementConfiguration emConfiguration, - ZohoConfiguration zohoConfiguration, DataSources datasources, SolrService solrService){ + ZohoConfiguration zohoConfiguration, DataSources datasources, SolrService solrService) { super(entityRecordRepository, vocabRepository, emConfiguration, zohoConfiguration, datasources, solrService); @@ -81,7 +81,14 @@ public boolean existsByEntityId(String entityId) { return entityRecordRepository.existsByEntityId(entityId); } - public Optional retrieveByEntityId(String entityId) { + /** + * Return the entity record from the database, only for internal use in this class + * Other classes should use use {@link #retrieveEntityRecord(String, String, boolean)} + * @param entityId + * @param profiles + * @return + */ + private Optional retrieveByEntityId(String entityId) { return Optional.ofNullable(entityRecordRepository.findByEntityId(entityId, null)); } @@ -94,29 +101,36 @@ public Optional retrieveByEntityId(String entityId) { * @return */ public List retrieveMultipleByEntityIds(List entityIds, - boolean excludeDisabled, boolean fetchFullRecord) { - List resp = entityRecordRepository.findByEntityIds(entityIds, excludeDisabled, fetchFullRecord); - //for the organizations, populate the aggregatesFrom field - if(fetchFullRecord && !resp.isEmpty()) { - for(EntityRecord record : resp) { - setAggregatesFromForOrganizations(record.getEntity()); + boolean excludeDisabled, boolean fetchFullRecord, String profiles) { + List resp = + entityRecordRepository.findByEntityIds(entityIds, excludeDisabled, fetchFullRecord); + // for the organizations, populate dinamically generated fields + if (fetchFullRecord && !resp.isEmpty()) { + for (EntityRecord record : resp) { + postProcessOrganizationRetrieval(profiles, record); } } - return resp; + return resp; } - - public List retrieveMultipleByEntityIdsOrCoreference(List entityIds) { - List records=entityRecordRepository.findByEntityIdsOrCoreference(entityIds); - //sorting the list in order of the input ids, and exclude duplicates - List recordsSorted=new ArrayList<>(); - //for improved performance use a hashset to verify if the record was added to the sorted list + + public List retrieveMultipleByEntityIdsOrCoreference(List entityIds, String profiles) { + List records = entityRecordRepository.findByEntityIdsOrCoreference(entityIds); + // sorting the list in order of the input ids, and exclude duplicates + List recordsSorted = new ArrayList<>(); + // for improved performance use a hashset to verify if the record was added to the sorted list Set foundIds = new HashSet<>(entityIds.size()); + EntityRecord foundRecord; for (String id : entityIds) { - Optional recordIdMatched= records.stream().filter(er -> id.equals(er.getEntityId()) || er.getEntity().getSameReferenceLinks().contains(id)).findFirst(); - //if the record was found and was not allready added to the sorted list - if(recordIdMatched.isPresent() && !foundIds.contains(recordIdMatched.get().getEntityId())) { - recordsSorted.add(recordIdMatched.get()); - foundIds.add(recordIdMatched.get().getEntityId()); + Optional recordIdMatched = records.stream().filter( + er -> id.equals(er.getEntityId()) || er.getEntity().getSameReferenceLinks().contains(id)) + .findFirst(); + // if the record was found and was not allready added to the sorted list + if (recordIdMatched.isPresent() && !foundIds.contains(recordIdMatched.get().getEntityId())) { + foundRecord = recordIdMatched.get(); + recordsSorted.add(foundRecord); + //populate dynamic fields for organizations + postProcessOrganizationRetrieval(profiles, foundRecord); + foundIds.add(foundRecord.getEntityId()); } } return recordsSorted; @@ -140,43 +154,32 @@ public EntityRecord retrieveEntityRecord(String entityUri, String profiles, throw new EntityRemovedException( String.format(EntityRecordUtils.ENTITY_ID_REMOVED_MSG, entityUri)); } - - //for the organizations, populate the aggregatesFrom field - setAggregatesFromForOrganizations(entityRecord.getEntity()); - - // dereference morphia @Reference fields (e.g. the organization country) - if (EntityProfile.hasDereferenceProfile(profiles)) { - dereferenceLinkedEntities(entityRecord); - } + // the method checks if the record contains an organization + postProcessOrganizationRetrieval(profiles, entityRecord); return entityRecord; } - private void setAggregatesFromForOrganizations(Entity entity) { - if(EntityTypes.Organization.getEntityType().equals(entity.getType())) { - Organization org = (Organization) entity; - org.setAggregatesFrom(entityRecordRepository.findAggregatesFrom(entity.getEntityId())); - } - } - - void dereferenceLinkedEntities(EntityRecord entityRecord) { - // dereference links for organizations + void postProcessOrganizationRetrieval(String profiles, EntityRecord entityRecord) { if (EntityTypes.isOrganization(entityRecord.getEntity().getType())) { - dereferenceLinkedEntities((Organization) entityRecord.getEntity()); + // for the organizations, populate the aggregatesFrom field + Organization org = (Organization) entityRecord.getEntity(); + org.setAggregatesFrom(entityRecordRepository.findAggregatesFrom(org.getEntityId())); + + // dereference morphia @Reference fields (e.g. the organization country) + if (EntityProfile.hasDereferenceProfile(profiles)) { + dereferenceLinkedEntities(org); + } } } - - private void dereferenceLinkedEntities(Organization org) { + + public void dereferenceLinkedEntities(Organization org) { // dereference country if (org.getCountryId() != null) { - EntityRecord countryRecord = entityRecordRepository.findByEntityId(org.getCountryId(), new String[] {EntityRecordFields.ENTITY}); - if (countryRecord != null) { - Place country = new Place(); - country.setEntityId(countryRecord.getEntity().getEntityId()); - country.setPrefLabel(countryRecord.getEntity().getPrefLabel()); - org.setCountry(country); - } + EntityRecord countryRecord = entityRecordRepository.findByEntityId(org.getCountryId(), + new String[] {EntityRecordFields.ENTITY}); + setDereferencedCountry(org, countryRecord); } // dereference role if (org.getEuropeanaRoleIds() != null && !org.getEuropeanaRoleIds().isEmpty()) { @@ -212,7 +215,8 @@ public String getRedirectUriWhenNotFound(EntityTypes type, String identifier, } else if (corefEntities.size() == 1) { // found alternative entity String redirectionEntityId = corefEntities.get(0).getEntityId(); - return EntityRecordUtils.buildRedirectionLocation(identifier, redirectionEntityId, request.getRequestURI(), request.getQueryString()); + return EntityRecordUtils.buildRedirectionLocation(identifier, redirectionEntityId, + request.getRequestURI(), request.getQueryString()); } else { throw entityNotFoundException; } @@ -229,22 +233,22 @@ public String getRedirectUriWhenNotFound(EntityTypes type, String identifier, * @throws MultipleChoicesException if multiple candidates are found for redirection, typically * generated by inconsistencies in the database */ - public String getRedirectUriWhenDeprecated(EntityRecord deprecatedEntity, String identifier, - HttpServletRequest request) - throws EntityRemovedException, MultipleChoicesException { + public String getRedirectUriWhenDeprecated(EntityRecord deprecatedEntity, String identifier, + HttpServletRequest request) throws EntityRemovedException, MultipleChoicesException { // search by the entity id, using the corefs of the disabled entity List allCorefEuropeanaIds = deprecatedEntity.getEntity().getSameReferenceLinks() .stream().filter(el -> el.startsWith(WebEntityFields.BASE_DATA_EUROPEANA_URI)) .collect(Collectors.toList()); List entitiesRedirect = - retrieveMultipleByEntityIds(allCorefEuropeanaIds, true, false); + retrieveMultipleByEntityIds(allCorefEuropeanaIds, true, false, null); if (entitiesRedirect.size() > 1) { throw new MultipleChoicesException(String.format( EntityRecordUtils.MULTIPLE_CHOICES_FOR_REDIRECTION_MSG, deprecatedEntity.getEntityId(), EntityRecordUtils.getEntityIds(entitiesRedirect).toString())); } else if (entitiesRedirect.size() == 1) { String redirectionEntityId = entitiesRedirect.get(0).getEntityId(); - return EntityRecordUtils.buildRedirectionLocation(identifier, redirectionEntityId, request.getRequestURI(), request.getQueryString()); + return EntityRecordUtils.buildRedirectionLocation(identifier, redirectionEntityId, + request.getRequestURI(), request.getQueryString()); } else { throw new EntityRemovedException( String.format(EntityRecordUtils.ENTITY_ID_REMOVED_MSG, deprecatedEntity.getEntityId())); @@ -359,12 +363,31 @@ void updateEntitiesInScheme(ConceptScheme scheme) { public void enableEntityRecord(EntityRecord entityRecord) throws EntityUpdateException { // disabled records have a date value (indicating when they were disabled) entityRecord.setDisabled(null); - saveEntityRecord(entityRecord); + EntityRecord savedEntityRecord = saveEntityRecord(entityRecord); // entity needs to be added back to the solr index + indexDereferencedEntity(savedEntityRecord); + } + + /** + * Method for reindexing the entity record. In case of organizations, the record is read again + * from databases using dereference profile + * + * @param entityRecord entity record + * @throws EntityUpdateException thrown if the retrieval or indexing fails + */ + public void indexDereferencedEntity(EntityRecord entityRecord) throws EntityUpdateException { try { - solrService.storeEntity(createSolrEntity(entityRecord)); - } catch (SolrServiceException e) { + EntityRecord recordToIndex; + // for organizations we need dereference + if (EntityTypes.isOrganization(entityRecord.getEntity().getType())) { + recordToIndex = retrieveEntityRecord(entityRecord.getEntityId(), + EntityProfile.dereference.name(), false); + } else { + recordToIndex = entityRecord; + } + solrService.storeEntity(createSolrEntity(recordToIndex)); + } catch (SolrServiceException | EntityNotFoundException | EntityRemovedException e) { throw new EntityUpdateException( "Cannot create solr record for entity with id: " + entityRecord.getEntityId(), e); } @@ -859,22 +882,29 @@ public void replaceEuropeanaProxy(final Entity updateRequestEntity, EntityRecord europeanaProxy.getProxyIn().setModified(Date.from(now())); } - + public void dropRepository() { this.entityRecordRepository.dropCollection(); } /** - * Fetches records matching the provided filter(s) + * Fetches a record page matching the provided filter(s) and profile + * Use dereference profile to retrieve dereferenced entities * - * @param start - * @param count - * @param queryFilters - * @return + * @param start the start index of the records page + * @param limit the number of records to retrieve + * @param queryFilters filters to be applied for retrieval + * @param profiles the profiles to be applied + * @return the list or retrieved records */ - public List findEntitiesWithFilter(int start, int count, Filter[] queryFilters) { - return this.entityRecordRepository.findWithCount(start, count, queryFilters); + public List findEntitiesWithFilter(int start, int limit, Filter[] queryFilters, String profiles) { + List records = entityRecordRepository.find(start, limit, queryFilters); + //need to post process organizations, when dereference is used + for (EntityRecord entityRecord : records) { + postProcessOrganizationRetrieval(profiles, entityRecord); + } + return records; } private void updateEntityAggregation(EntityRecord entityRecord, String entityId, Date timestamp) { @@ -962,15 +992,5 @@ void updateEuropeanaIDFieldInZoho(String zohoOrganizationUrl, String europeanaId throw new EntityCreationException(message, e); } } - - public void processReferenceFields(Entity entity) { - if (EntityTypes.isOrganization(entity.getType())) { - Organization org = (Organization) entity; - // update country reference - processCountryReference(org); - - // update role reference - processRoleReference(org); - } - } + } diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java index fe5221a5b..c0ca6fed5 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java @@ -23,7 +23,6 @@ import eu.europeana.entitymanagement.config.DataSources; import eu.europeana.entitymanagement.definitions.model.EntityRecord; import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; -import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; import eu.europeana.entitymanagement.mongo.repository.ZohoSyncRepository; import eu.europeana.entitymanagement.solr.exception.SolrServiceException; import eu.europeana.entitymanagement.solr.service.SolrService; @@ -41,13 +40,13 @@ public class ZohoSyncService extends BaseZohoAccess { @Autowired public ZohoSyncService(EntityRecordService entityRecordService, - EntityUpdateService entityUpdateService, EntityRecordRepository entityRecordRepository, + EntityUpdateService entityUpdateService, EntityManagementConfiguration emConfiguration, DataSources datasources, ZohoConfiguration zohoConfiguration, SolrService solrService, ZohoSyncRepository zohoSyncRepo) { - super(entityRecordService, entityUpdateService, entityRecordRepository, emConfiguration, + super(entityRecordService, entityUpdateService, emConfiguration, datasources, zohoConfiguration, solrService, zohoSyncRepo); } @@ -263,8 +262,9 @@ BatchOperations fillOperations(final List orgList) { */ List findEntityRecordsByProxyId(Set modifiedInZoho) { Filter proxyIdsFilter = Filters.in("proxies.proxyId", modifiedInZoho); - return entityRecordRepository.findWithCount(0, modifiedInZoho.size(), - new Filter[] {proxyIdsFilter}); + //save to index operations are run asynchronuously, no need to dereference organizations here + return entityRecordService.findEntitiesWithFilter(0, modifiedInZoho.size(), new Filter[] {proxyIdsFilter}, null); + } private void addOperation(BatchOperations operations, Long zohoId, Record zohoOrg, diff --git a/entity-management-web/src/test/java/eu/europeana/entitymanagement/EntityManagementAppTest.java b/entity-management-web/src/test/java/eu/europeana/entitymanagement/EntityManagementAppTest.java index 5f7dff042..fba01b329 100644 --- a/entity-management-web/src/test/java/eu/europeana/entitymanagement/EntityManagementAppTest.java +++ b/entity-management-web/src/test/java/eu/europeana/entitymanagement/EntityManagementAppTest.java @@ -1,8 +1,11 @@ package eu.europeana.entitymanagement; +import static org.junit.jupiter.api.Assertions.assertNotNull; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.ApplicationContext; import org.springframework.test.context.ActiveProfiles; /** Basic test for loading context */ @@ -11,7 +14,11 @@ @ActiveProfiles("test")//enable application-test.yml class EntityManagementAppTest { - @SuppressWarnings("squid:S2699") // we are aware that this test doesn't have any assertion + @Autowired + private ApplicationContext applicationContext; + @Test - void contextLoads() {} + void contextLoads() { + assertNotNull(applicationContext.getApplicationName()); + } } diff --git a/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java index 6b4038c7c..4f0625e56 100644 --- a/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java +++ b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java @@ -1,6 +1,5 @@ package eu.europeana.entitymanagement.web; -import static eu.europeana.entitymanagement.solr.SolrUtils.createSolrEntity; import java.util.List; import javax.xml.bind.JAXBContext; import org.junit.jupiter.api.Disabled; @@ -15,9 +14,9 @@ import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; import eu.europeana.entitymanagement.mongo.repository.VocabularyRepository; -import eu.europeana.entitymanagement.solr.exception.SolrServiceException; import eu.europeana.entitymanagement.solr.service.SolrService; import eu.europeana.entitymanagement.testutils.UnitTestUtils; +import eu.europeana.entitymanagement.web.service.EntityRecordService; import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; import eu.europeana.entitymanagement.web.xml.model.XmlConceptImpl; @@ -33,6 +32,9 @@ public class UtilityTests { @Autowired private EntityRecordRepository entityRecordRepository; + @Autowired + private EntityRecordService entityRecordService; + @Autowired VocabularyRepository vocabularyRepo; @@ -43,15 +45,10 @@ public class UtilityTests { * @throws EntityUpdateException */ // @Test - public void reindexSolrFromMongo() throws EntityUpdateException { + public void reindexSolrFromMongo() throws Exception { List allRecords = entityRecordRepository.findAll(0, 100); for(EntityRecord er : allRecords) { - try { - emSolrService.storeEntity(createSolrEntity(er)); - } catch (SolrServiceException e) { - throw new EntityUpdateException( - "Cannot create solr record for entity with id: " + er, e); - } + entityRecordService.indexDereferencedEntity(er); } } diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java index e35929518..399db40ee 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java @@ -24,7 +24,6 @@ import com.zoho.crm.api.UserSignature; import com.zoho.crm.api.dc.DataCenter.Environment; import com.zoho.crm.api.dc.EUDataCenter; -import com.zoho.crm.api.dc.USDataCenter; import com.zoho.crm.api.exception.SDKException; import com.zoho.crm.api.record.APIException; import com.zoho.crm.api.record.ActionHandler; diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java index b16c99c3f..286d2d024 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java @@ -80,6 +80,7 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String //update address country address.setVcardCountryName(extractCountryName(zohoCountryLabel)); + org.setCountryISO(StringUtils.substringAfterLast(zohoCountryLabel, ",").trim()); //update organization country id if(countryMappings.containsKey(zohoCountryLabel)) { //get country ID from mappings diff --git a/pom.xml b/pom.xml index 68a617801..fcdc12bc9 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,6 @@ 2.1.4 4.9.0 1.16.0 - 7.7.2 1.7 3.0.1 2.3.3