make fields accessible for refactoring, added draft implementation for

zoho data generator #EA-3641

entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java

@@ -9,7 +9,7 @@ public class DataSource {
   public static final String FREQ_STATIC = "static";
   public static final String EUROPEANA_ID = "europeana";
   public static final String ZOHO_ID = "zoho-crm";
-  public static final String ZOHO_HOST = "crm.zoho.com";
+  public static final String ZOHO_HOST = "crm.zoho.eu";
 
   @JacksonXmlProperty(isAttribute = true)
   private String url;

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java

@@ -239,6 +239,10 @@ public Object getFieldValue(Field field) throws IllegalAccessException {
 
   public void setFieldValue(Field field, Object value)
       throws IllegalAccessException {
+    if(TYPE.equals(field.getName())) {
+      //type is immutable must not be overwritten
+      return;
+    }
     field.set(this, value);
   }
 

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java

@@ -28,7 +28,7 @@
 // enable test config to use zoho mocking
 @Import(TestConfig.class)
 // enable tests only on local machine
-// @Disabled
+//@Disabled
 @SpringBootTest
 public class DereferenceServiceIT extends AbstractIntegrationTest {
 
@@ -65,7 +65,7 @@ public void dereferenceConceptById() throws Exception {
     assertEquals(8, entity.getNote().size());
   }
 
-  // @Test
+  @Test
   public void zohoOrganizationDereferenceTest() throws Exception {
     String organizationId = IntegrationTestUtils.ORGANIZATION_BNF_URI_ZOHO;    
     Dereferencer dereferencer =

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java

@@ -169,7 +169,7 @@ public class IntegrationTestUtils {
   public static final String INVALID_MIGRATION_ID = "http://www.testing.org/entity/testing";
 
   public static final String ORGANIZATION_BNF_URI_ZOHO =
-      "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000002112001";
+      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000938399";
   public static final String ORGANIZATION_NATURALIS_URI_ZOHO =
       "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000370517";
   public static final String ORGANIZATION_PCCE_URI_ZOHO =

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java

@@ -4,6 +4,7 @@
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 import org.mockito.stubbing.Answer;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.TestConfiguration;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Primary;
@@ -18,6 +19,9 @@ public class TestConfig {
   public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.com/crm/org51823723/tab/Accounts/";
   public static final String MOCK_ZOHO_COUNTRY_MAPPING_FILE = "/zoho_country_mapping_test.json";
 
+  @Autowired
+  ZohoConfiguration zohoConfiguration;
+
   /**
    * Since requests to Zoho are done via its SDK, and require authentication first, we mock out the
    * entire flow with Mockito.
@@ -28,7 +32,7 @@ public ZohoConfiguration configureZoho() throws Exception {
     ZohoConfiguration zohoConfiguration = Mockito.mock(ZohoConfiguration.class);
     ZohoAccessClient zohoClient = Mockito.mock(ZohoAccessClient.class);
     Mockito.when(zohoConfiguration.getZohoAccessClient()).thenReturn(zohoClient);
-    Mockito.when(zohoConfiguration.getZohoBaseUrl()).thenReturn(MOCK_ZOHO_BASE_URL);
+    Mockito.when(zohoConfiguration.getZohoBaseUrl()).thenReturn(zohoConfiguration.getZohoBaseUrl());
 
     // find matching JSON file based on zohoId argument, then create a Record object for it
     Mockito.doAnswer(

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java

@@ -99,7 +99,13 @@ public Record deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx
         List<Choice<?>> values = new ArrayList<Choice<?>>();
         currentNode.elements().forEachRemaining(v -> values.add(new Choice<String>(v.asText())));
         record.addKeyValue(key, values);
-      }
+      }else if (currentNode.isContainerNode()){
+        System.out.println("container node: " + key);
+      }else if(currentNode.isPojo()) {
+        System.out.println("pojo node: " + key);
+      }else if(currentNode.isObject()) {
+        System.out.println("object node: " + key);
+      } 
     }
 
     // add fields with numeric suffixes

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java

@@ -0,0 +1,59 @@
+package eu.europeana.entitymanagement.zoho.organization;
+
+import static org.junit.Assert.assertNotNull;
+import java.util.Optional;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.zoho.crm.api.record.Record;
+import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration;
+import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants;
+import eu.europeana.entitymanagement.config.SerializationConfig;
+import eu.europeana.entitymanagement.testutils.IntegrationTestUtils;
+import eu.europeana.entitymanagement.testutils.ZohoRecordTestDeserializer;
+
+/** JUnit test to check if DataSources are properly deserialized from XML */
+@SpringBootTest(classes = {SerializationConfig.class, ZohoConfiguration.class, EntityManagementConfiguration.class})
+public class ZohoTestDataGenerator {
+
+  @Autowired
+  @Qualifier(AppConfigConstants.BEAN_JSON_MAPPER)
+  private ObjectMapper jsonMapper;
+
+  @Autowired
+  ZohoConfiguration zohoConfiguration;
+  EntityManagementConfiguration emConfig;
+
+  @Bean
+  ZohoDereferenceService getZohoDereferenceService() {
+    return new ZohoDereferenceService(zohoConfiguration, emConfig);
+  }  
+
+//  @Test
+  //manually enable the test when data needs to be generated
+  public void generateBNFJson() throws Exception {
+    //get original zoho record
+    Optional<Record> zohoOrganization =
+        zohoConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(IntegrationTestUtils.ORGANIZATION_BNF_URI_ZOHO);
+    String zohoRecord = getZohoDereferenceService().serialize(zohoOrganization.get());
+    //if you need to see original enable : System.out.println(zohoRecord);
+    System.out.println(zohoRecord);
+
+    //deserialize data with the test deserializer
+    ZohoRecordTestDeserializer zohoRecordDeserializer = new ZohoRecordTestDeserializer();
+    //TODO: many fields are missing or need to be renamed in the deserializer, need to include all fields from ZohoMaping except for the sensitive ones (like email addresses)
+    Record zohoTestRecord = zohoRecordDeserializer.deserialize(jsonMapper.getFactory().createParser(zohoRecord), jsonMapper.getDeserializationContext());
+
+    //serialize test data
+    String zohoTestRecordJson = getZohoDereferenceService().serialize(zohoTestRecord);
+    System.out.println(zohoTestRecordJson);
+
+    Record zohoTestRecordFiltered = zohoRecordDeserializer.deserialize(jsonMapper.getFactory().createParser(zohoTestRecordJson), jsonMapper.getDeserializationContext());
+    assertNotNull(zohoTestRecordFiltered.getId());
+    assertNotNull(zohoTestRecordFiltered.getKeyValue("Account_Name"));
+
+   }
+}

entity-management-web/.gitignore

@@ -1,2 +1,3 @@
 /SDKLogs.log*
 **/log4j2.xml
+/sdk_tokens.txt

entity-management-web/src/main/java/eu/europeana/entitymanagement/web/EMController.java

@@ -72,6 +72,7 @@
 import eu.europeana.entitymanagement.vocabulary.WebEntityConstants;
 import eu.europeana.entitymanagement.web.service.DereferenceServiceLocator;
 import eu.europeana.entitymanagement.web.service.EntityRecordService;
+import eu.europeana.entitymanagement.zoho.utils.ZohoUtils;
 import io.swagger.annotations.ApiOperation;
 
 @RestController
@@ -521,8 +522,7 @@ public ResponseEntity<String> registerEntity(@RequestBody Entity europeanaProxyE
 
     // in case of Organization it must be the zoho Organization
     String creationRequestType = europeanaProxyEntity.getType();
-    if (EntityTypes.isOrganization(creationRequestType)
-        && !creationRequestId.contains(DataSource.ZOHO_HOST)) {
+    if (!ZohoUtils.isZohoOrganization(creationRequestId, creationRequestType)) {
       throw new HttpBadRequestException(String.format(
           "The Organization entity should come from Zoho and have the corresponding id format containing: %s",
           DataSource.ZOHO_HOST));

entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/DereferenceServiceLocator.java

@@ -35,6 +35,8 @@ public Dereferencer getDereferencer(String id, String entityType) {
     if (ZohoUtils.isZohoOrganization(id, entityType)) {
       return zohoDereferenceService;
     }
+
+    System.out.println();
 
     return metisDereferenceService;
   }

entity-management-web/src/main/resources/datasources.xml

@@ -10,7 +10,7 @@
     <!-- 'europeana' id required for this data source. See DataSources class  -->
     <source url="https://www.europeana.eu/"  					   rights="https://creativecommons.org/publicdomain/zero/1.0/"    id="europeana"/>
     <!-- 'europeana' id required for this data source. See DataSources class  -->
-    <source url="https://crm.zoho.com/"  					   		rights="https://creativecommons.org/publicdomain/zero/1.0/"    id="zoho-crm"/>
+    <source url="https://crm.zoho.eu/"  					   		rights="https://creativecommons.org/publicdomain/zero/1.0/"    id="zoho-crm"/>
     <source url="http://bib.arts.kuleuven.be/photoVocabulary/" 		rights="https://creativecommons.org/publicdomain/zero/1.0/"   id="photoconsortium" frequency="static"/>
 
 </config>

entity-management-zoho/pom.xml

@@ -23,6 +23,8 @@
             <groupId>com.zoho.crm</groupId>
             <artifactId>java-sdk</artifactId>
             <version>${zoho-sdk.version}</version>
+
+            <!--3.0.1-->
         </dependency>
         <dependency>
             <groupId>org.springframework</groupId>

entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java

@@ -23,6 +23,7 @@
 import com.zoho.crm.api.SDKConfig;
 import com.zoho.crm.api.UserSignature;
 import com.zoho.crm.api.dc.DataCenter.Environment;
+import com.zoho.crm.api.dc.EUDataCenter;
 import com.zoho.crm.api.dc.USDataCenter;
 import com.zoho.crm.api.exception.SDKException;
 import com.zoho.crm.api.record.APIException;
@@ -84,7 +85,8 @@ public ZohoAccessClient(
           new OAuthToken(clientId, clientSecret, refreshToken, TokenType.REFRESH, redirectUrl);
       SDKConfig sdkConfig =
           new SDKConfig.Builder().setAutoRefreshFields(false).setPickListValidation(true).build();
-      Environment environment = USDataCenter.PRODUCTION;
+      //Environment environment = USDataCenter.PRODUCTION;
+      Environment environment = EUDataCenter.PRODUCTION;
       String resourcePath = SystemUtils.getUserHome().getAbsolutePath();
       // Does not generate any tokens, we'll need to execute a command to do so
       Initializer.initialize(

entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java

@@ -1,19 +1,19 @@
 package eu.europeana.entitymanagement.zoho.organization;
 
 import java.text.SimpleDateFormat;
-import java.util.Map;
 import java.util.Optional;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
 import org.springframework.lang.NonNull;
 import org.springframework.stereotype.Service;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
 import com.zoho.crm.api.record.Record;
 import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration;
-import eu.europeana.entitymanagement.definitions.model.CountryMapping;
 import eu.europeana.entitymanagement.definitions.model.Entity;
 import eu.europeana.entitymanagement.dereference.Dereferencer;
 
@@ -34,8 +34,9 @@ public Optional<Entity> dereferenceEntityById(@NonNull String id) throws Excepti
 
     Optional<Record> zohoOrganization =
         zohoConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(id);
-    //    Gson resp = new Gson();
-    //    System.out.println(resp.toJson(zohoOrganization.get().getKeyValues()));
+
+
+      System.out.println(serialize(zohoOrganization.get())); 
 
     if(zohoOrganization.isPresent()) {
       return Optional.of(
@@ -57,7 +58,7 @@ public String serialize(Record zohoRecord) throws JsonProcessingException {
             .serializationInclusion(JsonInclude.Include.NON_NULL)
             .build();
     mapper.findAndRegisterModules();
-    return mapper.writeValueAsString(zohoRecord.getKeyValues());
+    return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(zohoRecord.getKeyValues());
   }
 
 }