redis caching

pom.xml

@@ -72,7 +72,6 @@
 		<sonar.cpd.exclusions>**/model/**/*</sonar.cpd.exclusions>
 		<aggregate.report.xml>translation-tests/target/site/jacoco-aggregate/jacoco.xml</aggregate.report.xml>
 		<sonar.coverage.jacoco.xmlReportPaths>${aggregate.report.xml}</sonar.coverage.jacoco.xmlReportPaths>
-
 
 	</properties>
 

translation-definitions/src/main/java/eu/europeana/api/translation/definitions/vocabulary/TranslationAppConstants.java

@@ -20,6 +20,7 @@ public class TranslationAppConstants {
   public static final String FALLBACK = "fallback";
   public static final String SOURCE_LANG = "source";
   public static final String TARGET_LANG = "target";
+  public static final String CACHING = "caching";
   public static final char LANG_DELIMITER = '-';
 
   //api request/response fields

translation-tests/pom.xml

@@ -44,6 +44,13 @@
 			<artifactId>mockwebserver</artifactId>
 			<scope>test</scope>
 		</dependency>
+
+		<dependency>
+			<groupId>it.ozimov</groupId>
+		  	<artifactId>embedded-redis</artifactId>
+		  	<version>0.7.2</version>
+		  	<scope>test</scope>
+		</dependency>
 	</dependencies>
 
 

translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/BaseTranslationTest.java

@@ -38,6 +38,7 @@
 public abstract class BaseTranslationTest extends IntegrationTestUtils {
 
   protected MockMvc mockMvc;
+  protected final static int redisPort=6370;
   protected static final Logger LOG = LogManager.getLogger(BaseTranslationTest.class);
 
   @Autowired
@@ -92,7 +93,7 @@ private void initServices() throws TranslationServiceConfigurationException, Lan
       this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).build();
     }
   }
-
+  
   @AfterAll
   private void stopServices() {
     //cannot stop the mock server here as all test classes are run by the same runner and the server is static variable
@@ -125,6 +126,8 @@ static void setProperties(DynamicPropertyRegistry registry) {
 
     registry.add("translation.google.projectId", () -> "project-id-test");
     registry.add("translation.google.usehttpclient", () -> "true");
+    registry.add("spring.redis.host", () -> "localhost");
+    registry.add("spring.redis.port", () -> redisPort);
   }
 
   /**

translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/IntegrationTestUtils.java

@@ -33,6 +33,7 @@ public abstract class IntegrationTestUtils {
   public static final String TRANSLATION_PANGEANIC_REQUEST_2 = "/content/pangeanic/translate/translate_pangeanic_request_2.json";
   public static final String TRANSLATION_PANGEANIC_RESPONSE_2 = "/content/pangeanic/translate/translate_pangeanic_response_2.json";
 
+  public static final String TRANSLATION_REQUEST_CACHING = "/content/translation_request_caching.json";
 
   public static final String TRANSLATION_WITH_FALLBACK = "/content/translation_with_fallback.json";
   public static final String TRANSLATION_BAD_REQUEST_1 = "/content/translation_bad_request_1.json";

translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/web/TranslationRestIT.java

@@ -5,8 +5,10 @@
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
@@ -28,6 +30,8 @@
 import eu.europeana.api.translation.tests.BaseTranslationTest;
 import eu.europeana.api.translation.tests.web.mock.MockGClient;
 import eu.europeana.api.translation.tests.web.mock.MockGServiceStub;
+import eu.europeana.api.translation.web.service.RedisCacheService;
+import redis.embedded.RedisServer;
 
 @SpringBootTest
 @AutoConfigureMockMvc
@@ -37,6 +41,9 @@ public class TranslationRestIT extends BaseTranslationTest {
 
   @Autowired GoogleTranslationService googleTranslationService;  
 
+  @Autowired
+  RedisCacheService redisCacheService;
+
   @Autowired 
   @Qualifier(BeanNames.BEAN_GOOGLE_TRANSLATION_CLIENT_WRAPPER)
   GoogleTranslationServiceClientWrapper clientWrapper;
@@ -92,6 +99,53 @@ void translationPangeanic() throws Exception {
     assertNotNull(serviceFieldValue);
   }
 
+  @Test
+  void translationWithCaching() throws Exception {
+    RedisServer redisServer = new RedisServer(redisPort);
+    redisServer.start();
+
+    String requestJson = getJsonStringInput(TRANSLATION_REQUEST_CACHING);
+    JSONObject reqJsonObj = new JSONObject(requestJson);
+    JSONArray inputTexts = (JSONArray) reqJsonObj.get(TranslationAppConstants.TEXT);
+    List<String> inputTextsList = new ArrayList<String>();
+    for(int i=0;i<inputTexts.length();i++) {
+      inputTextsList.add((String) inputTexts.get(i));
+    }    
+    String sourceLang=reqJsonObj.getString(TranslationAppConstants.SOURCE_LANG);
+    String targetLang=reqJsonObj.getString(TranslationAppConstants.TARGET_LANG);
+
+    mockMvc
+        .perform(
+            post(BASE_URL_TRANSLATE)
+              .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)
+              .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+              .content(requestJson))
+        .andExpect(status().isOk());
+
+    //check that there are data in the cache
+    List<String> redisContent = redisCacheService.getRedisCache(sourceLang, targetLang, inputTextsList);
+    assertTrue(redisContent.size()==2 && Collections.frequency(redisContent, null)==0);
+
+    String cachedResult = mockMvc
+        .perform(
+            post(BASE_URL_TRANSLATE)
+              .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)
+              .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+              .content(requestJson))
+        .andExpect(status().isOk())
+        .andReturn().getResponse().getContentAsString();
+
+    assertNotNull(cachedResult);
+    JSONObject json = new JSONObject(cachedResult);
+    String langFieldValue = json.getString(TranslationAppConstants.LANG);
+    assertNotNull(langFieldValue);    
+    JSONArray translations = json.getJSONArray(TranslationAppConstants.TRANSLATIONS);
+    assertTrue(translations.length()>0);
+
+    redisCacheService.deleteAll();
+    redisServer.stop();
+  }
+
   @Test
   void translationWithServiceParam() throws Exception {
     String requestJson = getJsonStringInput(TRANSLATION_REQUEST_2);

translation-tests/src/integration-test/resources/content/translation_bad_request_1.json

@@ -5,5 +5,6 @@
   "text": [
     "eine Textzeile auf Deutsch",
     "eine zweite Textzeile auf Deutsch"
-  ]
+  ],
+  "caching": false
 }

translation-tests/src/integration-test/resources/content/translation_bad_request_2.json

@@ -6,5 +6,6 @@
     "eine Textzeile auf Deutsch",
     "eine zweite Textzeile auf Deutsch"
   ],
-  "service": "wrong-param"
+  "service": "wrong-param",
+  "caching": false
 }

translation-tests/src/integration-test/resources/content/translation_request.json

@@ -6,5 +6,6 @@
     "eine Textzeile auf Deutsch",
     "eine zweite Textzeile auf Deutsch"
   ],
-  "service": "GOOGLE"
+  "service": "GOOGLE",
+  "caching": false
 }

translation-tests/src/integration-test/resources/content/translation_request_2.json

@@ -6,5 +6,6 @@
     "eine Textzeile auf Deutsch",
     "eine zweite Textzeile auf Deutsch"
   ],
-  "service": "PANGEANIC"
+  "service": "PANGEANIC",
+  "caching": false
 }

translation-tests/src/integration-test/resources/content/translation_request_caching.json

@@ -0,0 +1,11 @@
+{
+  "source": "de",
+  "target": "en",
+  "detect": false,
+  "text": [
+    "eine Textzeile auf Deutsch",
+    "eine zweite Textzeile auf Deutsch"
+  ],
+  "service": "PANGEANIC",
+  "caching": true
+}

translation-tests/src/integration-test/resources/content/translation_with_fallback.json

@@ -7,5 +7,6 @@
     "eine zweite Textzeile auf Deutsch"
   ],
   "service":"GOOGLE",
-  "fallback":"PANGEANIC"
+  "fallback":"PANGEANIC",
+  "caching": false
 }

translation-web/pom.xml

@@ -90,6 +90,11 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
+
+		<dependency>
+		    <groupId>org.springframework.boot</groupId>
+		    <artifactId>spring-boot-starter-data-redis</artifactId>
+		</dependency>		
 
 <!--
 		<dependency>

translation-web/src/main/java/eu/europeana/api/translation/config/TranslationApiAutoconfig.java

@@ -19,6 +19,8 @@
 import org.springframework.context.annotation.DependsOn;
 import org.springframework.context.annotation.PropertySource;
 import org.springframework.context.support.ReloadableResourceBundleMessageSource;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
 import eu.europeana.api.commons.config.i18n.I18nService;
 import eu.europeana.api.commons.config.i18n.I18nServiceImpl;
 import eu.europeana.api.commons.oauth2.service.impl.EuropeanaClientDetailsService;
@@ -135,6 +137,18 @@ public TranslationServiceProvider getTranslationServiceProvider() {
     this.translationServiceConfigProvider = new TranslationServiceProvider();
     return this.translationServiceConfigProvider;
   }
+
+  /*
+   * For windows use this link for help: https://www.baeldung.com/spring-data-redis-properties
+   * I.e. install the docker image for Redis (it can be without the password), and put the properties (e.g. spring.redis.host) 
+   * in the translation.(user.)properties file
+   */
+  @Bean
+  public RedisTemplate<?, ?> redisTemplate(RedisConnectionFactory connectionFactory) {
+    RedisTemplate<?, ?> template = new RedisTemplate<>();
+    template.setConnectionFactory(connectionFactory);
+    return template;
+  }
 
   @Override
   public void onApplicationEvent(ApplicationStartedEvent event) {

translation-web/src/main/java/eu/europeana/api/translation/model/TranslationRequest.java

@@ -1,5 +1,6 @@
 package eu.europeana.api.translation.model;
 
+import java.util.ArrayList;
 import java.util.List;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -15,11 +16,21 @@ public class TranslationRequest {
   private String service;
   private String fallback;
   private List<String> text;
+  private Boolean caching;
 
   public TranslationRequest() {
     super();
   }
 
+  public TranslationRequest(TranslationRequest copy) {
+    this.source=copy.getSource();
+    this.target=copy.getTarget();
+    this.service=copy.getService();
+    this.fallback=copy.getFallback();
+    this.caching=copy.getCaching();
+    this.text=new ArrayList<String>(copy.getText());
+  }
+
   public String getSource() {
     return source;
   }
@@ -65,4 +76,22 @@ public void setText(List<String> text) {
     this.text = text;
   }
 
+  public boolean isCaching() {
+    if(caching==null || caching.booleanValue()) {
+      return true;
+    }
+    else {
+      return false;
+    }
+  }
+
+  public Boolean getCaching() {
+    return caching;
+  }
+
+  @JsonSetter(TranslationAppConstants.CACHING)
+  public void setCaching(boolean caching) {
+    this.caching = caching;
+  }
+
 }

translation-web/src/main/java/eu/europeana/api/translation/web/TranslationController.java

@@ -2,6 +2,9 @@
 
 import static eu.europeana.api.translation.web.I18nErrorMessageKeys.ERROR_INVALID_PARAM_VALUE;
 import static eu.europeana.api.translation.web.I18nErrorMessageKeys.ERROR_MANDATORY_PARAM_EMPTY;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import javax.servlet.http.HttpServletRequest;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -10,24 +13,28 @@
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
+import eu.europeana.api.commons.error.EuropeanaI18nApiException;
 import eu.europeana.api.commons.web.http.HttpHeaders;
 import eu.europeana.api.commons.web.model.vocabulary.Operations;
 import eu.europeana.api.translation.definitions.language.LanguagePair;
 import eu.europeana.api.translation.definitions.vocabulary.TranslationAppConstants;
 import eu.europeana.api.translation.model.TranslationRequest;
 import eu.europeana.api.translation.model.TranslationResponse;
 import eu.europeana.api.translation.web.exception.ParamValidationException;
+import eu.europeana.api.translation.web.service.RedisCacheService;
 import eu.europeana.api.translation.web.service.TranslationWebService;
 import io.swagger.v3.oas.annotations.tags.Tag;
 @RestController
 @Tag(name = "Translation endpoint", description = "Perform text translation")
 public class TranslationController extends BaseRest {
 
   private final TranslationWebService translationService;
+  private final RedisCacheService redisCacheService;
 
   @Autowired
-  public TranslationController(TranslationWebService translationService) {
+  public TranslationController(TranslationWebService translationService, RedisCacheService redisCacheService) {
     this.translationService = translationService;
+    this.redisCacheService = redisCacheService;
   }
 
   @Tag(description = "Translation", name = "translate")
@@ -39,8 +46,14 @@ public ResponseEntity<String> translate(@RequestBody TranslationRequest translRe
     verifyWriteAccess(Operations.CREATE, request);
 
     validateRequest(translRequest);
-
-    TranslationResponse result = translationService.translate(translRequest);
+
+    TranslationResponse result = null;
+    if(translRequest.isCaching()) {
+      result = getCombinedCachedAndTranslatedResults(translRequest);
+    }
+    else {
+      result = translationService.translate(translRequest);
+    }
 
     String resultJson = serialize(result);
 
@@ -63,5 +76,43 @@ private void validateRequest(TranslationRequest translationRequest) throws Param
       throw new ParamValidationException(null, null, ERROR_INVALID_PARAM_VALUE, new String[] {LanguagePair.generateKey(TranslationAppConstants.SOURCE_LANG, TranslationAppConstants.TARGET_LANG) , languagePair.toString()});
     }
   }
+
+  private TranslationResponse getCombinedCachedAndTranslatedResults(TranslationRequest translRequest) throws EuropeanaI18nApiException {
+    TranslationResponse result=null;
+    List<String> redisResp = redisCacheService.getRedisCache(translRequest.getSource(), translRequest.getTarget(), translRequest.getText());
+    if(Collections.frequency(redisResp, null)>0) {
+      TranslationRequest newTranslReq = new TranslationRequest(translRequest);
+      List<String> newText = new ArrayList<String>();
+      for(int i=0;i<redisResp.size();i++) {
+        if(redisResp.get(i)==null) {
+          newText.add(translRequest.getText().get(i));
+        }
+      }
+      newTranslReq.setText(newText);
+      result = translationService.translate(newTranslReq);
+
+      //save the translations to the cache
+      redisCacheService.saveRedisCache(newTranslReq.getSource(), newTranslReq.getTarget(), newTranslReq.getText(), result.getTranslations());
+
+      //aggregate the redis and translation responses
+      List<String> finalText=new ArrayList<String>(redisResp);
+      int counterTranslated = 0;
+      for(int i=0;i<finalText.size();i++) {
+        if(finalText.get(i)==null) {
+          finalText.set(i, result.getTranslations().get(counterTranslated));
+          counterTranslated++;
+        }
+      }
+      result.setService(null);
+      result.setTranslations(finalText);       
+    }
+    else {
+      result=new TranslationResponse();
+      result.setLang(translRequest.getTarget());
+      result.setTranslations(redisResp);
+    }
 
+    return result;
+  }
+
 }