Merge pull request #24 from europeana/EA-3597-configure-redis

redis caching

k8s/overlays/cloud/deployment_patch.yaml.template

@@ -28,4 +28,14 @@ spec:
               cpu: "${CPU_REQUEST}m"
             limits:
               memory: "${MEMORY_LIMIT}M"
-              cpu: "${CPU_LIMIT}m"
+              cpu: "${CPU_LIMIT}m"
+          volumeMounts:
+            - name: redis-certificate
+              mountPath: "/opt/app/config/translation-redis-truststore.jks"
+              readOnly: true
+              subPath: translation-redis-truststore.jks
+      volumes:
+        - name: redis-certificate
+          secret:
+            secretName: redis-secret        
+

k8s/overlays/cloud/kustomization.yaml

@@ -8,4 +8,11 @@ patchesStrategicMerge:
   - deployment_patch.yaml
 
 commonLabels:
-  app: translation-api
+  app: translation-api
+
+
+# expects files to be in the same directory
+secretGenerator:
+  - name: redis-secret
+    files:
+      - translation-redis-truststore.jks

pom.xml

@@ -74,7 +74,6 @@
 		<sonar.cpd.exclusions>**/model/**/*</sonar.cpd.exclusions>
 		<aggregate.report.xml>translation-tests/target/site/jacoco-aggregate/jacoco.xml</aggregate.report.xml>
 		<sonar.coverage.jacoco.xmlReportPaths>${aggregate.report.xml}</sonar.coverage.jacoco.xmlReportPaths>
-
 
 	</properties>
 

translation-definitions/src/main/java/eu/europeana/api/translation/definitions/vocabulary/TranslationAppConstants.java

@@ -20,6 +20,7 @@ public class TranslationAppConstants {
   public static final String FALLBACK = "fallback";
   public static final String SOURCE_LANG = "source";
   public static final String TARGET_LANG = "target";
+  public static final String CACHING = "caching";
   public static final char LANG_DELIMITER = '-';
 
   //api request/response fields

translation-tests/pom.xml

@@ -44,6 +44,13 @@
 			<artifactId>mockwebserver</artifactId>
 			<scope>test</scope>
 		</dependency>
+
+		<dependency>
+			<groupId>it.ozimov</groupId>
+		  	<artifactId>embedded-redis</artifactId>
+		  	<version>0.7.2</version>
+		  	<scope>test</scope>
+		</dependency>
 	</dependencies>
 
 

translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/BaseTranslationTest.java

@@ -38,6 +38,7 @@
 public abstract class BaseTranslationTest extends IntegrationTestUtils {
 
   protected MockMvc mockMvc;
+  protected final static int redisPort=6370;
   protected static final Logger LOG = LogManager.getLogger(BaseTranslationTest.class);
 
   @Autowired
@@ -92,7 +93,7 @@ private void initServices() throws TranslationServiceConfigurationException, Lan
       this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).build();
     }
   }
-
+  
   @AfterAll
   private void stopServices() {
     //cannot stop the mock server here as all test classes are run by the same runner and the server is static variable
@@ -125,6 +126,7 @@ static void setProperties(DynamicPropertyRegistry registry) {
 
     registry.add("translation.google.projectId", () -> "project-id-test");
     registry.add("translation.google.usehttpclient", () -> "true");
+    registry.add("redis.connection.url", () -> "redis://localhost:" + redisPort + "/");
   }
 
   /**

translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/IntegrationTestUtils.java

@@ -33,6 +33,7 @@ public abstract class IntegrationTestUtils {
   public static final String TRANSLATION_PANGEANIC_REQUEST_2 = "/content/pangeanic/translate/translate_pangeanic_request_2.json";
   public static final String TRANSLATION_PANGEANIC_RESPONSE_2 = "/content/pangeanic/translate/translate_pangeanic_response_2.json";
 
+  public static final String TRANSLATION_REQUEST_CACHING = "/content/translation_request_caching.json";
 
   public static final String TRANSLATION_WITH_FALLBACK = "/content/translation_with_fallback.json";
   public static final String TRANSLATION_BAD_REQUEST_1 = "/content/translation_bad_request_1.json";

translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/web/TranslationRestIT.java

@@ -5,9 +5,12 @@
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
@@ -28,6 +31,8 @@
 import eu.europeana.api.translation.tests.BaseTranslationTest;
 import eu.europeana.api.translation.tests.web.mock.MockGClient;
 import eu.europeana.api.translation.tests.web.mock.MockGServiceStub;
+import eu.europeana.api.translation.web.service.RedisCacheService;
+import redis.embedded.RedisServer;
 
 @SpringBootTest
 @AutoConfigureMockMvc
@@ -37,16 +42,35 @@ public class TranslationRestIT extends BaseTranslationTest {
 
   @Autowired GoogleTranslationService googleTranslationService;  
 
+  @Autowired
+  RedisCacheService redisCacheService;
+
+  private static RedisServer redisServer = startRedisService();
+
   @Autowired 
   @Qualifier(BeanNames.BEAN_GOOGLE_TRANSLATION_CLIENT_WRAPPER)
   GoogleTranslationServiceClientWrapper clientWrapper;
 
   @BeforeAll
-  void mockGoogleTranslate() throws IOException {
+  void startMockServers() throws IOException {
     TranslationServiceClient googleClient = new MockGClient(new MockGServiceStub());
     clientWrapper.setClient(googleClient);
     googleTranslationService.init(clientWrapper);
   }
+
+  static RedisServer startRedisService() {
+    //start redis server
+    RedisServer redisServer = new RedisServer(redisPort);
+    redisServer.start();
+    return redisServer;
+  }
+
+  @AfterAll void stopRedis() {
+    if(redisServer != null) {
+      redisServer.stop();
+    }
+  }
+
 
   @Test
   void translationGoogle() throws Exception {
@@ -92,6 +116,50 @@ void translationPangeanic() throws Exception {
     assertNotNull(serviceFieldValue);
   }
 
+  @Test
+  void translationWithCaching() throws Exception {
+
+    String requestJson = getJsonStringInput(TRANSLATION_REQUEST_CACHING);
+    JSONObject reqJsonObj = new JSONObject(requestJson);
+    JSONArray inputTexts = (JSONArray) reqJsonObj.get(TranslationAppConstants.TEXT);
+    List<String> inputTextsList = new ArrayList<String>();
+    for(int i=0;i<inputTexts.length();i++) {
+      inputTextsList.add((String) inputTexts.get(i));
+    }    
+    String sourceLang=reqJsonObj.getString(TranslationAppConstants.SOURCE_LANG);
+    String targetLang=reqJsonObj.getString(TranslationAppConstants.TARGET_LANG);
+
+    mockMvc
+        .perform(
+            post(BASE_URL_TRANSLATE)
+              .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)
+              .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+              .content(requestJson))
+        .andExpect(status().isOk());
+
+    //check that there are data in the cache
+    List<String> redisContent = redisCacheService.getCachedTranslations(sourceLang, targetLang, inputTextsList);
+    assertTrue(redisContent.size()==2 && Collections.frequency(redisContent, null)==0);
+
+    String cachedResult = mockMvc
+        .perform(
+            post(BASE_URL_TRANSLATE)
+              .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)
+              .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+              .content(requestJson))
+        .andExpect(status().isOk())
+        .andReturn().getResponse().getContentAsString();
+
+    assertNotNull(cachedResult);
+    JSONObject json = new JSONObject(cachedResult);
+    String langFieldValue = json.getString(TranslationAppConstants.LANG);
+    assertNotNull(langFieldValue);    
+    JSONArray translations = json.getJSONArray(TranslationAppConstants.TRANSLATIONS);
+    assertTrue(translations.length()>0);
+
+    redisCacheService.deleteAll();
+  }
+
   @Test
   void translationWithServiceParam() throws Exception {
     String requestJson = getJsonStringInput(TRANSLATION_REQUEST_2);

translation-tests/src/integration-test/resources/content/translation_bad_request_1.json

@@ -5,5 +5,6 @@
   "text": [
     "eine Textzeile auf Deutsch",
     "eine zweite Textzeile auf Deutsch"
-  ]
+  ],
+  "caching": false
 }

translation-tests/src/integration-test/resources/content/translation_bad_request_2.json

@@ -6,5 +6,6 @@
     "eine Textzeile auf Deutsch",
     "eine zweite Textzeile auf Deutsch"
   ],
-  "service": "wrong-param"
+  "service": "wrong-param",
+  "caching": false
 }

translation-tests/src/integration-test/resources/content/translation_request.json

@@ -6,5 +6,6 @@
     "eine Textzeile auf Deutsch",
     "eine zweite Textzeile auf Deutsch"
   ],
-  "service": "GOOGLE"
+  "service": "GOOGLE",
+  "caching": false
 }

translation-tests/src/integration-test/resources/content/translation_request_2.json

@@ -6,5 +6,6 @@
     "eine Textzeile auf Deutsch",
     "eine zweite Textzeile auf Deutsch"
   ],
-  "service": "PANGEANIC"
+  "service": "PANGEANIC",
+  "caching": false
 }

translation-tests/src/integration-test/resources/content/translation_request_caching.json

@@ -0,0 +1,11 @@
+{
+  "source": "de",
+  "target": "en",
+  "detect": false,
+  "text": [
+    "eine Textzeile auf Deutsch",
+    "eine zweite Textzeile auf Deutsch"
+  ],
+  "service": "PANGEANIC",
+  "caching": true
+}

translation-tests/src/integration-test/resources/content/translation_with_fallback.json

@@ -7,5 +7,6 @@
     "eine zweite Textzeile auf Deutsch"
   ],
   "service":"GOOGLE",
-  "fallback":"PANGEANIC"
+  "fallback":"PANGEANIC",
+  "caching": false
 }

translation-web/pom.xml

@@ -90,6 +90,11 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
+
+		<dependency>
+		    <groupId>org.springframework.boot</groupId>
+		    <artifactId>spring-boot-starter-data-redis</artifactId>
+		</dependency>		
 
 <!--
 		<dependency>

translation-web/src/main/java/eu/europeana/api/translation/TranslationApp.java

@@ -5,6 +5,7 @@
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
 import org.springframework.boot.autoconfigure.mongo.embedded.EmbeddedMongoAutoConfiguration;
@@ -24,7 +25,9 @@
     MongoMetricsAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class,
     SolrAutoConfiguration.class,
     // DataSources are manually configured (for EM and batch DBs)
-    DataSourceAutoConfiguration.class})
+    DataSourceAutoConfiguration.class,
+    //redis configured by application to allow disabling it
+    RedisAutoConfiguration.class})
 public class TranslationApp{
 
   /**

translation-web/src/main/java/eu/europeana/api/translation/config/BeanNames.java

@@ -17,4 +17,5 @@ public interface BeanNames {
   String BEAN_SERVICE_PROVIDER = "translationServiceProvider";
   String BEAN_SERVICE_CONFIG_INFO_CONTRIBUTOR =
       "translationServiceConfigInfoContributor";
+  String BEAN_REDIS_CACHE_SERVICE = "redisCacheService";
 }