1) added redis test containers; 2) improved translation/caching

statistics
europeana · Dec 3, 2024 · b666698 · b666698
1 parent 14768b5
commit b666698
Show file tree

Hide file tree

Showing 6 changed files with 107 additions and 40 deletions.
diff --git a/pom.xml b/pom.xml
@@ -93,6 +93,9 @@
 		<aggregate.report.xml>translation-tests/target/site/jacoco-aggregate/jacoco.xml</aggregate.report.xml>
 		<sonar.coverage.jacoco.xmlReportPaths>${aggregate.report.xml}</sonar.coverage.jacoco.xmlReportPaths>
 
+		<!-- test dependencies -->
+		<version.testcontainers>1.19.7</version.testcontainers>
+
 	</properties>
 
 	<dependencyManagement>

diff --git a/...src/main/java/eu/europeana/api/translation/definitions/model/TranslationCachingStats.java b/...src/main/java/eu/europeana/api/translation/definitions/model/TranslationCachingStats.java
@@ -0,0 +1,38 @@
+package eu.europeana.api.translation.definitions.model;
+
+/**
+ * The class for storing the translation and caching statistics
+ */
+public class TranslationCachingStats {
+
+  private int numLinesCached;
+  private int numCharsCached;
+  private int numLinesToBeTranslated;
+  private int numCharsToBeTranslated;
+
+  public int getNumLinesCached() {
+    return numLinesCached;
+  }
+  public void setNumLinesCached(int numLinesCached) {
+    this.numLinesCached = numLinesCached;
+  }
+  public int getNumCharsCached() {
+    return numCharsCached;
+  }
+  public void setNumCharsCached(int numCharsCached) {
+    this.numCharsCached = numCharsCached;
+  }
+  public int getNumLinesToBeTranslated() {
+    return numLinesToBeTranslated;
+  }
+  public void setNumLinesToBeTranslated(int numLinesToBeTranslated) {
+    this.numLinesToBeTranslated = numLinesToBeTranslated;
+  }
+  public int getNumCharsToBeTranslated() {
+    return numCharsToBeTranslated;
+  }
+  public void setNumCharsToBeTranslated(int numCharsToBeTranslated) {
+    this.numCharsToBeTranslated = numCharsToBeTranslated;
+  }
+
+}
diff --git a/translation-tests/pom.xml b/translation-tests/pom.xml
@@ -71,6 +71,18 @@
 			</exclusions>
 		</dependency>
 
+		<dependency>
+		    <groupId>org.springframework.boot</groupId>
+		    <artifactId>spring-boot-starter-data-redis</artifactId>
+		</dependency>
+
+		<dependency> 
+		    <groupId>org.testcontainers</groupId> 
+		    <artifactId>testcontainers</artifactId> 
+		    <version>${version.testcontainers}</version>
+		    <scope>test</scope> 
+		</dependency>
+
 		<dependency>
 			<groupId>com.squareup.okhttp3</groupId>
 			<artifactId>mockwebserver</artifactId>

diff --git a/...sts/src/integration-test/java/eu/europeana/api/translation/tests/BaseTranslationTest.java b/...sts/src/integration-test/java/eu/europeana/api/translation/tests/BaseTranslationTest.java
@@ -22,6 +22,8 @@
 import org.springframework.test.web.servlet.MockMvc;
 import org.springframework.test.web.servlet.setup.MockMvcBuilders;
 import org.springframework.web.context.WebApplicationContext;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.utility.DockerImageName;
 import eu.europeana.api.translation.TranslationApp;
 import eu.europeana.api.translation.config.TranslationServiceProvider;
 import eu.europeana.api.translation.service.etranslation.ETranslationTranslationService;
@@ -39,7 +41,7 @@
 public abstract class BaseTranslationTest extends IntegrationTestUtils {
 
   protected MockMvc mockMvc;
-  protected final static int redisPort=16370;
+
   protected static final Logger LOG = LogManager.getLogger(BaseTranslationTest.class);
 
   @Autowired
@@ -51,6 +53,17 @@ public abstract class BaseTranslationTest extends IntegrationTestUtils {
   /** Maps Metis dereferenciation URIs to mocked XML responses */
   public static final Map<String, String> LANG_DETECT_RESPONSE_MAP = initLanguageDetectMap();
   public static final Map<String, String> TRANSLATION_RESPONSE_MAP = initTranslationMap();
+
+  //start redis test container
+  protected final static int redisContainerPort=6379;
+  private static GenericContainer<?> redis_container = startRedisTestContainer();
+  private static GenericContainer<?> startRedisTestContainer() {
+    @SuppressWarnings("resource")
+    GenericContainer<?> redis = 
+      new GenericContainer<>(DockerImageName.parse("redis:5.0.3-alpine")).withExposedPorts(redisContainerPort);
+    redis.start();
+    return redis;
+  }
 
   /** MockWebServer needs to be static, so we can inject its port into the Spring context. */
   private static MockWebServer mockPangeanic = startPangeanicMockServer();
@@ -130,7 +143,7 @@ static void setProperties(DynamicPropertyRegistry registry) {
 
     registry.add("translation.google.projectId", () -> "project-id-test");
     registry.add("translation.google.usehttpclient", () -> "true");
-    registry.add("redis.connection.url", () -> "redis://localhost:" + redisPort + "/");
+    registry.add("redis.connection.url", () -> "redis://"+ redis_container.getHost() + ":" + redis_container.getMappedPort(redisContainerPort).toString() + "/");
     registry.add("translation.eTranslation.baseUrl", () -> ETranslationTranslationService.FAKE_BASE_URL_FOR_TESTING);
     registry.add("translation.eTranslation.credentials", () -> "");
     registry.add("translation.eTranslation.truncate", () -> false);

diff --git a/...s/src/integration-test/java/eu/europeana/api/translation/tests/web/TranslationRestIT.java b/...s/src/integration-test/java/eu/europeana/api/translation/tests/web/TranslationRestIT.java
@@ -12,7 +12,6 @@
 import java.util.List;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
-import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
@@ -37,7 +36,6 @@
 import eu.europeana.api.translation.tests.web.mock.MockGClient;
 import eu.europeana.api.translation.tests.web.mock.MockGServiceStub;
 import eu.europeana.api.translation.web.service.RedisCacheService;
-import redis.embedded.RedisServer;
 
 @SpringBootTest
 @AutoConfigureMockMvc
@@ -49,9 +47,7 @@ public class TranslationRestIT extends BaseTranslationTest {
 
   @Autowired
   RedisCacheService redisCacheService;
-
-  private static RedisServer redisServer = startRedisService();
-
+
   public static final String LANGUAGE_EN = "en";
 
 
@@ -65,21 +61,7 @@ void startMockServers() throws IOException {
     clientWrapper.setClient(googleClient);
     googleTranslationService.init(clientWrapper);
   }
-
-  static RedisServer startRedisService() {
-    //start redis server
-    RedisServer redisServer = new RedisServer(redisPort);
-    redisServer.start();
-    return redisServer;
-  }
-
-  @AfterAll void stopRedis() {
-    if(redisServer != null) {
-      redisServer.stop();
-    }
-  }
-
-
+
   @Test
   void translationGoogle() throws Exception {
     String requestJson = getJsonStringInput(TRANSLATION_REQUEST);
@@ -263,7 +245,7 @@ void translationPangeanicNoSrcMultipleLanguages() throws Exception {
             .accept(MediaType.APPLICATION_JSON)
             .contentType(MediaType.APPLICATION_JSON)
             .content(requestJson))
-//        .andExpect(status().isOk())
+        .andExpect(status().isOk())
         .andReturn().getResponse()
         .getContentAsString();
 

diff --git a/...-web/src/main/java/eu/europeana/api/translation/web/service/CachedTranslationService.java b/...-web/src/main/java/eu/europeana/api/translation/web/service/CachedTranslationService.java
@@ -4,6 +4,7 @@
 import javax.validation.constraints.NotNull;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import eu.europeana.api.translation.definitions.model.TranslationCachingStats;
 import eu.europeana.api.translation.definitions.model.TranslationObj;
 import eu.europeana.api.translation.service.AbstractTranslationService;
 import eu.europeana.api.translation.service.TranslationService;
@@ -39,19 +40,48 @@ public boolean isSupported(String srcLang, String trgLang) {
     return true;
   }
 
+  private TranslationCachingStats computeTranslationCachingStats(List<TranslationObj> allTranslObjs) { 
+    int numLinesCached=0;
+    int numCharsCached=0;
+    int numLinesToBeTranslated=0;
+    int numCharsToBeTranslated=0;
+    for(TranslationObj translObj : allTranslObjs) {
+      if(translObj.isRetrievedFromCache()) {
+        numLinesCached += 1;
+        numCharsCached += translObj.getTranslation().length();
+      }
+      if(translObj.getTranslation() == null) {
+        //objects sent for translation
+        numLinesToBeTranslated += 1;
+        numCharsToBeTranslated += translObj.getText().length();
+      }
+    }
+
+    TranslationCachingStats stats = new TranslationCachingStats();
+    stats.setNumLinesCached(numLinesCached);
+    stats.setNumCharsCached(numCharsCached);
+    stats.setNumLinesToBeTranslated(numLinesToBeTranslated);
+    stats.setNumCharsToBeTranslated(numCharsToBeTranslated);
+    return stats;
+  }
+
   @Override
   public void translate(List<TranslationObj> translationObjs) throws TranslationException {
     //fill the non translatable texts, e.g. empty Strings
-    processNonTranslatable(translationObjs);
-
-
-    fillTranslationForSameLanguage(translationObjs);
-
+    processNonTranslatable(translationObjs);   
 
     if(isCachingEnabled()) {
       redisCacheService.fillWithCachedTranslations(translationObjs);
     }
 
+    //logging the number of translated/cached lines and chars
+    if(logger.isInfoEnabled()) {
+      TranslationCachingStats stats = computeTranslationCachingStats(translationObjs);
+      logger.info("Tracking cache usage: numLinesCached={}, numCharsCached={}, numLinesToBeTranslated={}, "
+          + "numCharsToBeTranslated={}", stats.getNumLinesCached(), stats.getNumCharsCached(), 
+          stats.getNumLinesToBeTranslated(), stats.getNumCharsToBeTranslated());
+    }
+
     List<TranslationObj> toTranslate = translationObjs.stream().filter(
         t -> t.getTranslation() == null).toList();
 
@@ -62,18 +92,7 @@ public void translate(List<TranslationObj> translationObjs) throws TranslationEx
         redisCacheService.store(toTranslate);  
       }
     }
-
-    //logging the number of translated/cached lines and chars
-    int numLinesCached=(int) translationObjs.stream().filter(el -> el.isRetrievedFromCache()).count();
-    int numCharsCached=translationObjs.stream().filter(el -> el.isRetrievedFromCache()).map(el -> el.getTranslation().length()).reduce(0, Integer::sum);
-    int numLinesTranslated=toTranslate.size();
-    int numCharsTranslated=toTranslate.stream().map(el -> el.getText().length()).reduce(0, Integer::sum);
-    if(logger.isInfoEnabled()) {
-      logger.info("Tracking cache usage: numLinesCached={}, numCharsCached={}, numLinesTranslated={}, "
-          + "numCharsTranslated={}", numLinesCached, numCharsCached, numLinesTranslated, numCharsTranslated);
-    }
-
-
+
   }