From b66669812296844eee16e2b3faa6cf7bc075391b Mon Sep 17 00:00:00 2001 From: SrdjanStevanetic Date: Tue, 3 Dec 2024 16:34:47 +0100 Subject: [PATCH] 1) added redis test containers; 2) improved translation/caching statistics --- pom.xml | 3 ++ .../model/TranslationCachingStats.java | 38 +++++++++++++ translation-tests/pom.xml | 12 +++++ .../tests/BaseTranslationTest.java | 17 +++++- .../tests/web/TranslationRestIT.java | 24 ++------- .../web/service/CachedTranslationService.java | 53 +++++++++++++------ 6 files changed, 107 insertions(+), 40 deletions(-) create mode 100644 translation-definitions/src/main/java/eu/europeana/api/translation/definitions/model/TranslationCachingStats.java diff --git a/pom.xml b/pom.xml index 1f2a041a..0c11ac77 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,9 @@ translation-tests/target/site/jacoco-aggregate/jacoco.xml ${aggregate.report.xml} + + 1.19.7 + diff --git a/translation-definitions/src/main/java/eu/europeana/api/translation/definitions/model/TranslationCachingStats.java b/translation-definitions/src/main/java/eu/europeana/api/translation/definitions/model/TranslationCachingStats.java new file mode 100644 index 00000000..c455df46 --- /dev/null +++ b/translation-definitions/src/main/java/eu/europeana/api/translation/definitions/model/TranslationCachingStats.java @@ -0,0 +1,38 @@ +package eu.europeana.api.translation.definitions.model; + +/** + * The class for storing the translation and caching statistics + */ +public class TranslationCachingStats { + + private int numLinesCached; + private int numCharsCached; + private int numLinesToBeTranslated; + private int numCharsToBeTranslated; + + public int getNumLinesCached() { + return numLinesCached; + } + public void setNumLinesCached(int numLinesCached) { + this.numLinesCached = numLinesCached; + } + public int getNumCharsCached() { + return numCharsCached; + } + public void setNumCharsCached(int numCharsCached) { + this.numCharsCached = numCharsCached; + } + public int getNumLinesToBeTranslated() { + return numLinesToBeTranslated; + } + public void setNumLinesToBeTranslated(int numLinesToBeTranslated) { + this.numLinesToBeTranslated = numLinesToBeTranslated; + } + public int getNumCharsToBeTranslated() { + return numCharsToBeTranslated; + } + public void setNumCharsToBeTranslated(int numCharsToBeTranslated) { + this.numCharsToBeTranslated = numCharsToBeTranslated; + } + +} diff --git a/translation-tests/pom.xml b/translation-tests/pom.xml index 261c8da1..91799166 100644 --- a/translation-tests/pom.xml +++ b/translation-tests/pom.xml @@ -71,6 +71,18 @@ + + org.springframework.boot + spring-boot-starter-data-redis + + + + org.testcontainers + testcontainers + ${version.testcontainers} + test + + com.squareup.okhttp3 mockwebserver diff --git a/translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/BaseTranslationTest.java b/translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/BaseTranslationTest.java index 248b6db4..ad1fd853 100644 --- a/translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/BaseTranslationTest.java +++ b/translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/BaseTranslationTest.java @@ -22,6 +22,8 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; import eu.europeana.api.translation.TranslationApp; import eu.europeana.api.translation.config.TranslationServiceProvider; import eu.europeana.api.translation.service.etranslation.ETranslationTranslationService; @@ -39,7 +41,7 @@ public abstract class BaseTranslationTest extends IntegrationTestUtils { protected MockMvc mockMvc; - protected final static int redisPort=16370; + protected static final Logger LOG = LogManager.getLogger(BaseTranslationTest.class); @Autowired @@ -51,6 +53,17 @@ public abstract class BaseTranslationTest extends IntegrationTestUtils { /** Maps Metis dereferenciation URIs to mocked XML responses */ public static final Map LANG_DETECT_RESPONSE_MAP = initLanguageDetectMap(); public static final Map TRANSLATION_RESPONSE_MAP = initTranslationMap(); + + //start redis test container + protected final static int redisContainerPort=6379; + private static GenericContainer redis_container = startRedisTestContainer(); + private static GenericContainer startRedisTestContainer() { + @SuppressWarnings("resource") + GenericContainer redis = + new GenericContainer<>(DockerImageName.parse("redis:5.0.3-alpine")).withExposedPorts(redisContainerPort); + redis.start(); + return redis; + } /** MockWebServer needs to be static, so we can inject its port into the Spring context. */ private static MockWebServer mockPangeanic = startPangeanicMockServer(); @@ -130,7 +143,7 @@ static void setProperties(DynamicPropertyRegistry registry) { registry.add("translation.google.projectId", () -> "project-id-test"); registry.add("translation.google.usehttpclient", () -> "true"); - registry.add("redis.connection.url", () -> "redis://localhost:" + redisPort + "/"); + registry.add("redis.connection.url", () -> "redis://"+ redis_container.getHost() + ":" + redis_container.getMappedPort(redisContainerPort).toString() + "/"); registry.add("translation.eTranslation.baseUrl", () -> ETranslationTranslationService.FAKE_BASE_URL_FOR_TESTING); registry.add("translation.eTranslation.credentials", () -> ""); registry.add("translation.eTranslation.truncate", () -> false); diff --git a/translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/web/TranslationRestIT.java b/translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/web/TranslationRestIT.java index 0e3cd6ea..1320f953 100644 --- a/translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/web/TranslationRestIT.java +++ b/translation-tests/src/integration-test/java/eu/europeana/api/translation/tests/web/TranslationRestIT.java @@ -12,7 +12,6 @@ import java.util.List; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; @@ -37,7 +36,6 @@ import eu.europeana.api.translation.tests.web.mock.MockGClient; import eu.europeana.api.translation.tests.web.mock.MockGServiceStub; import eu.europeana.api.translation.web.service.RedisCacheService; -import redis.embedded.RedisServer; @SpringBootTest @AutoConfigureMockMvc @@ -49,9 +47,7 @@ public class TranslationRestIT extends BaseTranslationTest { @Autowired RedisCacheService redisCacheService; - - private static RedisServer redisServer = startRedisService(); - + public static final String LANGUAGE_EN = "en"; @@ -65,21 +61,7 @@ void startMockServers() throws IOException { clientWrapper.setClient(googleClient); googleTranslationService.init(clientWrapper); } - - static RedisServer startRedisService() { - //start redis server - RedisServer redisServer = new RedisServer(redisPort); - redisServer.start(); - return redisServer; - } - - @AfterAll void stopRedis() { - if(redisServer != null) { - redisServer.stop(); - } - } - - + @Test void translationGoogle() throws Exception { String requestJson = getJsonStringInput(TRANSLATION_REQUEST); @@ -263,7 +245,7 @@ void translationPangeanicNoSrcMultipleLanguages() throws Exception { .accept(MediaType.APPLICATION_JSON) .contentType(MediaType.APPLICATION_JSON) .content(requestJson)) -// .andExpect(status().isOk()) + .andExpect(status().isOk()) .andReturn().getResponse() .getContentAsString(); diff --git a/translation-web/src/main/java/eu/europeana/api/translation/web/service/CachedTranslationService.java b/translation-web/src/main/java/eu/europeana/api/translation/web/service/CachedTranslationService.java index e82d0e2a..a1250b96 100644 --- a/translation-web/src/main/java/eu/europeana/api/translation/web/service/CachedTranslationService.java +++ b/translation-web/src/main/java/eu/europeana/api/translation/web/service/CachedTranslationService.java @@ -4,6 +4,7 @@ import javax.validation.constraints.NotNull; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import eu.europeana.api.translation.definitions.model.TranslationCachingStats; import eu.europeana.api.translation.definitions.model.TranslationObj; import eu.europeana.api.translation.service.AbstractTranslationService; import eu.europeana.api.translation.service.TranslationService; @@ -39,19 +40,48 @@ public boolean isSupported(String srcLang, String trgLang) { return true; } + private TranslationCachingStats computeTranslationCachingStats(List allTranslObjs) { + int numLinesCached=0; + int numCharsCached=0; + int numLinesToBeTranslated=0; + int numCharsToBeTranslated=0; + for(TranslationObj translObj : allTranslObjs) { + if(translObj.isRetrievedFromCache()) { + numLinesCached += 1; + numCharsCached += translObj.getTranslation().length(); + } + if(translObj.getTranslation() == null) { + //objects sent for translation + numLinesToBeTranslated += 1; + numCharsToBeTranslated += translObj.getText().length(); + } + } + + TranslationCachingStats stats = new TranslationCachingStats(); + stats.setNumLinesCached(numLinesCached); + stats.setNumCharsCached(numCharsCached); + stats.setNumLinesToBeTranslated(numLinesToBeTranslated); + stats.setNumCharsToBeTranslated(numCharsToBeTranslated); + return stats; + } + @Override public void translate(List translationObjs) throws TranslationException { //fill the non translatable texts, e.g. empty Strings - processNonTranslatable(translationObjs); - - - fillTranslationForSameLanguage(translationObjs); - + processNonTranslatable(translationObjs); if(isCachingEnabled()) { redisCacheService.fillWithCachedTranslations(translationObjs); } + //logging the number of translated/cached lines and chars + if(logger.isInfoEnabled()) { + TranslationCachingStats stats = computeTranslationCachingStats(translationObjs); + logger.info("Tracking cache usage: numLinesCached={}, numCharsCached={}, numLinesToBeTranslated={}, " + + "numCharsToBeTranslated={}", stats.getNumLinesCached(), stats.getNumCharsCached(), + stats.getNumLinesToBeTranslated(), stats.getNumCharsToBeTranslated()); + } + List toTranslate = translationObjs.stream().filter( t -> t.getTranslation() == null).toList(); @@ -62,18 +92,7 @@ public void translate(List translationObjs) throws TranslationEx redisCacheService.store(toTranslate); } } - - //logging the number of translated/cached lines and chars - int numLinesCached=(int) translationObjs.stream().filter(el -> el.isRetrievedFromCache()).count(); - int numCharsCached=translationObjs.stream().filter(el -> el.isRetrievedFromCache()).map(el -> el.getTranslation().length()).reduce(0, Integer::sum); - int numLinesTranslated=toTranslate.size(); - int numCharsTranslated=toTranslate.stream().map(el -> el.getText().length()).reduce(0, Integer::sum); - if(logger.isInfoEnabled()) { - logger.info("Tracking cache usage: numLinesCached={}, numCharsCached={}, numLinesTranslated={}, " - + "numCharsTranslated={}", numLinesCached, numCharsCached, numLinesTranslated, numCharsTranslated); - } - - + }