bug fixes and optimized for crawler

browser.js

@@ -3,7 +3,7 @@ const { getTranslations, isBrowser, createLanguageSelect, setOptions } = require
 if (isBrowser()) {
   window.weployScriptTag = document.currentScript;
 
-  const translationCache = window.localStorage.getItem("translationCache");
+  const translationCache = window.localStorage.getItem("translationCachePerPage");
   try {
     const parsedTranslationCache = JSON.parse(translationCache);
     if (parsedTranslationCache && typeof parsedTranslationCache === "object") {
@@ -50,12 +50,6 @@ if (isBrowser()) {
     timeout: timeout
   }
 
-  // create language selector first
-  // if (createSelector) {
-  //   setOptions(apiKey, options);
-  //   createLanguageSelect(apiKey, { isInit : true });
-  // }
-
   document.addEventListener("DOMContentLoaded", function() {
     getTranslations(apiKey, options)
   });

index.js

@@ -9,6 +9,7 @@ const { debounce } = require('./utils/debounce.js');
 const extractTextNodes = require('./utils/translation/extractTextNodes.js');
 const getTranslationsFromAPI = require('./utils/translation/getTranslationsFromAPI.js');
 const { renderWeploySelectorState } = require('./utils/selector/renderWeploySelectorState.js');
+const getTranslationCacheFromCloudflare = require('./utils/translation/getTranslationCacheFromCloudflare.js');
 
 var isDomListenerAdded;
 
@@ -76,7 +77,7 @@ function processTextNodes(textNodes = [], language = "", apiKey = "") {
       reject("Original language is not translatable");
     })
   }
-  return new Promise(async (resolve, reject) => {
+  return new Promise(async (resolve) => {
     // Remove empty strings
     const cleanTextNodes = textNodes.filter(
       (textNode) =>
@@ -88,19 +89,24 @@ function processTextNodes(textNodes = [], language = "", apiKey = "") {
       window.translationCache = {}
     }
 
+    // Initialize cache per page if not exist yet
+    if (!window.translationCache[window.location.pathname]) {
+      window.translationCache[window.location.pathname] = {};
+    }
+
     // Initialize language cache if not exist yet
-    if (!window.translationCache[language]) {
-      window.translationCache[language] = {};
+    if (!window.translationCache[window.location.pathname][language]) {
+      window.translationCache[window.location.pathname][language] = {};
     }
 
     let notInCache = [];
 
     // Check cache for each textNode
     cleanTextNodes.forEach((node) => {
       const text = node.textContent;
-      const cacheValues = Object.values(window.translationCache[language] || {});
+      const cacheValues = Object.values(window.translationCache[window.location.pathname][language] || {});
       if (
-        !window.translationCache[language][text] // check in key
+        !window.translationCache[window.location.pathname][language][text] // check in key
         && !cacheValues.includes(text) // check in value (to handle nodes that already translated)
       ) {
         notInCache.push(text); // If not cached, add to notInCache array
@@ -111,43 +117,57 @@ function processTextNodes(textNodes = [], language = "", apiKey = "") {
       window.weployError = false;
       window.weployTranslating = true;
       renderWeploySelectorState({ shouldUpdateActiveLang: false });
+
+      const cacheFromCloudFlare = await getTranslationCacheFromCloudflare(language, apiKey);
+      window.translationCache[window.location.pathname][language] = {
+        ...(window.translationCache?.[window.location.pathname]?.[language] || {}),
+        ...cacheFromCloudFlare
+      }
+
+      const notCachedInCDN = notInCache.filter(text => !cacheFromCloudFlare[text]);
 
-      // If there are translations not in cache, fetch them from the API
-      getTranslationsFromAPI(notInCache, language, apiKey).then(
-        (response) => {
-          notInCache.forEach((text, index) => {
-            // Cache the new translations
-            window.translationCache[language][text] = response[index] || text;
-          });
-
-          // Update textNodes from the cache
-          cleanTextNodes.forEach((node) => {
-            const text = node.textContent;
-            if(window.translationCache[language][text]) {
-              // make sure text is still the same before replacing
-              if(node.textContent == text) {
-                node.textContent = window.translationCache[language][text];
-              }
+      try {
+        // If there are translations not in cache, fetch them from the API
+        const response = notCachedInCDN.length ? await getTranslationsFromAPI(notCachedInCDN, language, apiKey) : [];
+
+        notInCache.forEach((text, index) => {
+          // Cache the new translations
+          window.translationCache[window.location.pathname][language][text] = response[index] || cacheFromCloudFlare[text] || text;
+
+          // If the translation is not available, cache the original text
+          if (window.translationCache[window.location.pathname][language][text] == "weploy-untranslated") {
+            window.translationCache[window.location.pathname][language][text] = text;
+          }
+        });
+
+        // Update textNodes from the cache
+        cleanTextNodes.forEach((node) => {
+          const text = node.textContent;
+          if(window.translationCache[window.location.pathname][language][text]) {
+            // make sure text is still the same before replacing
+            if(node.textContent == text) {
+              node.textContent = window.translationCache[window.location.pathname][language][text];
             }
-          });
+          }
+        });
+
+        if (isBrowser()) window.localStorage.setItem("translationCachePerPage", JSON.stringify(window.translationCache));
 
-          if (isBrowser()) window.localStorage.setItem("translationCache", JSON.stringify(window.translationCache));
-          resolve(undefined);
-        }
-      ).catch(err => {
+        resolve(undefined);
+      } catch(err) {
         // console.error(err); // Log the error and resolve the promise without changing textNodes
         resolve(undefined);
-      });
+      }
     } else {
       // If all translations are cached, directly update textNodes from cache
       cleanTextNodes.forEach((node) => {
         const text = node.textContent;
-        if(window.translationCache[language][text]) {
-          node.textContent = window.translationCache[language][text];
+        if(window.translationCache[window.location.pathname][language][text]) {
+          node.textContent = window.translationCache[window.location.pathname][language][text];
         }
       });
 
-      if (isBrowser()) window.localStorage.setItem("translationCache", JSON.stringify(window.translationCache));
+      if (isBrowser()) window.localStorage.setItem("translationCachePerPage", JSON.stringify(window.translationCache));
       resolve(undefined);
     }
   });
@@ -172,7 +192,7 @@ function modifyHtmlStrings(rootElement, language, apiKey) {
 async function startTranslationCycle(node, apiKey, delay) {
   const lang = await getLanguageFromLocalStorage();
 
-  return new Promise(async (resolve, reject) => {
+  return new Promise(async (resolve) => {
     if (!delay) {
       await modifyHtmlStrings(node, lang, apiKey).catch(console.log)
       resolve(undefined)
@@ -201,6 +221,7 @@ function getDefinedLanguages(originalLanguage, allowedLanguages = []) {
 
 function setOptions(apiKey, optsArgs) {
   const mappedOpts = {
+    ...optsArgs,
     timeout: optsArgs.timeout == null ? 0 : optsArgs.timeout,
     pathOptions: optsArgs.pathOptions || {},
     apiKey,
@@ -209,11 +230,10 @@ function setOptions(apiKey, optsArgs) {
   }
 
   setWeployOptions(mappedOpts)
-  setWeployActiveLang(mappedOpts?.definedLanguages?.[0]?.lang)
+  // setWeployActiveLang(mappedOpts?.definedLanguages?.[0]?.lang)
 }
 
 async function getTranslations(apiKey, optsArgs = {}) {
-
   try {
     setOptions(apiKey, optsArgs)
 
@@ -278,6 +298,7 @@ async function getTranslations(apiKey, optsArgs = {}) {
 //@deprecated
 function switchLanguage(language) {
   localStorage.setItem("language", language);
+  setWeployActiveLang(language);
   const updatedUrl = addOrReplaceLangParam(window.location.href, language);
   setTimeout(() => {
     window.location.href = updatedUrl;

utils/compressions.js

@@ -17,7 +17,48 @@ function decompressArrayBuffer(byteArray, encoding) {
   });
 }
 
+async function compressToString(inputString, encoding) {
+  try {
+    // Compress the input string to an ArrayBuffer
+    const compressedBuffer = await compressToArrayBuffer(inputString, encoding);
+
+    // Convert the ArrayBuffer to a base64-encoded string
+    const base64String = btoa(String.fromCharCode(...new Uint8Array(compressedBuffer)));
+
+    return base64String;
+  } catch (error) {
+    console.error("Error compressing string:", error);
+    throw error;
+  }
+}
+
+function base64ToArrayBuffer(base64) {
+  var binaryString = window.atob(base64);
+  var len = binaryString.length;
+  var bytes = new Uint8Array(len);
+  for (var i = 0; i < len; i++) {
+      bytes[i] = binaryString.charCodeAt(i);
+  }
+  return bytes.buffer;
+}
+
+async function decompressString(base64String, encoding) {
+  try {
+    // Decode the base64-encoded string to an ArrayBuffer
+    const compressedBuffer = base64ToArrayBuffer(base64String)
+
+    // Decompress the ArrayBuffer
+    const decompressed = await decompressArrayBuffer(compressedBuffer, encoding);
+    return decompressed;
+  } catch (error) {
+    console.error("Error decompressing string:", error);
+    throw error;
+  }
+}
+
 module.exports = {
   compressToArrayBuffer,
   decompressArrayBuffer,
+  compressToString,
+  decompressString
 };

utils/configs.js

@@ -1,6 +1,10 @@
+const detectRobot = require("./detectRobot")
+
 // check if code runs on server or client
 const isBrowser = () => typeof window !== 'undefined'
 const API_URL = "https://api.tasksource.io"
+const CDN_URL = ""
+const KV_URL = "https://cdn.weploy-d8b.workers.dev"
 const SHOULD_COMPRESS_PAYLOAD = true
 
 /** Translation Options */
@@ -11,6 +15,18 @@ function getWeployOptions() {
     if (!window.weployOptions) {
       setWeployOptions({})
     }
+
+    const userAgent = window.navigator.userAgent;
+    const isRobot = detectRobot(userAgent);
+
+    if (isRobot) {
+      setWeployOptions({
+        useBrowserLanguage: false,
+        isRobot: true
+      })
+      return window.weployOptions;
+    }
+
     return window.weployOptions;
   } else {
     if (!weployOptions) {
@@ -24,8 +40,9 @@ function setWeployOptions(value = {}) {
   if (isBrowser()) {
     window.weployOptions = {
       ...(window.weployOptions || {}),
-      ...value
+      ...value,
     };
+
   } else {
     weployOptions = {
       ...(weployOptions || {}),
@@ -66,6 +83,8 @@ module.exports = {
   getWeployActiveLang,
   setWeployActiveLang,
   API_URL,
+  CDN_URL,
+  KV_URL,
   SHOULD_COMPRESS_PAYLOAD,
   weployOptions
 }

utils/detectRobot.js

@@ -0,0 +1,18 @@
+const detectRobot = (userAgent) => {
+  const robots = new RegExp([
+    /bot/,/spider/,/crawl/,                            // GENERAL TERMS
+    /APIs-Google/,/AdsBot/,/Googlebot/,                // GOOGLE ROBOTS
+    /mediapartners/,/Google Favicon/,
+    /FeedFetcher/,/Google-Read-Aloud/,
+    /DuplexWeb-Google/,/googleweblight/,
+    /bing/,/yandex/,/baidu/,/duckduck/,/yahoo/,        // OTHER ENGINES
+    /ecosia/,/ia_archiver/,
+    /facebook/,/instagram/,/pinterest/,/reddit/,       // SOCIAL MEDIA
+    /slack/,/twitter/,/whatsapp/,/youtube/,
+    /semrush/,                                         // OTHER
+  ].map((r) => r.source).join("|"),"i");               // BUILD REGEXP + "i" FLAG
+
+  return robots.test(userAgent);
+};
+
+module.exports = detectRobot;

utils/languages/getSelectedLanguage.js

@@ -1,6 +1,7 @@
-const { getWeployOptions } = require("../configs");
+const { getWeployOptions, setWeployActiveLang } = require("../configs");
 const { fetchLanguageList } = require("./fetchLanguageList");
 
+//@deprecated
 function getSelectedLanguage() {
   return new Promise((resolve, reject) => {
     const search = window.location.search;
@@ -10,6 +11,7 @@ function getSelectedLanguage() {
 
     if (paramsLang && (paramsLang != localStorageLang)) {
       localStorage.setItem("language", paramsLang);
+      setWeployActiveLang(paramsLang);
     }
 
     let language = paramsLang || localStorageLang;
@@ -30,17 +32,24 @@ async function getLanguageFromLocalStorage() {
 
   if (paramsLang && (paramsLang != localStorageLang)) {
     localStorage.setItem("language", paramsLang);
+    setWeployActiveLang(paramsLang);
+    return paramsLang;
   }
-  let language = paramsLang || localStorageLang;
-
+
   const availableLangs = await fetchLanguageList(apiKey);
+  let language = paramsLang || localStorageLang;
+
   if (!availableLangs.find(l => l.lang == language)) {
-    saveLanguageToLocalStorage(availableLangs, optsArgs.useBrowserLanguage);
+    saveDefaultLanguageToLocalStorage(availableLangs, optsArgs.useBrowserLanguage);
+  } else {
+    setWeployActiveLang(language);
   }
+
   return language; // Get the language from local storage
 }
 
-function saveLanguageToLocalStorage(availableLangs = [], useBrowserLang = true) {
+// this only for the first time when the user visits the site and the language is not set
+function saveDefaultLanguageToLocalStorage(availableLangs = [], useBrowserLang = true) {
   const language = window.navigator.language; // Get browser language (usually in this format: en-US)
   const langIsoCode = language && language.length >= 2 ? language.substring(0, 2) : null // Get the language ISO code
   const langInAvailableLangs = availableLangs.find(lang => lang.lang == langIsoCode) //  Check if the language is in the available languages
@@ -54,10 +63,11 @@ function saveLanguageToLocalStorage(availableLangs = [], useBrowserLang = true)
   const langToSave = useBrowserLang ? langInAvailableLangsOrFirst : availableLangs[0].lang // If useBrowserLang is true, use the language from the browser, otherwise use the first available language
   // Save the language to local storage
   localStorage.setItem("language", langToSave);
+  setWeployActiveLang(langToSave);
 }
 
 module.exports = {
   getSelectedLanguage,
   getLanguageFromLocalStorage,
-  saveLanguageToLocalStorage
+  saveDefaultLanguageToLocalStorage
 }