diff --git a/browser.js b/browser.js index cf5511e..d80cf10 100644 --- a/browser.js +++ b/browser.js @@ -3,7 +3,7 @@ const { getTranslations, isBrowser, createLanguageSelect, setOptions } = require if (isBrowser()) { window.weployScriptTag = document.currentScript; - const translationCache = window.localStorage.getItem("translationCache"); + const translationCache = window.localStorage.getItem("translationCachePerPage"); try { const parsedTranslationCache = JSON.parse(translationCache); if (parsedTranslationCache && typeof parsedTranslationCache === "object") { @@ -50,12 +50,6 @@ if (isBrowser()) { timeout: timeout } - // create language selector first - // if (createSelector) { - // setOptions(apiKey, options); - // createLanguageSelect(apiKey, { isInit : true }); - // } - document.addEventListener("DOMContentLoaded", function() { getTranslations(apiKey, options) }); diff --git a/index.js b/index.js index 4ecdd7e..5614a71 100644 --- a/index.js +++ b/index.js @@ -9,6 +9,7 @@ const { debounce } = require('./utils/debounce.js'); const extractTextNodes = require('./utils/translation/extractTextNodes.js'); const getTranslationsFromAPI = require('./utils/translation/getTranslationsFromAPI.js'); const { renderWeploySelectorState } = require('./utils/selector/renderWeploySelectorState.js'); +const getTranslationCacheFromCloudflare = require('./utils/translation/getTranslationCacheFromCloudflare.js'); var isDomListenerAdded; @@ -76,7 +77,7 @@ function processTextNodes(textNodes = [], language = "", apiKey = "") { reject("Original language is not translatable"); }) } - return new Promise(async (resolve, reject) => { + return new Promise(async (resolve) => { // Remove empty strings const cleanTextNodes = textNodes.filter( (textNode) => @@ -88,9 +89,14 @@ function processTextNodes(textNodes = [], language = "", apiKey = "") { window.translationCache = {} } + // Initialize cache per page if not exist yet + if (!window.translationCache[window.location.pathname]) { + window.translationCache[window.location.pathname] = {}; + } + // Initialize language cache if not exist yet - if (!window.translationCache[language]) { - window.translationCache[language] = {}; + if (!window.translationCache[window.location.pathname][language]) { + window.translationCache[window.location.pathname][language] = {}; } let notInCache = []; @@ -98,9 +104,9 @@ function processTextNodes(textNodes = [], language = "", apiKey = "") { // Check cache for each textNode cleanTextNodes.forEach((node) => { const text = node.textContent; - const cacheValues = Object.values(window.translationCache[language] || {}); + const cacheValues = Object.values(window.translationCache[window.location.pathname][language] || {}); if ( - !window.translationCache[language][text] // check in key + !window.translationCache[window.location.pathname][language][text] // check in key && !cacheValues.includes(text) // check in value (to handle nodes that already translated) ) { notInCache.push(text); // If not cached, add to notInCache array @@ -111,43 +117,57 @@ function processTextNodes(textNodes = [], language = "", apiKey = "") { window.weployError = false; window.weployTranslating = true; renderWeploySelectorState({ shouldUpdateActiveLang: false }); + + const cacheFromCloudFlare = await getTranslationCacheFromCloudflare(language, apiKey); + window.translationCache[window.location.pathname][language] = { + ...(window.translationCache?.[window.location.pathname]?.[language] || {}), + ...cacheFromCloudFlare + } + + const notCachedInCDN = notInCache.filter(text => !cacheFromCloudFlare[text]); - // If there are translations not in cache, fetch them from the API - getTranslationsFromAPI(notInCache, language, apiKey).then( - (response) => { - notInCache.forEach((text, index) => { - // Cache the new translations - window.translationCache[language][text] = response[index] || text; - }); - - // Update textNodes from the cache - cleanTextNodes.forEach((node) => { - const text = node.textContent; - if(window.translationCache[language][text]) { - // make sure text is still the same before replacing - if(node.textContent == text) { - node.textContent = window.translationCache[language][text]; - } + try { + // If there are translations not in cache, fetch them from the API + const response = notCachedInCDN.length ? await getTranslationsFromAPI(notCachedInCDN, language, apiKey) : []; + + notInCache.forEach((text, index) => { + // Cache the new translations + window.translationCache[window.location.pathname][language][text] = response[index] || cacheFromCloudFlare[text] || text; + + // If the translation is not available, cache the original text + if (window.translationCache[window.location.pathname][language][text] == "weploy-untranslated") { + window.translationCache[window.location.pathname][language][text] = text; + } + }); + + // Update textNodes from the cache + cleanTextNodes.forEach((node) => { + const text = node.textContent; + if(window.translationCache[window.location.pathname][language][text]) { + // make sure text is still the same before replacing + if(node.textContent == text) { + node.textContent = window.translationCache[window.location.pathname][language][text]; } - }); + } + }); + + if (isBrowser()) window.localStorage.setItem("translationCachePerPage", JSON.stringify(window.translationCache)); - if (isBrowser()) window.localStorage.setItem("translationCache", JSON.stringify(window.translationCache)); - resolve(undefined); - } - ).catch(err => { + resolve(undefined); + } catch(err) { // console.error(err); // Log the error and resolve the promise without changing textNodes resolve(undefined); - }); + } } else { // If all translations are cached, directly update textNodes from cache cleanTextNodes.forEach((node) => { const text = node.textContent; - if(window.translationCache[language][text]) { - node.textContent = window.translationCache[language][text]; + if(window.translationCache[window.location.pathname][language][text]) { + node.textContent = window.translationCache[window.location.pathname][language][text]; } }); - if (isBrowser()) window.localStorage.setItem("translationCache", JSON.stringify(window.translationCache)); + if (isBrowser()) window.localStorage.setItem("translationCachePerPage", JSON.stringify(window.translationCache)); resolve(undefined); } }); @@ -172,7 +192,7 @@ function modifyHtmlStrings(rootElement, language, apiKey) { async function startTranslationCycle(node, apiKey, delay) { const lang = await getLanguageFromLocalStorage(); - return new Promise(async (resolve, reject) => { + return new Promise(async (resolve) => { if (!delay) { await modifyHtmlStrings(node, lang, apiKey).catch(console.log) resolve(undefined) @@ -201,6 +221,7 @@ function getDefinedLanguages(originalLanguage, allowedLanguages = []) { function setOptions(apiKey, optsArgs) { const mappedOpts = { + ...optsArgs, timeout: optsArgs.timeout == null ? 0 : optsArgs.timeout, pathOptions: optsArgs.pathOptions || {}, apiKey, @@ -209,11 +230,10 @@ function setOptions(apiKey, optsArgs) { } setWeployOptions(mappedOpts) - setWeployActiveLang(mappedOpts?.definedLanguages?.[0]?.lang) + // setWeployActiveLang(mappedOpts?.definedLanguages?.[0]?.lang) } async function getTranslations(apiKey, optsArgs = {}) { - try { setOptions(apiKey, optsArgs) @@ -278,6 +298,7 @@ async function getTranslations(apiKey, optsArgs = {}) { //@deprecated function switchLanguage(language) { localStorage.setItem("language", language); + setWeployActiveLang(language); const updatedUrl = addOrReplaceLangParam(window.location.href, language); setTimeout(() => { window.location.href = updatedUrl; diff --git a/utils/compressions.js b/utils/compressions.js index 22057e9..f92baf2 100644 --- a/utils/compressions.js +++ b/utils/compressions.js @@ -17,7 +17,48 @@ function decompressArrayBuffer(byteArray, encoding) { }); } +async function compressToString(inputString, encoding) { + try { + // Compress the input string to an ArrayBuffer + const compressedBuffer = await compressToArrayBuffer(inputString, encoding); + + // Convert the ArrayBuffer to a base64-encoded string + const base64String = btoa(String.fromCharCode(...new Uint8Array(compressedBuffer))); + + return base64String; + } catch (error) { + console.error("Error compressing string:", error); + throw error; + } +} + +function base64ToArrayBuffer(base64) { + var binaryString = window.atob(base64); + var len = binaryString.length; + var bytes = new Uint8Array(len); + for (var i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes.buffer; +} + +async function decompressString(base64String, encoding) { + try { + // Decode the base64-encoded string to an ArrayBuffer + const compressedBuffer = base64ToArrayBuffer(base64String) + + // Decompress the ArrayBuffer + const decompressed = await decompressArrayBuffer(compressedBuffer, encoding); + return decompressed; + } catch (error) { + console.error("Error decompressing string:", error); + throw error; + } +} + module.exports = { compressToArrayBuffer, decompressArrayBuffer, + compressToString, + decompressString }; diff --git a/utils/configs.js b/utils/configs.js index d70d6ec..1e82a48 100644 --- a/utils/configs.js +++ b/utils/configs.js @@ -1,6 +1,10 @@ +const detectRobot = require("./detectRobot") + // check if code runs on server or client const isBrowser = () => typeof window !== 'undefined' const API_URL = "https://api.tasksource.io" +const CDN_URL = "" +const KV_URL = "https://cdn.weploy-d8b.workers.dev" const SHOULD_COMPRESS_PAYLOAD = true /** Translation Options */ @@ -11,6 +15,18 @@ function getWeployOptions() { if (!window.weployOptions) { setWeployOptions({}) } + + const userAgent = window.navigator.userAgent; + const isRobot = detectRobot(userAgent); + + if (isRobot) { + setWeployOptions({ + useBrowserLanguage: false, + isRobot: true + }) + return window.weployOptions; + } + return window.weployOptions; } else { if (!weployOptions) { @@ -24,8 +40,9 @@ function setWeployOptions(value = {}) { if (isBrowser()) { window.weployOptions = { ...(window.weployOptions || {}), - ...value + ...value, }; + } else { weployOptions = { ...(weployOptions || {}), @@ -66,6 +83,8 @@ module.exports = { getWeployActiveLang, setWeployActiveLang, API_URL, + CDN_URL, + KV_URL, SHOULD_COMPRESS_PAYLOAD, weployOptions } diff --git a/utils/detectRobot.js b/utils/detectRobot.js new file mode 100644 index 0000000..b756ea8 --- /dev/null +++ b/utils/detectRobot.js @@ -0,0 +1,18 @@ +const detectRobot = (userAgent) => { + const robots = new RegExp([ + /bot/,/spider/,/crawl/, // GENERAL TERMS + /APIs-Google/,/AdsBot/,/Googlebot/, // GOOGLE ROBOTS + /mediapartners/,/Google Favicon/, + /FeedFetcher/,/Google-Read-Aloud/, + /DuplexWeb-Google/,/googleweblight/, + /bing/,/yandex/,/baidu/,/duckduck/,/yahoo/, // OTHER ENGINES + /ecosia/,/ia_archiver/, + /facebook/,/instagram/,/pinterest/,/reddit/, // SOCIAL MEDIA + /slack/,/twitter/,/whatsapp/,/youtube/, + /semrush/, // OTHER + ].map((r) => r.source).join("|"),"i"); // BUILD REGEXP + "i" FLAG + + return robots.test(userAgent); +}; + +module.exports = detectRobot; diff --git a/utils/languages/getSelectedLanguage.js b/utils/languages/getSelectedLanguage.js index 248adca..fe885f7 100644 --- a/utils/languages/getSelectedLanguage.js +++ b/utils/languages/getSelectedLanguage.js @@ -1,6 +1,7 @@ -const { getWeployOptions } = require("../configs"); +const { getWeployOptions, setWeployActiveLang } = require("../configs"); const { fetchLanguageList } = require("./fetchLanguageList"); +//@deprecated function getSelectedLanguage() { return new Promise((resolve, reject) => { const search = window.location.search; @@ -10,6 +11,7 @@ function getSelectedLanguage() { if (paramsLang && (paramsLang != localStorageLang)) { localStorage.setItem("language", paramsLang); + setWeployActiveLang(paramsLang); } let language = paramsLang || localStorageLang; @@ -30,17 +32,24 @@ async function getLanguageFromLocalStorage() { if (paramsLang && (paramsLang != localStorageLang)) { localStorage.setItem("language", paramsLang); + setWeployActiveLang(paramsLang); + return paramsLang; } - let language = paramsLang || localStorageLang; - + const availableLangs = await fetchLanguageList(apiKey); + let language = paramsLang || localStorageLang; + if (!availableLangs.find(l => l.lang == language)) { - saveLanguageToLocalStorage(availableLangs, optsArgs.useBrowserLanguage); + saveDefaultLanguageToLocalStorage(availableLangs, optsArgs.useBrowserLanguage); + } else { + setWeployActiveLang(language); } + return language; // Get the language from local storage } -function saveLanguageToLocalStorage(availableLangs = [], useBrowserLang = true) { +// this only for the first time when the user visits the site and the language is not set +function saveDefaultLanguageToLocalStorage(availableLangs = [], useBrowserLang = true) { const language = window.navigator.language; // Get browser language (usually in this format: en-US) const langIsoCode = language && language.length >= 2 ? language.substring(0, 2) : null // Get the language ISO code const langInAvailableLangs = availableLangs.find(lang => lang.lang == langIsoCode) // Check if the language is in the available languages @@ -54,10 +63,11 @@ function saveLanguageToLocalStorage(availableLangs = [], useBrowserLang = true) const langToSave = useBrowserLang ? langInAvailableLangsOrFirst : availableLangs[0].lang // If useBrowserLang is true, use the language from the browser, otherwise use the first available language // Save the language to local storage localStorage.setItem("language", langToSave); + setWeployActiveLang(langToSave); } module.exports = { getSelectedLanguage, getLanguageFromLocalStorage, - saveLanguageToLocalStorage + saveDefaultLanguageToLocalStorage } diff --git a/utils/translation/getTranslationCacheFromCDN.js b/utils/translation/getTranslationCacheFromCDN.js new file mode 100644 index 0000000..fa79cd7 --- /dev/null +++ b/utils/translation/getTranslationCacheFromCDN.js @@ -0,0 +1,51 @@ +const { decompressString } = require("../compressions"); +const { CDN_URL } = require("../configs"); + +async function getTranslationCacheFromCDN(language, apiKey) { + if (!language) { + throw new Error("WeployError: Missing language"); + } + + if (!apiKey) { + throw new Error("WeployError: Missing API Key"); + } + const langIso = language.substr(0, 2); + const cacheKey = `${apiKey}-${encodeURIComponent(window.location.pathname)}-${langIso}`; + + return await new Promise((resolve) => { + fetch(CDN_URL + `/weploy/get-translation-cache/${cacheKey}.html`, { + method: "GET", + headers: { + "weployskip": "yes" + }, + }) + .then((response) => response.text()) + .then((data) => { + const prefix = ``; + const isCacheCorrect = data.startsWith(prefix); + if (!isCacheCorrect) { + resolve({}); + return; + } + const prefixRemoved = data.replace(prefix, "") + const suffixRemoved = prefixRemoved.replace(``, "") + return suffixRemoved + }) + .then((str) => { + if (!str) { + resolve({}) + return; + } + return decompressString(str, 'gzip') + }) + .then(JSON.parse) + .then(resolve) + .catch((err) => { + // console.error(err); + // window.weployError = err.message; + resolve({}); + }) + }); +} + +module.exports = getTranslationCacheFromCDN; diff --git a/utils/translation/getTranslationCacheFromCloudflare.js b/utils/translation/getTranslationCacheFromCloudflare.js new file mode 100644 index 0000000..ba90711 --- /dev/null +++ b/utils/translation/getTranslationCacheFromCloudflare.js @@ -0,0 +1,17 @@ +const { isBrowser } = require("../configs"); +const getTranslationCacheFromCDN = require("./getTranslationCacheFromCDN"); +const getTranslationCacheFromKV = require("./getTranslationCacheFromKV"); + +async function getTranslationCacheFromCloudflare(language, apiKey) { + if (window?.cloudflareCache) return window?.cloudflareCache; + + const isUsingKV = true; + const cache = !isUsingKV ? + await getTranslationCacheFromCDN(language, apiKey).catch(() => ({})) : + await getTranslationCacheFromKV(language, apiKey).catch(() => ({})); + + if (isBrowser()) window.cloudflareCache = cache; + return cache; +} + +module.exports = getTranslationCacheFromCloudflare; diff --git a/utils/translation/getTranslationCacheFromKV.js b/utils/translation/getTranslationCacheFromKV.js new file mode 100644 index 0000000..1d0476c --- /dev/null +++ b/utils/translation/getTranslationCacheFromKV.js @@ -0,0 +1,41 @@ +const { decompressString } = require("../compressions"); +const { KV_URL } = require("../configs"); + +async function getTranslationCacheFromKV(language, apiKey) { + if (!language) { + throw new Error("WeployError: Missing language"); + } + + if (!apiKey) { + throw new Error("WeployError: Missing API Key"); + } + const langIso = language.substr(0, 2); + const cacheKey = `${apiKey}-${window.location.pathname}-${langIso}`; + + return await new Promise((resolve) => { + fetch(KV_URL, { + method: "GET", + headers: { + "cachekey": cacheKey, + "weployskip": "yes" + }, + }) + .then((r) => r.text()) + .then((str) => { + if (!str) { + resolve({}) + return; + } + return decompressString(str, 'gzip') + }) + .then(JSON.parse) + .then(resolve) + .catch((err) => { + // console.error(err); + // window.weployError = err.message; + resolve({}); + }) + }); +} + +module.exports = getTranslationCacheFromKV;