From 9b946918221903eaab38103b5e9393cc9a490a6d Mon Sep 17 00:00:00 2001 From: Shuyou Date: Mon, 19 Aug 2024 19:29:27 +0800 Subject: [PATCH] feat: update translate --- package-lock.json | 52 ++++++++++++++++++++++++++- package.json | 8 +++-- tools/cache.json | 28 +++++++-------- tools/{translate.js => translate.mjs} | 14 +++++--- tools/{utils.js => utils.mjs} | 28 +++++++++++---- 5 files changed, 102 insertions(+), 28 deletions(-) rename tools/{translate.js => translate.mjs} (85%) rename tools/{utils.js => utils.mjs} (59%) diff --git a/package-lock.json b/package-lock.json index deaf4b2b5..2573df0ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,9 +9,19 @@ "version": "1.0.0", "license": "Apache-2.0", "devDependencies": { - "remarkable": "^2.0.1" + "@zilliz/toolkit": "^0.0.9", + "highlight.js": "^11.10.0", + "katex": "^0.16.11", + "remarkable": "^2.0.1", + "remarkable-katex": "^1.2.1" } }, + "node_modules/@zilliz/toolkit": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/@zilliz/toolkit/-/toolkit-0.0.9.tgz", + "integrity": "sha512-6+NwTS2R94lOOlLx6WyC3vrvSCyYomBDCD+j3kJPyc23+oK223Efk9ASHTE0tAvZafWFExmTf9VPlWKd1hXePg==", + "dev": true + }, "node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -30,6 +40,40 @@ "tslib": "^2.3.0" } }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "dev": true, + "engines": { + "node": ">= 12" + } + }, + "node_modules/highlight.js": { + "version": "11.10.0", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.10.0.tgz", + "integrity": "sha512-SYVnVFswQER+zu1laSya563s+F8VDGt7o35d4utbamowvUNLLMovFqwCLSocpZTz3MgaSRA1IbqRWZv97dtErQ==", + "dev": true, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/katex": { + "version": "0.16.11", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.11.tgz", + "integrity": "sha512-RQrI8rlHY92OLf3rho/Ts8i/XvjgguEjOkO1BEXcU3N8BqPpSzBNwV/G0Ukr+P/l3ivvJUE/Fa/CwbS6HesGNQ==", + "dev": true, + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, "node_modules/remarkable": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/remarkable/-/remarkable-2.0.1.tgz", @@ -46,6 +90,12 @@ "node": ">= 6.0.0" } }, + "node_modules/remarkable-katex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/remarkable-katex/-/remarkable-katex-1.2.1.tgz", + "integrity": "sha512-Y1VquJBZnaVsfsVcKW2hmjT+pDL7mp8l5WAVlvuvViltrdok2m1AIKmJv8SsH+mBY84PoMw67t3kTWw1dIm8+g==", + "dev": true + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/package.json b/package.json index 98fbd1e07..070d25206 100644 --- a/package.json +++ b/package.json @@ -3,12 +3,16 @@ "version": "1.0.0", "description": "Welcome to Milvus documentation!", "scripts": { - "translate": "node scripts/translate.js", + "translate": "node tools/translate.mjs", "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", "devDependencies": { - "remarkable": "^2.0.1" + "@zilliz/toolkit": "^0.0.9", + "highlight.js": "^11.10.0", + "katex": "^0.16.11", + "remarkable": "^2.0.1", + "remarkable-katex": "^1.2.1" }, "license": "Apache-2.0" } diff --git a/tools/cache.json b/tools/cache.json index b3bb8d617..2b19746d1 100644 --- a/tools/cache.json +++ b/tools/cache.json @@ -1,9 +1,9 @@ { - "site/en/about/comparison.md": "2024-08-16T08:51:08.832Z", + "site/en/about/comparison.md": "2024-08-19T11:18:16.864Z", "site/en/about/limitations.md": "2024-08-16T08:51:08.837Z", "site/en/about/milvus_adopters.md": "2024-08-16T08:51:08.843Z", "site/en/about/overview.md": "2024-08-16T08:51:08.852Z", - "site/en/about/roadmap.md": "2024-08-16T08:51:08.857Z", + "site/en/about/roadmap.md": "2024-08-19T11:18:16.874Z", "site/en/adminGuide/allocate.md": "2024-08-16T08:51:08.864Z", "site/en/adminGuide/authenticate.md": "2024-08-16T08:51:08.870Z", "site/en/adminGuide/backup/h2m.md": "2024-08-16T08:51:08.874Z", @@ -25,9 +25,9 @@ "site/en/adminGuide/configure-querynode-localdisk.md": "2024-08-16T08:51:08.957Z", "site/en/adminGuide/configure_access_logs.md": "2024-08-16T08:51:08.960Z", "site/en/adminGuide/configure_operator.md": "2024-08-16T08:51:08.965Z", - "site/en/adminGuide/deploy_etcd.md": "2024-08-16T08:51:08.980Z", - "site/en/adminGuide/deploy_pulsar.md": "2024-08-16T08:51:08.984Z", - "site/en/adminGuide/deploy_s3.md": "2024-08-16T08:51:08.995Z", + "site/en/adminGuide/deploy_etcd.md": "2024-08-19T11:18:16.882Z", + "site/en/adminGuide/deploy_pulsar.md": "2024-08-19T11:18:16.889Z", + "site/en/adminGuide/deploy_s3.md": "2024-08-19T11:18:16.902Z", "site/en/adminGuide/dynamic_config.md": "2024-08-16T08:51:09.000Z", "site/en/adminGuide/limit_collection_counts.md": "2024-08-16T08:51:09.004Z", "site/en/adminGuide/m2m_legacy.md": "2024-08-16T08:51:09.008Z", @@ -94,10 +94,10 @@ "site/en/getstarted/run-milvus-docker/install_standalone-docker-compose.md": "2024-08-16T08:51:09.217Z", "site/en/getstarted/run-milvus-docker/install_standalone-docker.md": "2024-08-16T08:51:09.219Z", "site/en/getstarted/run-milvus-docker/prerequisite-docker.md": "2024-08-16T08:51:09.221Z", - "site/en/getstarted/run-milvus-gpu/install_cluster-helm-gpu.md": "2024-08-16T08:51:09.223Z", + "site/en/getstarted/run-milvus-gpu/install_cluster-helm-gpu.md": "2024-08-19T11:18:16.917Z", "site/en/getstarted/run-milvus-gpu/install_standalone-docker-compose-gpu.md": "2024-08-16T08:51:09.226Z", "site/en/getstarted/run-milvus-gpu/prerequisite-gpu.md": "2024-08-16T08:51:09.228Z", - "site/en/getstarted/run-milvus-k8s/install_cluster-helm.md": "2024-08-16T08:51:09.231Z", + "site/en/getstarted/run-milvus-k8s/install_cluster-helm.md": "2024-08-19T11:18:16.926Z", "site/en/getstarted/run-milvus-k8s/install_cluster-milvusoperator.md": "2024-08-16T08:51:09.244Z", "site/en/getstarted/run-milvus-k8s/prerequisite-helm.md": "2024-08-16T08:51:09.249Z", "site/en/getstarted/standalone/install_standalone-aptyum.md": "2024-08-16T08:51:09.251Z", @@ -130,10 +130,10 @@ "site/en/integrations/langchain/basic_usage_langchain.md": "2024-08-16T08:51:09.318Z", "site/en/integrations/langchain/integrate_with_langchain.md": "2024-08-16T08:51:09.320Z", "site/en/integrations/langchain/milvus_hybrid_search_retriever.md": "2024-08-16T08:51:09.322Z", - "site/en/migrate/es2m.md": "2024-08-16T08:51:09.325Z", - "site/en/migrate/f2m.md": "2024-08-16T08:51:09.327Z", + "site/en/migrate/es2m.md": "2024-08-19T11:18:16.937Z", + "site/en/migrate/f2m.md": "2024-08-19T11:18:16.950Z", "site/en/migrate/from-m2x.md": "2024-08-16T08:51:09.329Z", - "site/en/migrate/m2m.md": "2024-08-16T08:51:09.331Z", + "site/en/migrate/m2m.md": "2024-08-19T11:18:16.957Z", "site/en/migrate/migrate_overview.md": "2024-08-16T08:51:09.333Z", "site/en/migrate/milvusdm_install.md": "2024-08-16T08:51:09.335Z", "site/en/reference/architecture/architecture.md": "2024-08-16T08:51:09.337Z", @@ -147,7 +147,7 @@ "site/en/reference/boolean.md": "2024-08-16T08:51:09.355Z", "site/en/reference/consistency.md": "2024-08-16T08:51:09.357Z", "site/en/reference/coordinator_ha.md": "2024-08-16T08:51:09.360Z", - "site/en/reference/disk_index.md": "2024-08-16T08:51:09.362Z", + "site/en/reference/disk_index.md": "2024-08-19T11:18:16.962Z", "site/en/reference/glossary.md": "2024-08-16T08:51:09.365Z", "site/en/reference/gpu_index.md": "2024-08-16T08:51:09.367Z", "site/en/reference/index.md": "2024-08-16T08:51:09.370Z", @@ -207,7 +207,7 @@ "site/en/userGuide/enable-dynamic-field.md": "2024-08-16T08:51:09.493Z", "site/en/userGuide/insert-update-delete.md": "2024-08-16T08:51:09.495Z", "site/en/userGuide/load_balance.md": "2024-08-16T08:51:09.509Z", - "site/en/userGuide/manage-collections.md": "2024-08-16T08:51:09.510Z", + "site/en/userGuide/manage-collections.md": "2024-08-19T11:18:16.970Z", "site/en/userGuide/manage-indexes/index-scalar-fields.md": "2024-08-16T08:51:09.513Z", "site/en/userGuide/manage-indexes/index-vector-fields.md": "2024-08-16T08:51:09.514Z", "site/en/userGuide/manage-indexes/index-with-gpu.md": "2024-08-16T08:51:09.518Z", @@ -215,8 +215,8 @@ "site/en/userGuide/manage_databases.md": "2024-08-16T08:51:09.523Z", "site/en/userGuide/search-query-get/get-and-scalar-query.md": "2024-08-16T08:51:09.526Z", "site/en/userGuide/search-query-get/multi-vector-search.md": "2024-08-16T08:51:09.529Z", - "site/en/userGuide/search-query-get/single-vector-search.md": "2024-08-16T08:51:09.532Z", - "site/en/userGuide/search-query-get/with-iterators.md": "2024-08-16T08:51:09.534Z", + "site/en/userGuide/search-query-get/single-vector-search.md": "2024-08-19T11:18:16.981Z", + "site/en/userGuide/search-query-get/with-iterators.md": "2024-08-19T11:18:16.989Z", "site/en/userGuide/tools/birdwatcher_install_guides.md": "2024-08-16T08:51:09.536Z", "site/en/userGuide/tools/birdwatcher_overview.md": "2024-08-16T08:51:09.538Z", "site/en/userGuide/tools/birdwatcher_usage_guides.md": "2024-08-16T08:51:09.540Z", diff --git a/tools/translate.js b/tools/translate.mjs similarity index 85% rename from tools/translate.js rename to tools/translate.mjs index 2dfa42867..eba8e5912 100644 --- a/tools/translate.js +++ b/tools/translate.mjs @@ -1,5 +1,10 @@ -const fs = require("fs"); -const { traverseDirectory, translate, mkdir } = require("./utils"); +import fs from "fs"; +import { + traverseDirectory, + translate, + mkdir, + renderDocHTML, +} from "./utils.mjs"; const targetDirectory = "./site/en"; const sourceLang = "en"; @@ -11,7 +16,7 @@ async function bootstrap() { ? JSON.parse(fs.readFileSync(cacheFile, "utf8") || "{}") : {}; - // get all md files in the site/en/ directory + // get all md files in the site/en/ directory const mdFiles = traverseDirectory(targetDirectory); console.log(`--> Found ${mdFiles.length} files...`); @@ -25,10 +30,11 @@ async function bootstrap() { for (let path of updatedFiles) { const content = fs.readFileSync(path, "utf8"); + const htmlContent = renderDocHTML(content); for (let targetLang of targetLangs) { const translateContent = await translate({ - text: content, + text: htmlContent, targetLang, }); const targetFilePath = path.replace(sourceLang, targetLang); diff --git a/tools/utils.js b/tools/utils.mjs similarity index 59% rename from tools/utils.js rename to tools/utils.mjs index 0f6686c6b..d1b9f0138 100644 --- a/tools/utils.js +++ b/tools/utils.mjs @@ -1,7 +1,14 @@ -const fs = require("fs"); -const path = require("path"); +import fs from "fs"; +import path from "path"; +import { Milvus } from "@zilliz/toolkit"; -function traverseDirectory(dirPath, fileList = []) { +/** + * Each branch needs to be updated. + */ +const VERSION = "v2.4.x"; +const PATH = "/docs/"; + +export function traverseDirectory(dirPath, fileList = []) { const files = fs.readdirSync(dirPath); files.forEach((file) => { @@ -19,7 +26,7 @@ function traverseDirectory(dirPath, fileList = []) { } const dirCache = {}; -function mkdir(filePath) { +export function mkdir(filePath) { const pathArr = filePath.split("/"); let dir = pathArr[0]; for (let i = 1; i < pathArr.length; i++) { @@ -31,10 +38,17 @@ function mkdir(filePath) { } } -async function translate(params) { +export function renderDocHTML(content) { + const { tree } = Milvus.md2html(content, { + showAnchor: true, + version: VERSION, + path: PATH, + }); + return tree; +} + +export async function translate(params) { const { text, sourceLang = "EN", targetLang } = params; // Translation logic return text; } - -module.exports = { traverseDirectory, translate, mkdir };