Skip to content

Commit

Permalink
feat: update translate
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhanshuyou
zhanshuyou committed Aug 19, 2024
1 parent 93d1984 commit 9b94691
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 28 deletions.
52 changes: 51 additions & 1 deletion package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 6 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,16 @@
"version": "1.0.0",
"description": "Welcome to Milvus documentation!",
"scripts": {
"translate": "node scripts/translate.js",
"translate": "node tools/translate.mjs",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"devDependencies": {
"remarkable": "^2.0.1"
"@zilliz/toolkit": "^0.0.9",
"highlight.js": "^11.10.0",
"katex": "^0.16.11",
"remarkable": "^2.0.1",
"remarkable-katex": "^1.2.1"
},
"license": "Apache-2.0"
}
28 changes: 14 additions & 14 deletions tools/cache.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"site/en/about/comparison.md": "2024-08-16T08:51:08.832Z",
"site/en/about/comparison.md": "2024-08-19T11:18:16.864Z",
"site/en/about/limitations.md": "2024-08-16T08:51:08.837Z",
"site/en/about/milvus_adopters.md": "2024-08-16T08:51:08.843Z",
"site/en/about/overview.md": "2024-08-16T08:51:08.852Z",
"site/en/about/roadmap.md": "2024-08-16T08:51:08.857Z",
"site/en/about/roadmap.md": "2024-08-19T11:18:16.874Z",
"site/en/adminGuide/allocate.md": "2024-08-16T08:51:08.864Z",
"site/en/adminGuide/authenticate.md": "2024-08-16T08:51:08.870Z",
"site/en/adminGuide/backup/h2m.md": "2024-08-16T08:51:08.874Z",
Expand All @@ -25,9 +25,9 @@
"site/en/adminGuide/configure-querynode-localdisk.md": "2024-08-16T08:51:08.957Z",
"site/en/adminGuide/configure_access_logs.md": "2024-08-16T08:51:08.960Z",
"site/en/adminGuide/configure_operator.md": "2024-08-16T08:51:08.965Z",
"site/en/adminGuide/deploy_etcd.md": "2024-08-16T08:51:08.980Z",
"site/en/adminGuide/deploy_pulsar.md": "2024-08-16T08:51:08.984Z",
"site/en/adminGuide/deploy_s3.md": "2024-08-16T08:51:08.995Z",
"site/en/adminGuide/deploy_etcd.md": "2024-08-19T11:18:16.882Z",
"site/en/adminGuide/deploy_pulsar.md": "2024-08-19T11:18:16.889Z",
"site/en/adminGuide/deploy_s3.md": "2024-08-19T11:18:16.902Z",
"site/en/adminGuide/dynamic_config.md": "2024-08-16T08:51:09.000Z",
"site/en/adminGuide/limit_collection_counts.md": "2024-08-16T08:51:09.004Z",
"site/en/adminGuide/m2m_legacy.md": "2024-08-16T08:51:09.008Z",
Expand Down Expand Up @@ -94,10 +94,10 @@
"site/en/getstarted/run-milvus-docker/install_standalone-docker-compose.md": "2024-08-16T08:51:09.217Z",
"site/en/getstarted/run-milvus-docker/install_standalone-docker.md": "2024-08-16T08:51:09.219Z",
"site/en/getstarted/run-milvus-docker/prerequisite-docker.md": "2024-08-16T08:51:09.221Z",
"site/en/getstarted/run-milvus-gpu/install_cluster-helm-gpu.md": "2024-08-16T08:51:09.223Z",
"site/en/getstarted/run-milvus-gpu/install_cluster-helm-gpu.md": "2024-08-19T11:18:16.917Z",
"site/en/getstarted/run-milvus-gpu/install_standalone-docker-compose-gpu.md": "2024-08-16T08:51:09.226Z",
"site/en/getstarted/run-milvus-gpu/prerequisite-gpu.md": "2024-08-16T08:51:09.228Z",
"site/en/getstarted/run-milvus-k8s/install_cluster-helm.md": "2024-08-16T08:51:09.231Z",
"site/en/getstarted/run-milvus-k8s/install_cluster-helm.md": "2024-08-19T11:18:16.926Z",
"site/en/getstarted/run-milvus-k8s/install_cluster-milvusoperator.md": "2024-08-16T08:51:09.244Z",
"site/en/getstarted/run-milvus-k8s/prerequisite-helm.md": "2024-08-16T08:51:09.249Z",
"site/en/getstarted/standalone/install_standalone-aptyum.md": "2024-08-16T08:51:09.251Z",
Expand Down Expand Up @@ -130,10 +130,10 @@
"site/en/integrations/langchain/basic_usage_langchain.md": "2024-08-16T08:51:09.318Z",
"site/en/integrations/langchain/integrate_with_langchain.md": "2024-08-16T08:51:09.320Z",
"site/en/integrations/langchain/milvus_hybrid_search_retriever.md": "2024-08-16T08:51:09.322Z",
"site/en/migrate/es2m.md": "2024-08-16T08:51:09.325Z",
"site/en/migrate/f2m.md": "2024-08-16T08:51:09.327Z",
"site/en/migrate/es2m.md": "2024-08-19T11:18:16.937Z",
"site/en/migrate/f2m.md": "2024-08-19T11:18:16.950Z",
"site/en/migrate/from-m2x.md": "2024-08-16T08:51:09.329Z",
"site/en/migrate/m2m.md": "2024-08-16T08:51:09.331Z",
"site/en/migrate/m2m.md": "2024-08-19T11:18:16.957Z",
"site/en/migrate/migrate_overview.md": "2024-08-16T08:51:09.333Z",
"site/en/migrate/milvusdm_install.md": "2024-08-16T08:51:09.335Z",
"site/en/reference/architecture/architecture.md": "2024-08-16T08:51:09.337Z",
Expand All @@ -147,7 +147,7 @@
"site/en/reference/boolean.md": "2024-08-16T08:51:09.355Z",
"site/en/reference/consistency.md": "2024-08-16T08:51:09.357Z",
"site/en/reference/coordinator_ha.md": "2024-08-16T08:51:09.360Z",
"site/en/reference/disk_index.md": "2024-08-16T08:51:09.362Z",
"site/en/reference/disk_index.md": "2024-08-19T11:18:16.962Z",
"site/en/reference/glossary.md": "2024-08-16T08:51:09.365Z",
"site/en/reference/gpu_index.md": "2024-08-16T08:51:09.367Z",
"site/en/reference/index.md": "2024-08-16T08:51:09.370Z",
Expand Down Expand Up @@ -207,16 +207,16 @@
"site/en/userGuide/enable-dynamic-field.md": "2024-08-16T08:51:09.493Z",
"site/en/userGuide/insert-update-delete.md": "2024-08-16T08:51:09.495Z",
"site/en/userGuide/load_balance.md": "2024-08-16T08:51:09.509Z",
"site/en/userGuide/manage-collections.md": "2024-08-16T08:51:09.510Z",
"site/en/userGuide/manage-collections.md": "2024-08-19T11:18:16.970Z",
"site/en/userGuide/manage-indexes/index-scalar-fields.md": "2024-08-16T08:51:09.513Z",
"site/en/userGuide/manage-indexes/index-vector-fields.md": "2024-08-16T08:51:09.514Z",
"site/en/userGuide/manage-indexes/index-with-gpu.md": "2024-08-16T08:51:09.518Z",
"site/en/userGuide/manage-partitions.md": "2024-08-16T08:51:09.520Z",
"site/en/userGuide/manage_databases.md": "2024-08-16T08:51:09.523Z",
"site/en/userGuide/search-query-get/get-and-scalar-query.md": "2024-08-16T08:51:09.526Z",
"site/en/userGuide/search-query-get/multi-vector-search.md": "2024-08-16T08:51:09.529Z",
"site/en/userGuide/search-query-get/single-vector-search.md": "2024-08-16T08:51:09.532Z",
"site/en/userGuide/search-query-get/with-iterators.md": "2024-08-16T08:51:09.534Z",
"site/en/userGuide/search-query-get/single-vector-search.md": "2024-08-19T11:18:16.981Z",
"site/en/userGuide/search-query-get/with-iterators.md": "2024-08-19T11:18:16.989Z",
"site/en/userGuide/tools/birdwatcher_install_guides.md": "2024-08-16T08:51:09.536Z",
"site/en/userGuide/tools/birdwatcher_overview.md": "2024-08-16T08:51:09.538Z",
"site/en/userGuide/tools/birdwatcher_usage_guides.md": "2024-08-16T08:51:09.540Z",
Expand Down
14 changes: 10 additions & 4 deletions tools/translate.js → tools/translate.mjs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
const fs = require("fs");
const { traverseDirectory, translate, mkdir } = require("./utils");
import fs from "fs";
import {
traverseDirectory,
translate,
mkdir,
renderDocHTML,
} from "./utils.mjs";

const targetDirectory = "./site/en";
const sourceLang = "en";
Expand All @@ -11,7 +16,7 @@ async function bootstrap() {
? JSON.parse(fs.readFileSync(cacheFile, "utf8") || "{}")
: {};

// get all md files in the site/en/ directory
// get all md files in the site/en/ directory
const mdFiles = traverseDirectory(targetDirectory);

console.log(`--> Found ${mdFiles.length} files...`);
Expand All @@ -25,10 +30,11 @@ async function bootstrap() {

for (let path of updatedFiles) {
const content = fs.readFileSync(path, "utf8");
const htmlContent = renderDocHTML(content);

for (let targetLang of targetLangs) {
const translateContent = await translate({
text: content,
text: htmlContent,
targetLang,
});
const targetFilePath = path.replace(sourceLang, targetLang);
Expand Down
28 changes: 21 additions & 7 deletions tools/utils.js → tools/utils.mjs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
const fs = require("fs");
const path = require("path");
import fs from "fs";
import path from "path";
import { Milvus } from "@zilliz/toolkit";

function traverseDirectory(dirPath, fileList = []) {
/**
* Each branch needs to be updated.
*/
const VERSION = "v2.4.x";
const PATH = "/docs/";

export function traverseDirectory(dirPath, fileList = []) {
const files = fs.readdirSync(dirPath);

files.forEach((file) => {
Expand All @@ -19,7 +26,7 @@ function traverseDirectory(dirPath, fileList = []) {
}

const dirCache = {};
function mkdir(filePath) {
export function mkdir(filePath) {
const pathArr = filePath.split("/");
let dir = pathArr[0];
for (let i = 1; i < pathArr.length; i++) {
Expand All @@ -31,10 +38,17 @@ function mkdir(filePath) {
}
}

async function translate(params) {
export function renderDocHTML(content) {
const { tree } = Milvus.md2html(content, {
showAnchor: true,
version: VERSION,
path: PATH,
});
return tree;
}

export async function translate(params) {
const { text, sourceLang = "EN", targetLang } = params;
// Translation logic
return text;
}

module.exports = { traverseDirectory, translate, mkdir };

0 comments on commit 9b94691

Please sign in to comment.