From 3d1d2bf11568aa4302ee0b70341b81e7418b1d3b Mon Sep 17 00:00:00 2001 From: Cody Littley <56973212+cody-littley@users.noreply.github.com> Date: Fri, 10 Jan 2025 14:04:39 -0600 Subject: [PATCH] Fix v2 doc collisions with v1. Make protoc multithreaded for doc generation. (#1096) Signed-off-by: Cody Littley --- .gitattributes | 6 + api/builder/generate-docs.sh | 107 ++-- api/docs/common.html | 109 ++-- api/docs/common.md | 63 +- api/docs/common_v2.html | 559 ++++++++++++++++++ api/docs/common_v2.md | 118 ++++ api/docs/eigenda-protos.html | 8 +- api/docs/eigenda-protos.md | 12 +- api/docs/retriever.html | 38 +- api/docs/retriever.md | 35 +- api/docs/retriever_v2.html | 493 +++++++++++++++ api/docs/retriever_v2.md | 105 ++++ .../v2/{common.pb.go => common_v2.pb.go} | 166 +++--- api/grpc/disperser/v2/disperser_v2.pb.go | 334 +++++------ api/grpc/node/v2/node_v2.pb.go | 104 ++-- .../{retriever.pb.go => retriever_v2.pb.go} | 116 ++-- ...ver_grpc.pb.go => retriever_v2_grpc.pb.go} | 4 +- .../v2/{common.proto => common_v2.proto} | 0 api/proto/disperser/v2/disperser_v2.proto | 2 +- api/proto/node/v2/node_v2.proto | 2 +- .../{retriever.proto => retriever_v2.proto} | 2 +- 21 files changed, 1832 insertions(+), 551 deletions(-) create mode 100644 .gitattributes create mode 100644 api/docs/common_v2.html create mode 100644 api/docs/common_v2.md create mode 100644 api/docs/retriever_v2.html create mode 100644 api/docs/retriever_v2.md rename api/grpc/common/v2/{common.pb.go => common_v2.pb.go} (59%) rename api/grpc/retriever/v2/{retriever.pb.go => retriever_v2.pb.go} (51%) rename api/grpc/retriever/v2/{retriever_grpc.pb.go => retriever_v2_grpc.pb.go} (97%) rename api/proto/common/v2/{common.proto => common_v2.proto} (100%) rename api/proto/retriever/v2/{retriever.proto => retriever_v2.proto} (97%) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..6133218a0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Auto-generated files should not be rendered in diffs. +api/docs/*.md linguist-generated=true +api/docs/*.html linguist-generated=true +*.pb.go linguist-generated=true +inabox/deploy/env_vars.go linguist-generated=true +# contracts/bindings/*.go linguist-generated=true Enable once bindings are checked in CI \ No newline at end of file diff --git a/api/builder/generate-docs.sh b/api/builder/generate-docs.sh index 011e0da37..85badf023 100755 --- a/api/builder/generate-docs.sh +++ b/api/builder/generate-docs.sh @@ -26,61 +26,92 @@ done # Sort the proto files alphabetically. Required for deterministic output. IFS=$'\n' PROTO_FILES=($(sort <<<"${PROTO_FILES[*]}")); unset IFS -# Generate unified HTML doc -echo "Generating unified HTML documentation..." -docker run --rm \ - -v "${DOCS_DIR}":/out \ - -v "${PROTO_DIR}":/protos \ - pseudomuto/protoc-gen-doc \ - "${PROTO_FILES[@]}" \ - --doc_opt=html,eigenda-protos.html 2>/dev/null - -if [ $? -ne 0 ]; then - echo "Failed to generate unified HTML documentation." - exit 1 -fi +PID_LIST=() -# Generate unified markdown doc -echo "Generating unified markdown documentation..." -docker run --rm \ - -v "${DOCS_DIR}":/out \ - -v "${PROTO_DIR}":/protos \ - pseudomuto/protoc-gen-doc \ - "${PROTO_FILES[@]}" \ - --doc_opt=markdown,eigenda-protos.md 2>/dev/null - -if [ $? -ne 0 ]; then - echo "Failed to generate unified markdown documentation." - exit 1 -fi +# Generate unified HTML doc +generateUnifiedHTML() { + echo "Generating unified HTML documentation..." + docker run --rm \ + -v "${DOCS_DIR}":/out \ + -v "${PROTO_DIR}":/protos \ + pseudomuto/protoc-gen-doc \ + "${PROTO_FILES[@]}" \ + --doc_opt=html,eigenda-protos.html 2>/dev/null & -# Generate individual markdown/HTML docs -for PROTO_FILE in "${PROTO_FILES[@]}"; do - PROTO_NAME=$(basename "${PROTO_FILE}" .proto) + if [ $? -ne 0 ]; then + echo "Failed to generate unified HTML documentation." + exit 1 + fi +} +generateUnifiedHTML & +PID_LIST+=($!) - echo "Generating markdown documentation for ${PROTO_NAME}..." +# Generate unified markdown doc +generateUnifiedMD() { + echo "Generating unified markdown documentation..." docker run --rm \ -v "${DOCS_DIR}":/out \ -v "${PROTO_DIR}":/protos \ pseudomuto/protoc-gen-doc \ - "${PROTO_FILE}" \ - --doc_opt=markdown,"${PROTO_NAME}.md" 2>/dev/null + "${PROTO_FILES[@]}" \ + --doc_opt=markdown,eigenda-protos.md 2>/dev/null if [ $? -ne 0 ]; then - echo "Failed to generate documentation for ${PROTO_NAME}." + echo "Failed to generate unified markdown documentation." exit 1 fi +} +generateUnifiedMD & +PID_LIST+=($!) - echo "Generating HTML documentation for ${PROTO_NAME}..." +# Generate markdown docs for a proto file. First argument is the name of the proto file, second is the path. +generateMD() { + echo "Generating markdown documentation for ${1}..." docker run --rm \ -v "${DOCS_DIR}":/out \ -v "${PROTO_DIR}":/protos \ pseudomuto/protoc-gen-doc \ - "${PROTO_FILE}" \ - --doc_opt=html,"${PROTO_NAME}.html" 2>/dev/null + "${2}" \ + --doc_opt=markdown,"${1}.md" 2>/dev/null if [ $? -ne 0 ]; then - echo "Failed to generate documentation for ${PROTO_NAME}." + echo "Failed to generate documentation for ${1}." exit 1 fi -done \ No newline at end of file +} + +# Generate HTML docs for a proto file. First argument is the name of the proto file, second is the path. +generateHTML() { + echo "Generating HTML documentation for ${1}..." + docker run --rm \ + -v "${DOCS_DIR}":/out \ + -v "${PROTO_DIR}":/protos \ + pseudomuto/protoc-gen-doc \ + "${2}" \ + --doc_opt=html,"${1}.html" 2>/dev/null + + if [ $? -ne 0 ]; then + echo "Failed to generate documentation for ${1}." + exit 1 + fi +} + +# Generate individual markdown/HTML docs +for PROTO_FILE in "${PROTO_FILES[@]}"; do + PROTO_NAME=$(basename "${PROTO_FILE}" .proto) + + generateMD "${PROTO_NAME}" "${PROTO_FILE}" & + PID_LIST+=($!) + + generateHTML "${PROTO_NAME}" "${PROTO_FILE}" & + PID_LIST+=($!) +done + +# Wait for all processes to finish +for PID in "${PID_LIST[@]}"; do + wait $PID + if [ $? -ne 0 ]; then + echo "Failed to wait for process $PID." + exit 1 + fi +done diff --git a/api/docs/common.html b/api/docs/common.html index 1c47ca8d1..522008e9f 100644 --- a/api/docs/common.html +++ b/api/docs/common.html @@ -175,23 +175,19 @@

Table of Contents

  • - common/v2/common.proto + common/common.proto