From 0ab242a97b8be95d5793a21cb71493ee0b17b031 Mon Sep 17 00:00:00 2001 From: Lubos Mjachky Date: Wed, 3 Jul 2024 17:27:43 +0200 Subject: [PATCH] Remove legacy docs left-overs from the CI [noissue] --- .github/template_gitref | 2 +- .github/workflows/nightly.yml | 52 ---- .github/workflows/publish.yml | 61 ----- .github/workflows/scripts/docs-publisher.py | 262 -------------------- .github/workflows/scripts/publish_docs.sh | 48 ---- .github/workflows/scripts/script.sh | 6 - doc_requirements.txt | 11 - docs/template_gitref | 2 +- template_config.yml | 6 +- 9 files changed, 3 insertions(+), 447 deletions(-) delete mode 100755 .github/workflows/scripts/docs-publisher.py delete mode 100755 .github/workflows/scripts/publish_docs.sh diff --git a/.github/template_gitref b/.github/template_gitref index 1d2b065e1..1ca899df2 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-349-g4925dbe +2021.08.26-350-g90b0185 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6fce25fc2..233819a11 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -65,55 +65,3 @@ jobs: branch: "changelog/update" delete-branch: true path: "pulp_container" - - publish: - runs-on: ubuntu-latest - needs: test - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_container" - - - uses: actions/download-artifact@v4 - with: - name: "plugin_package" - path: "pulp_container/dist/" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1' - echo ::endgroup:: - - - name: "Set environment variables" - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Download built docs - uses: actions/download-artifact@v4 - with: - name: "docs.tar" - path: "pulp_container" - - - name: Download Python client docs - uses: actions/download-artifact@v4 - with: - name: "python-client-docs.tar" - path: "pulp_container" - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar -C ./docs - .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e112f554b..be92f8d28 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -114,20 +114,6 @@ jobs: pulp_container/container-ruby-client.tar if-no-files-found: "error" overwrite: true - - name: Build docs - run: | - export DJANGO_SETTINGS_MODULE=pulpcore.app.settings - export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py - make -C docs/ PULP_URL="https://pulp" diagrams html - tar -cvf docs/docs.tar docs/_build - - - name: "Upload built docs" - uses: actions/upload-artifact@v4 - with: - name: "docs.tar" - path: "pulp_container/docs/docs.tar" - if-no-files-found: "error" - overwrite: true - name: "Logs" if: always() @@ -258,52 +244,6 @@ jobs: - name: "Publish client to rubygems" run: | bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} - publish-docs: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_container" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install 'packaging~=21.3' requests - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Download built docs" - uses: "actions/download-artifact@v4" - with: - name: "docs.tar" - path: "pulp_container/" - - - name: "Download Python client docs" - uses: "actions/download-artifact@v4" - with: - name: "python-client-docs.tar" - path: "pulp_container/" - - - name: "Publish docs to pulpproject.org" - run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }} create-gh-release: runs-on: "ubuntu-latest" @@ -312,7 +252,6 @@ jobs: - "publish-package" - "publish-python-bindings" - "publish-ruby-bindings" - - "publish-docs" steps: - name: "Create release on GitHub" diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py deleted file mode 100755 index 20584018a..000000000 --- a/.github/workflows/scripts/docs-publisher.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import subprocess -import os -import re -from shutil import rmtree -import tempfile -import requests -import json -from packaging import version - -WORKING_DIR = os.environ["WORKSPACE"] - -VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" -RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" - -USERNAME = "doc_builder_pulp_container" -HOSTNAME = "8.43.85.236" - -SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_container/" - - -def make_directory_with_rsync(remote_paths_list): - """ - Ensure the remote directory path exists. - - :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the - remote. - :type remote_paths_list: a list of strings, with each string representing a directory. - """ - try: - tempdir_path = tempfile.mkdtemp() - cwd = os.getcwd() - os.chdir(tempdir_path) - os.makedirs(os.sep.join(remote_paths_list)) - remote_path_arg = "%s@%s:%s%s" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - remote_paths_list[0], - ) - local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep - rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command) - if exit_code != 0: - raise RuntimeError("An error occurred while creating remote directories.") - finally: - rmtree(tempdir_path) - os.chdir(cwd) - - -def ensure_dir(target_dir, clean=True): - """ - Ensure that the directory specified exists and is empty. - - By default this will delete the directory if it already exists - - :param target_dir: The directory to process - :type target_dir: str - :param clean: Whether or not the directory should be removed and recreated - :type clean: bool - """ - if clean: - rmtree(target_dir, ignore_errors=True) - try: - os.makedirs(target_dir) - except OSError: - pass - - -def main(): - """ - Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--build-type", required=True, help="Build type: nightly, tag or changelog." - ) - parser.add_argument("--branch", required=True, help="Branch or tag name.") - opts = parser.parse_args() - - build_type = opts.build_type - - branch = opts.branch - - publish_at_root = False - - # rsync the docs - print("rsync the docs") - docs_directory = os.sep.join([WORKING_DIR, "docs"]) - local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep - if build_type == "nightly": - # This is a nightly build - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - make_directory_with_rsync(["en", branch, build_type]) - rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: - # Only publish docs at the root if this is the latest version - r = requests.get("https://pypi.org/pypi/pulp-container/json") - latest_version = version.parse(json.loads(r.text)["info"]["version"]) - docs_version = version.parse(branch) - # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. - # It doesn't prevent incorrect docs being published at root if 2 releases are done close - # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately - # after. - if docs_version >= latest_version: - publish_at_root = True - # Post releases should use the x.y.z part of the version string to form a path - if "post" in branch: - branch = ".".join(branch.split(".")[:-1]) - - # This is a GA build. - # publish to the root of docs.pulpproject.org - if publish_at_root: - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "en", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - # This is a pre-release - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "nightly", - "--exclude", - "testing", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "changelog": - if branch != "main": - raise RuntimeError("Can only publish CHANGELOG from main") - # Publish the CHANGELOG from main branch at the root directory - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - changelog_local_path_arg = os.path.join(local_path_arg, "changes.html") - rsync_command = [ - "rsync", - "-vzh", - "--omit-dir-times", - changelog_local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - raise RuntimeError("Build type must be either 'nightly', 'tag' or 'changelog'.") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh deleted file mode 100755 index 281042405..000000000 --- a/.github/workflows/scripts/publish_docs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -mkdir ~/.ssh -touch ~/.ssh/pulp-infra -chmod 600 ~/.ssh/pulp-infra -echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra - -echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts -chmod 644 ~/.ssh/known_hosts - -export PYTHONUNBUFFERED=1 -export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export WORKSPACE=$PWD - -# start the ssh agent -eval "$(ssh-agent -s)" -ssh-add ~/.ssh/pulp-infra - -python3 .github/workflows/scripts/docs-publisher.py --build-type "$1" --branch "$2" - -if [[ "$GITHUB_WORKFLOW" == "Container changelog update" ]]; then - # Do not build bindings docs on changelog update - exit -fi - -mkdir -p ../container-bindings -tar -xvf container-python-client-docs.tar --directory ../container-bindings -pushd ../container-bindings - -# publish to docs.pulpproject.org/pulp_container_client -rsync -avzh site/ doc_builder_pulp_container@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_container_client/ - -# publish to docs.pulpproject.org/pulp_container_client/en/{release} -rsync -avzh site/ doc_builder_pulp_container@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_container_client/en/"$2" -popd diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 43d64637f..30e02cab4 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -30,13 +30,7 @@ if [[ "$TEST" = "docs" ]]; then if [[ "$GITHUB_WORKFLOW" == "Container CI" ]]; then towncrier build --yes --version 4.0.0.ci fi - # Unified Docs Build pulp-docs build - # Legacy Docs Build - cd docs - make PULP_URL="$PULP_URL" diagrams html - tar -cvf docs.tar ./_build - cd .. exit fi diff --git a/doc_requirements.txt b/doc_requirements.txt index f41b19a40..598d5f9b8 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -7,15 +7,4 @@ -r requirements.txt towncrier -# Legacy docs -plantuml -sphinx~=7.1.2 -sphinx-rtd-theme==1.3.0 -sphinxcontrib-jquery -sphinxcontrib-openapi -mistune<4.0.0 -Jinja2<3.2 - -# Unified docs pulp-docs @ git+https://github.com/pulp/pulp-docs@main -# Extra requirements diff --git a/docs/template_gitref b/docs/template_gitref index 1d2b065e1..1ca899df2 100644 --- a/docs/template_gitref +++ b/docs/template_gitref @@ -1 +1 @@ -2021.08.26-349-g4925dbe +2021.08.26-350-g90b0185 diff --git a/template_config.yml b/template_config.yml index cff8021ed..8c85aa515 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-347-gc4a2504 +# generated with plugin_template@2021.08.26-350-g90b0185 api_root: /pulp/ black: true @@ -23,7 +23,6 @@ disabled_redis_runners: [] doc_requirements_from_pulpcore: false docker_fixtures: false docs_test: true -extra_docs_requirements: [] flake8: true flake8_ignore: [] github_org: pulp @@ -42,7 +41,6 @@ plugins: name: pulp_container post_job_template: null pre_job_template: null -publish_docs_to_pulpprojectdotorg: true pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} @@ -87,6 +85,4 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true -use_legacy_docs: true -use_unified_docs: true