diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 7ae9f8d44..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[bumpversion] -current_version = 2.19.4.dev -commit = False -tag = False -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? -serialize = - {major}.{minor}.{patch}.{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = prod -first_value = dev -values = - dev - prod - -[bumpversion:file:./pulp_container/app/__init__.py] - -[bumpversion:file:./setup.py] - -[bumpversion:file:./docs/conf.py] diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 338ff6974..d57b44ed2 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -9,12 +9,18 @@ ADD ./{{ item.name }} ./{{ item.name }} # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads # Hacking botocore (https://github.com/boto/botocore/pull/1990) -RUN pip3 install +# This MUST be the ONLY call to pip install in inside the container. +RUN pip3 install --upgrade pip setuptools wheel && \ + rm -rf /root/.cache/pip && \ + pip3 install {%- if s3_test | default(false) -%} {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} {%- for item in plugins -%} {{ " " }}{{ item.source }} +{%- if item.upperbounds | default(false) -%} +{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt +{%- endif -%} {%- if item.lowerbounds | default(false) -%} {{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt {%- endif -%} @@ -22,6 +28,8 @@ RUN pip3 install {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ + rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 024df4e4c..4ed28f42a 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -49,7 +49,7 @@ AZURE_CONTAINER = "pulp-test" AZURE_LOCATION = "pulp3" AZURE_OVERWRITE_FILES = True AZURE_URL_EXPIRATION_SECS = 120 -AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;' +AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' {% endif %} {% if gcp_test | default(false) %} diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt new file mode 100644 index 000000000..2617a4089 --- /dev/null +++ b/.ci/assets/ci_constraints.txt @@ -0,0 +1,7 @@ +# Pulpcore versions without the openapi command do no longer work in the CI +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* + + + +tablib!=3.6.0 +# 3.6.0: This release introduced a regression removing the "html" optional dependency. diff --git a/.ci/assets/release_requirements.txt b/.ci/assets/release_requirements.txt index c064e9477..6635a8723 100644 --- a/.ci/assets/release_requirements.txt +++ b/.ci/assets/release_requirements.txt @@ -1,3 +1,3 @@ -bump2version +bump-my-version gitpython towncrier diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py new file mode 100755 index 000000000..4d4c7353a --- /dev/null +++ b/.ci/scripts/calc_constraints.py @@ -0,0 +1,137 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_container' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import fileinput +import urllib.request +import sys +from packaging.requirements import Requirement +from packaging.version import Version +import yaml + +try: + import tomllib +except ImportError: + import tomli as tomllib + + +CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" + + +def fetch_pulpcore_upper_bound(requirement): + with urllib.request.urlopen(CORE_TEMPLATE_URL) as f: + template = yaml.safe_load(f.read()) + supported_versions = template["supported_release_branches"] + supported_versions.append(template["latest_release_branch"]) + applicable_versions = sorted( + requirement.specifier.filter((Version(v) for v in supported_versions)) + ) + if len(applicable_versions) == 0: + raise Exception("No supported pulpcore version in required range.") + return f"{requirement.name}~={applicable_versions[-1]}" + + +def split_comment(line): + split_line = line.split("#", maxsplit=1) + try: + comment = " # " + split_line[1].strip() + except IndexError: + comment = "" + return split_line[0].strip(), comment + + +def to_upper_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + if requirement.name == "pulpcore": + # An exception to allow for pulpcore deprecation policy. + return fetch_pulpcore_upper_bound(requirement) + for spec in requirement.specifier: + if spec.operator == "~=": + return f"# NO BETTER CONSTRAINT: {req}" + if spec.operator == "<=": + operator = "==" + max_version = spec.version + return f"{requirement.name}{operator}{max_version}" + if spec.operator == "<": + operator = "~=" + version = Version(spec.version) + if version.micro != 0: + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" + elif version.minor != 0: + max_version = f"{version.major}.{version.minor - 1}" + elif version.major != 0: + max_version = f"{version.major - 1}.0" + else: + return f"# NO BETTER CONSTRAINT: {req}" + return f"{requirement.name}{operator}{max_version}" + return f"# NO UPPER BOUND: {req}" + + +def to_lower_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + for spec in requirement.specifier: + if spec.operator == ">=": + if requirement.name == "pulpcore": + # Currently an exception to allow for pulpcore bugfix releases. + # TODO Semver libraries should be allowed too. + operator = "~=" + else: + operator = "==" + min_version = spec.version + return f"{requirement.name}{operator}{min_version}" + return f"# NO LOWER BOUND: {req}" + + +def main(): + """Calculate constraints for the lower bound of dependencies where possible.""" + parser = argparse.ArgumentParser( + prog=sys.argv[0], + description="Calculate constraints for the lower or upper bound of dependencies where " + "possible.", + ) + parser.add_argument("-u", "--upper", action="store_true") + parser.add_argument("filename", nargs="*") + args = parser.parse_args() + + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) + else: + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/calc_deps_lowerbounds.py b/.ci/scripts/calc_deps_lowerbounds.py deleted file mode 100755 index 0dd1d43ae..000000000 --- a/.ci/scripts/calc_deps_lowerbounds.py +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -from packaging.requirements import Requirement - - -def main(): - """Calculate the lower bound of dependencies where possible.""" - with open("requirements.txt") as req_file: - for line in req_file: - try: - requirement = Requirement(line) - except ValueError: - print(line.strip()) - else: - for spec in requirement.specifier: - if spec.operator == ">=": - if requirement.name == "pulpcore": - operator = "~=" - else: - operator = "==" - min_version = str(spec)[2:] - print(f"{requirement.name}{operator}{min_version}") - break - else: - print(line.strip()) - - -if __name__ == "__main__": - main() diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 7775fed82..da45be6a8 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -1,28 +1,21 @@ #!/usr/bin/env python -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - import argparse import re import os +import tomllib import yaml +from pathlib import Path from tempfile import TemporaryDirectory from packaging.version import Version from git import Repo -UPSTREAM_REMOTE = "https://github.com/pulp/pulp_container.git" -DEFAULT_BRANCH = "main" RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"] Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"] -def main(): +def options(): """Check which branches need a release.""" parser = argparse.ArgumentParser() parser.add_argument( @@ -32,17 +25,60 @@ def main(): "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) - opts = parser.parse_args() + return parser.parse_args() + + +def template_config(): + # Assume this script lies in .ci/scripts + path = Path(__file__).absolute().parent.parent.parent / "template_config.yml" + return yaml.safe_load(path.read_text()) + + +def current_version(repo, commitish): + try: + pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml")) + try: + current_version = pyproject_toml["project"]["version"] + except Exception: + current_version = pyproject_toml["tool"]["bumpversion"]["current_version"] + except Exception: + current_version = repo.git.grep( + "current_version", commitish, "--", ".bumpversion.cfg" + ).split("=")[-1] + return Version(current_version) + + +def check_pyproject_dependencies(repo, from_commit, to_commit): + try: + old_pyproject = tomllib.load(repo.show("{from_commit}:pyproject.toml")) + old_dependencies = set(old_pyproject["project"]["dependencies"]) + new_pyproject = tomllib.load(repo.show("{to_commit}:pyproject.toml")) + new_dependencies = set(new_pyproject["project"]["dependencies"]) + if old_dependencies != new_dependencies: + return ["dependencies"] + else: + return [] + except Exception as e: + print(f"WARNING: Comparing the dependencies in pyproject.toml failed. ({e})") + # Gathering more details failed. + return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."] + +def main(options, template_config): with TemporaryDirectory() as d: # Clone from upstream to ensure we have updated branches & main + GITHUB_ORG = template_config["github_org"] + PLUGIN_NAME = template_config["plugin_name"] + UPSTREAM_REMOTE = f"https://github.com/{GITHUB_ORG}/{PLUGIN_NAME}.git" + DEFAULT_BRANCH = template_config["plugin_default_branch"] + repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none") heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")] available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] available_branches.sort(key=lambda ver: Version(ver)) available_branches.append(DEFAULT_BRANCH) - branches = opts.branches + branches = options.branches if branches == "supported": with open(f"{d}/template_config.yml", mode="r") as f: tc = yaml.safe_load(f) @@ -64,6 +100,7 @@ def main(): for branch in branches: if branch != DEFAULT_BRANCH: # Check if a Z release is needed + reasons = [] changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") z_changelog = False for change in changes.split("\n"): @@ -76,23 +113,32 @@ def main(): ) elif ext in Z_CHANGELOG_EXTS: z_changelog = True + if z_changelog: + reasons.append("Backports") last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}") req_txt_diff = repo.git.diff( f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" ) - if z_changelog or req_txt_diff: + if req_txt_diff: + reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.extend(check_pyproject_dependencies(repo, last_tag, f"origin/{branch}")) + + if reasons: curr_version = Version(last_tag) assert curr_version.base_version.startswith( branch ), "Current-version has to belong to the current branch!" next_version = Version(f"{branch}.{curr_version.micro + 1}") - reason = "CHANGES" if z_changelog else "requirements.txt" print( f"A Z-release is needed for {branch}, " f"Prev: {last_tag}, " f"Next: {next_version.base_version}, " - f"Reason: {reason}" + f"Reason: {','.join(reasons)}" ) releases.append(next_version) else: @@ -101,15 +147,10 @@ def main(): for change in changes.split("\n"): _, ext = os.path.splitext(change) if ext in Y_CHANGELOG_EXTS: - # We don't put Y release bumps in the commit message, check file instead - # The 'current_version' is always the next version to release - next_version = repo.git.grep( - "current_version", DEFAULT_BRANCH, "--", ".bumpversion.cfg" - ).split("=")[-1] - next_version = Version(next_version) - print( - f"A new Y-release is needed! New Version: {next_version.base_version}" - ) + # We don't put Y release bumps in the commit message, check file instead. + # The 'current_version' is always the dev of the next version to release. + next_version = current_version(repo, DEFAULT_BRANCH).base_version + print(f"A new Y-release is needed! New Version: {next_version}") releases.append(next_version) break @@ -118,4 +159,4 @@ def main(): if __name__ == "__main__": - main() + main(options(), template_config()) diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index cb9a2101f..c0c883a9d 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,11 +5,13 @@ # # For more info visit https://github.com/pulp/plugin_template +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -20,17 +22,33 @@ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -49,18 +67,21 @@ and req.name != "pulp-container-client" ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") - ops = [op for op, ver in req.specs] - spec = str(req.specs) + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) +if __name__ == "__main__": + main() diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 379d22ef4..c11557c11 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -1,3 +1,4 @@ +#!/bin/env python3 # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use @@ -8,13 +9,14 @@ import itertools import os import re +import tomllib -import toml from git import GitCommandError, Repo from packaging.version import parse as parse_version # Read Towncrier settings -tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"] +with open("pyproject.toml", "rb") as fp: + tc_settings = tomllib.load(fp)["tool"]["towncrier"] CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( @@ -79,7 +81,7 @@ def main(): old_length = len(main_changes) for branch in branches: - print(f"Looking at branch {branch}") + print(f"Looking for './{CHANGELOG_FILE}' at branch {branch}") try: changelog = get_changelog(repo, branch) except GitCommandError: @@ -101,7 +103,7 @@ def main(): for change in main_changes: fp.write(change[1]) - repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE) + repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE) if __name__ == "__main__": diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py new file mode 100755 index 000000000..0c478a212 --- /dev/null +++ b/.ci/scripts/pr_labels.py @@ -0,0 +1,60 @@ +#!/bin/env python3 + +# This script is running with elevated privileges from the main branch against pull requests. + +import re +import sys +import tomllib +from pathlib import Path + +from git import Repo + + +def main(): + assert len(sys.argv) == 3 + + with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) + BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT") + ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)") + CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$") + try: + CHANGELOG_EXTS = { + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + } + except KeyError: + CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"} + + repo = Repo(".") + + base_commit = repo.commit(sys.argv[1]) + head_commit = repo.commit(sys.argv[2]) + + pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}")) + + labels = { + "multi-commit": len(pr_commits) > 1, + "cherry-pick": False, + "no-issue": False, + "no-changelog": False, + "wip": False, + } + for commit in pr_commits: + labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None + no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None + labels["no-issue"] |= no_issue + cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None + labels["cherry-pick"] |= cherry_pick + changelog_snippets = [ + k + for k in commit.stats.files + if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS + ] + labels["no-changelog"] |= not changelog_snippets + + print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v))) + print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v))) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index a94b3e310..64c4f529e 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -9,21 +9,16 @@ import sys from pathlib import Path import subprocess - - import os import warnings from github import Github - -NO_ISSUE = "[noissue]" CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] +KEYWORDS = ["fixes", "closes"] + sha = sys.argv[1] message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - -KEYWORDS = ["fixes", "closes"] - g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp_container") @@ -64,15 +59,5 @@ def __check_changelog(issue): for issue in pattern.findall(message): __check_status(issue) __check_changelog(issue) -else: - if NO_ISSUE in message: - print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) - elif "Merge" in message and "cherry picked from commit" in message: - pass - else: - sys.exit( - "Error: no attached issues found for {sha}. If this was intentional, add " - " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) - ) print("Commit message for {sha} passed.".format(sha=sha[0:7])) diff --git a/.github/template_gitref b/.github/template_gitref index 2f00223c7..70eef8995 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-328-g59a6718 +2021.08.26-405-g3845bbc diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4009b915b..4cc5016f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,22 +23,91 @@ jobs: with: fetch-depth: 1 path: "pulp_container" - - uses: "actions/setup-python@v4" + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel + pip install build packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Build package" run: | - python3 setup.py sdist bdist_wheel --python-tag py3 + python3 -m build twine check dist/* + - name: "Install built packages" + run: | + pip install dist/pulp_container-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt + - name: "Generate api specs" + run: | + pulpcore-manager openapi --file "api.json" + pulpcore-manager openapi --bindings --component "container" --file "container-api.json" - name: "Upload Package whl" - uses: "actions/upload-artifact@v3" + uses: "actions/upload-artifact@v4" with: name: "plugin_package" path: "pulp_container/dist/" if-no-files-found: "error" retention-days: 5 + overwrite: true + - name: "Upload API specs" + uses: "actions/upload-artifact@v4" + with: + name: "api_spec" + path: | + pulp_container/api.json + pulp_container/container-api.json + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Python bindings packages" + run: | + .github/workflows/scripts/build_python_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v4" + with: + name: "python-client.tar" + path: | + pulp_container/container-python-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload python client docs" + uses: "actions/upload-artifact@v4" + with: + name: "python-client-docs.tar" + path: | + pulp_container/container-python-client-docs.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Ruby bindings packages" + run: | + .github/workflows/scripts/build_ruby_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload Ruby client" + uses: "actions/upload-artifact@v4" + with: + name: "ruby-client.tar" + path: | + pulp_container/container-ruby-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true +... diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml deleted file mode 100644 index f3532e4ce..000000000 --- a/.github/workflows/changelog.yml +++ /dev/null @@ -1,58 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - ---- -name: "Container changelog update" -on: - push: - branches: - - "main" - paths: - - "CHANGES.rst" - - "CHANGES.md" - workflow_dispatch: - -jobs: - - update-changelog: - runs-on: "ubuntu-latest" - strategy: - fail-fast: false - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - - - uses: "actions/setup-python@v4" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install -r doc_requirements.txt - echo ::endgroup:: - - - name: "Fake api schema" - run: | - mkdir -p docs/_build/html - echo "{}" > docs/_build/html/api.json - mkdir -p docs/_static - echo "{}" > docs/_static/api.json - - name: "Build Docs" - run: | - make diagrams html - working-directory: "./docs" - env: - PULP_CONTENT_ORIGIN: "http://localhost/" - - - name: "Publish changlog to pulpproject.org" - run: | - .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} - env: - PULP_DOCS_KEY: "${{ secrets.PULP_DOCS_KEY }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f52c0be94..7c561551f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: with: fetch-depth: 0 path: "pulp_container" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" @@ -42,9 +42,9 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" run: | .github/workflows/scripts/check_commit.sh - - name: "Verify requirements files" - run: | - python .ci/scripts/check_requirements.py + + docs: + uses: "./.github/workflows/docs.yml" lint: uses: "./.github/workflows/lint.yml" @@ -56,6 +56,9 @@ jobs: test: needs: "build" uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] deprecations: runs-on: "ubuntu-latest" @@ -67,10 +70,11 @@ jobs: mkdir -p "pulp_container" working-directory: "." - name: "Download Deprecations" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: "deprecations" + pattern: "deprecations-*" path: "pulp_container" + merge-multiple: true - name: "Print deprecations" run: | cat deprecations-*.txt | sort -u @@ -83,6 +87,7 @@ jobs: - "check-commits" - "lint" - "test" + - "docs" if: "always()" steps: - name: "Collect needed jobs results" diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 2dedc3b4a..0f51c4d14 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -26,14 +26,20 @@ jobs: fetch-depth: 0 path: "pulp_container" - - uses: "actions/setup-python@v4" + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/plugin_template" + path: "plugin_template" + + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version jinja2 pyyaml packaging + pip install bump-my-version packaging -r plugin_template/requirements.txt echo ::endgroup:: - name: "Setting secrets" @@ -48,7 +54,7 @@ jobs: run: | # Just to be sure... git checkout main - NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')" + NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')" if [ -z "$NEW_BRANCH" ] then echo Could not determine the new branch name. @@ -64,20 +70,13 @@ jobs: - name: Bump version on main branch working-directory: pulp_container run: | - bump2version --no-commit minor + bump-my-version bump --no-commit minor - name: Remove entries from CHANGES directory working-directory: pulp_container run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + - - name: Checkout plugin template - uses: actions/checkout@v4 - with: - repository: pulp/plugin_template - path: plugin_template - fetch-depth: 0 - - name: Update CI branches in template_config working-directory: plugin_template run: | @@ -94,10 +93,8 @@ jobs: branch: minor-version-bump base: main title: Bump minor version - body: '[noissue]' commit-message: | Bump minor version - [noissue] delete-branch: true - name: Push release branch diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..f23d6392c --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,54 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_container' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Docs" +on: + workflow_call: + +jobs: + test: + if: "endsWith(github.base_ref, 'main')" + runs-on: "ubuntu-20.04" + defaults: + run: + working-directory: "pulp_container" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_container" + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Setup cache key" + run: | + git ls-remote https://github.com/pulp/pulp-docs main | tee pulp-docs-main-sha + - uses: "actions/cache@v4" + with: + path: "~/.cache/pip" + key: ${{ runner.os }}-pip-${{ hashFiles('pulp-docs-main-sha') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install -r doc_requirements.txt + echo ::endgroup:: + - name: "Build changelog" + run: | + towncrier build --yes --version 4.0.0.ci + - name: "Build docs" + run: | + pulp-docs build + + no-test: + if: "!endsWith(github.base_ref, 'main')" + runs-on: "ubuntu-20.04" + steps: + - run: | + echo "Skip docs testing on non-main branches." diff --git a/.github/workflows/kanban.yml b/.github/workflows/kanban.yml deleted file mode 100644 index bc2356d38..000000000 --- a/.github/workflows/kanban.yml +++ /dev/null @@ -1,103 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -# Manage issues in a project board using https://github.com/leonsteinhaeuser/project-beta-automations - ---- -name: Kanban -on: - pull_request_target: - issues: - types: - - labeled - - reopened - - assigned - - closed - -env: - free_to_take: Free to take - in_progress: In Progress - needs_review: Needs review - done: Done - -jobs: - # only prio-list labeled items should be added to the board - add-to-project-board: - if: github.event_name == 'issues' && contains(github.event.issue.labels.*.name, 'prio-list') && contains(fromJson('["labeled", "reopened"]'), github.event.action) - runs-on: ubuntu-latest - steps: - - name: Add issue to Free-to-take list - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.free_to_take }} # Target status - - move-to-inprogress: - if: github.event_name == 'issues' && github.event.action == 'assigned' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the In Progress column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.in_progress }} # Target status - - find-linked-issues: - if: github.event_name == 'pull_request_target' - runs-on: ubuntu-latest - name: Find issues linked to a PR - outputs: - linked-issues: ${{ steps.linked-issues.outputs.issues }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Get Linked Issues Action - uses: kin/gh-action-get-linked-issues@v1.0 - id: linked-issues - with: - access-token: ${{ secrets.RELEASE_TOKEN }} - - move-to-needs-review: - if: github.event_name == 'pull_request_target' && contains(fromJson(needs.find-linked-issues.outputs.linked-issues).*.issue.state, 'open') - runs-on: ubuntu-latest - name: Move linked issues to Needs Review - needs: find-linked-issues - strategy: - max-parallel: 3 - matrix: - issues: ${{ fromJSON(needs.find-linked-issues.outputs.linked-issues) }} - steps: - - name: Move to Needs Review - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ matrix.issues.issue.node_id }} - operation_mode: status - status_value: ${{ env.needs_review }} # Target status - - move-to-done: - if: github.event_name == 'issues' && github.event.action == 'closed' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the Done column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.done }} # Target status diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9b5357331..05aeb0a54 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ defaults: jobs: lint: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v4" @@ -24,7 +24,7 @@ jobs: fetch-depth: 1 path: "pulp_container" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -34,29 +34,42 @@ jobs: pip install -r lint_requirements.txt echo ::endgroup:: - - name: Lint workflow files + - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + # run black separately from flake8 to get a diff - - name: Run black + - name: "Run black" run: | black --version black --check --diff . # Lint code. - - name: Run flake8 - run: flake8 + - name: "Run flake8" + run: | + flake8 - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py + + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 83aac5cec..dfa570d63 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -29,6 +29,9 @@ jobs: test: needs: "build" uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] changelog: runs-on: ubuntu-latest @@ -38,7 +41,7 @@ jobs: fetch-depth: 0 path: "pulp_container" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -65,55 +68,4 @@ jobs: branch: "changelog/update" delete-branch: true path: "pulp_container" - - publish: - runs-on: ubuntu-latest - needs: test - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_container" - - - uses: actions/download-artifact@v3 - with: - name: "plugin_package" - path: "pulp_container/dist/" - - - uses: "actions/setup-python@v4" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1' - echo ::endgroup:: - - - name: "Set environment variables" - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Download built docs - uses: actions/download-artifact@v3 - with: - name: "docs.tar" - path: "pulp_container" - - - name: Download Python client docs - uses: actions/download-artifact@v3 - with: - name: "python-client-docs.tar" - path: "pulp_container" - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar -C ./docs - .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} +... diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 2e06e6144..ec114480b 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -6,57 +6,62 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Container PR static checks +name: "Container PR static checks" on: pull_request_target: - types: [opened, synchronize, reopened] + types: ["opened", "synchronize", "reopened"] # This workflow runs with elevated permissions. # Do not even think about running a single bit of code from the PR. # Static analysis should be fine however. concurrency: - group: ${{ github.event.pull_request.number }}-${{ github.workflow }} + group: "${{ github.event.pull_request.number }}-${{ github.workflow }}" cancel-in-progress: true jobs: - single_commit: - runs-on: ubuntu-latest - name: Label multiple commit PR + apply_labels: + runs-on: "ubuntu-latest" + name: "Label PR" permissions: - pull-requests: write + pull-requests: "write" steps: - uses: "actions/checkout@v4" with: fetch-depth: 0 - - name: Commit Count Check + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Determine PR labels" run: | + pip install GitPython==3.1.42 git fetch origin ${{ github.event.pull_request.head.sha }} - echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV" - - uses: actions/github-script@v7 + python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" + - uses: "actions/github-script@v7" + name: "Apply PR Labels" with: script: | - const labelName = "multi-commit"; - const { COMMIT_COUNT } = process.env; + const { ADD_LABELS, REMOVE_LABELS } = process.env; - if (COMMIT_COUNT == 1) - { - try { - await github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - }); - } catch(err) { + if (REMOVE_LABELS.length) { + for await (const labelName of REMOVE_LABELS.split(",")) { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } } } - else - { + if (ADD_LABELS.length) { await github.rest.issues.addLabels({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - labels: [labelName], + labels: ADD_LABELS.split(","), }); } +... diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8dbfc4d2f..1f70d8a89 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,122 +19,10 @@ defaults: jobs: build: uses: "./.github/workflows/build.yml" - - build-bindings-docs: - needs: - - "build" - runs-on: "ubuntu-latest" - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: "docs" - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_container" - - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - repository: "pulp/pulp-openapi-generator" - path: "pulp-openapi-generator" - - - uses: "actions/setup-python@v4" - with: - python-version: "3.11" - - - uses: "actions/download-artifact@v3" - with: - name: "plugin_package" - path: "pulp_container/dist/" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_container/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: "Before Install" - run: | - .github/workflows/scripts/before_install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install" - run: | - .github/workflows/scripts/install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - name: "Upload python client packages" - uses: "actions/upload-artifact@v3" - with: - name: "python-client.tar" - path: "pulp_container/container-python-client.tar" - if-no-files-found: "error" - - - name: "Upload python client docs" - uses: "actions/upload-artifact@v3" - with: - name: "python-client-docs.tar" - path: "pulp_container/container-python-client-docs.tar" - if-no-files-found: "error" - - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v3" - with: - name: "ruby-client.tar" - path: "pulp_container/container-ruby-client.tar" - if-no-files-found: "error" - - name: Build docs - run: | - export DJANGO_SETTINGS_MODULE=pulpcore.app.settings - export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py - make -C docs/ PULP_URL="https://pulp" diagrams html - tar -cvf docs/docs.tar docs/_build - - - name: "Upload built docs" - uses: actions/upload-artifact@v3 - with: - name: "docs.tar" - path: "pulp_container/docs/docs.tar" - - - name: "Logs" - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" publish-package: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -145,12 +33,12 @@ jobs: fetch-depth: 1 path: "pulp_container" - - uses: "actions/download-artifact@v3" + - uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_container/dist/" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -172,7 +60,7 @@ jobs: publish-python-bindings: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -184,7 +72,7 @@ jobs: path: "pulp_container" - name: "Download Python client" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "python-client.tar" path: "pulp_container/" @@ -193,7 +81,7 @@ jobs: run: | tar -xvf container-python-client.tar - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -215,7 +103,7 @@ jobs: publish-ruby-bindings: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -227,7 +115,7 @@ jobs: path: "pulp_container" - name: "Download Ruby client" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "ruby-client.tar" path: "pulp_container/" @@ -249,61 +137,14 @@ jobs: - name: "Publish client to rubygems" run: | bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} - publish-docs: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_container" - - - uses: "actions/setup-python@v4" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install 'packaging~=21.3' requests - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Download built docs" - uses: "actions/download-artifact@v3" - with: - name: "docs.tar" - path: "pulp_container/" - - - name: "Download Python client docs" - uses: "actions/download-artifact@v3" - with: - name: "python-client-docs.tar" - path: "pulp_container/" - - - name: "Publish docs to pulpproject.org" - run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }} create-gh-release: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" - "publish-package" - "publish-python-bindings" - "publish-ruby-bindings" - - "publish-docs" steps: - name: "Create release on GitHub" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be1d16849..59d77a76f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,14 +28,14 @@ jobs: path: "pulp_container" token: ${{ secrets.RELEASE_TOKEN }} - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version towncrier + pip install bump-my-version towncrier echo ::endgroup:: - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index f00aea7cc..f68a2eaf3 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -45,11 +45,6 @@ if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 -fi - if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml @@ -57,7 +52,7 @@ fi for i in {1..3} do - ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3 + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 done if [[ $s -gt 0 ]] then @@ -65,9 +60,11 @@ then exit $s fi +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt +fi if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt - sed -i 's/\[.*\]//g' lowerbounds_constraints.txt + python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index b6e5fcc4f..0f13bb1fa 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -36,12 +36,6 @@ tail -v -n +1 .ci/ansible/Containerfile cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" cmd_prefix bash -c "usermod -a -G wheel pulp" -SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds") -if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then - # Many functional tests require these - cmd_prefix dnf install -yq lsof which -fi - if [[ "${REDIS_DISABLED:-false}" == true ]]; then cmd_prefix bash -c "s6-rc -d change redis" echo "The Redis service was disabled for $TEST" diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/build_python_client.sh similarity index 50% rename from .github/workflows/scripts/install_python_client.sh rename to .github/workflows/scripts/build_python_client.sh index e73865ff0..ec4988962 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -1,5 +1,8 @@ #!/bin/bash +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -python-client.tar and -python-client-docs.tar file in the plugins root directory. + # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use @@ -12,36 +15,18 @@ set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -source .github/workflows/scripts/utils.sh - -PULP_URL="${PULP_URL:-https://pulp}" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "container" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - pushd ../pulp-openapi-generator -rm -rf pulp_container-client +rm -rf "pulp_container-client" -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=container" - USE_LOCAL_API_JSON=1 ./generate.sh pulp_container python "$VERSION" -else - ./generate.sh pulp_container python "$VERSION" -fi +./gen-client.sh "../pulp_container/container-api.json" "container" python "pulp_container" pushd pulp_container-client python setup.py sdist bdist_wheel --python-tag py3 -twine check "dist/pulp_container_client-$VERSION-py3-none-any.whl" -twine check "dist/pulp_container-client-$VERSION.tar.gz" +twine check "dist/pulp_container_client-"*"-py3-none-any.whl" +twine check "dist/pulp_container-client-"*".tar.gz" -cmd_prefix pip3 install "/root/pulp-openapi-generator/pulp_container-client/dist/pulp_container_client-${VERSION}-py3-none-any.whl" -tar cvf ../../pulp_container/container-python-client.tar ./dist +tar cvf "../../pulp_container/container-python-client.tar" ./dist find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; find ./docs/* -exec sed -i 's/README//g' {} \; diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh new file mode 100755 index 000000000..deb3e8e39 --- /dev/null +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -ruby-client.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_container' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +rm -rf "pulp_container-client" + +./gen-client.sh "../pulp_container/container-api.json" "container" ruby "pulp_container" + +pushd pulp_container-client +gem build pulp_container_client +tar cvf "../../pulp_container/container-ruby-client.tar" "./pulp_container_client-"*".gem" +popd +popd diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py deleted file mode 100755 index 20584018a..000000000 --- a/.github/workflows/scripts/docs-publisher.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import subprocess -import os -import re -from shutil import rmtree -import tempfile -import requests -import json -from packaging import version - -WORKING_DIR = os.environ["WORKSPACE"] - -VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" -RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" - -USERNAME = "doc_builder_pulp_container" -HOSTNAME = "8.43.85.236" - -SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_container/" - - -def make_directory_with_rsync(remote_paths_list): - """ - Ensure the remote directory path exists. - - :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the - remote. - :type remote_paths_list: a list of strings, with each string representing a directory. - """ - try: - tempdir_path = tempfile.mkdtemp() - cwd = os.getcwd() - os.chdir(tempdir_path) - os.makedirs(os.sep.join(remote_paths_list)) - remote_path_arg = "%s@%s:%s%s" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - remote_paths_list[0], - ) - local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep - rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command) - if exit_code != 0: - raise RuntimeError("An error occurred while creating remote directories.") - finally: - rmtree(tempdir_path) - os.chdir(cwd) - - -def ensure_dir(target_dir, clean=True): - """ - Ensure that the directory specified exists and is empty. - - By default this will delete the directory if it already exists - - :param target_dir: The directory to process - :type target_dir: str - :param clean: Whether or not the directory should be removed and recreated - :type clean: bool - """ - if clean: - rmtree(target_dir, ignore_errors=True) - try: - os.makedirs(target_dir) - except OSError: - pass - - -def main(): - """ - Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--build-type", required=True, help="Build type: nightly, tag or changelog." - ) - parser.add_argument("--branch", required=True, help="Branch or tag name.") - opts = parser.parse_args() - - build_type = opts.build_type - - branch = opts.branch - - publish_at_root = False - - # rsync the docs - print("rsync the docs") - docs_directory = os.sep.join([WORKING_DIR, "docs"]) - local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep - if build_type == "nightly": - # This is a nightly build - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - make_directory_with_rsync(["en", branch, build_type]) - rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: - # Only publish docs at the root if this is the latest version - r = requests.get("https://pypi.org/pypi/pulp-container/json") - latest_version = version.parse(json.loads(r.text)["info"]["version"]) - docs_version = version.parse(branch) - # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. - # It doesn't prevent incorrect docs being published at root if 2 releases are done close - # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately - # after. - if docs_version >= latest_version: - publish_at_root = True - # Post releases should use the x.y.z part of the version string to form a path - if "post" in branch: - branch = ".".join(branch.split(".")[:-1]) - - # This is a GA build. - # publish to the root of docs.pulpproject.org - if publish_at_root: - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "en", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - # This is a pre-release - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "nightly", - "--exclude", - "testing", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "changelog": - if branch != "main": - raise RuntimeError("Can only publish CHANGELOG from main") - # Publish the CHANGELOG from main branch at the root directory - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - changelog_local_path_arg = os.path.join(local_path_arg, "changes.html") - rsync_command = [ - "rsync", - "-vzh", - "--omit-dir-times", - changelog_local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - raise RuntimeError("Build type must be either 'nightly', 'tag' or 'changelog'.") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 14e62d239..4d188d79d 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,24 +15,19 @@ set -euv source .github/workflows/scripts/utils.sh -PLUGIN_VERSION="$(sed -n -e 's/^\s*current_version\s*=\s*//p' .bumpversion.cfg | python -c 'from packaging.version import Version; print(Version(input()))')" +PLUGIN_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" PLUGIN_SOURCE="./pulp_container/dist/pulp_container-${PLUGIN_VERSION}-py3-none-any.whl" export PULP_API_ROOT="/pulp/" PIP_REQUIREMENTS=("pulp-cli") -if [[ "$TEST" = "docs" || "$TEST" = "publish" ]] -then - PIP_REQUIREMENTS+=("-r" "doc_requirements.txt") -fi +# This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} -if [[ "$TEST" != "docs" ]] -then - PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" - git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -fi +# Check out the pulp-cli branch matching the installed version. +PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" +git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli cd .ci/ansible/ if [ "$TEST" = "s3" ]; then @@ -55,6 +50,11 @@ if [[ -f ../../ci_requirements.txt ]]; then ci_requirements: true VARSYAML fi +if [ "$TEST" = "pulp" ]; then + cat >> vars/main.yaml << VARSYAML + upperbounds: true +VARSYAML +fi if [ "$TEST" = "lowerbounds" ]; then cat >> vars/main.yaml << VARSYAML lowerbounds: true @@ -102,18 +102,12 @@ pulp_scenario_env: {}\ fi if [ "$TEST" = "azure" ]; then - mkdir -p azurite - cd azurite - openssl req -newkey rsa:2048 -x509 -nodes -keyout azkey.pem -new -out azcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-azurite" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN" - sudo cp azcert.pem /usr/local/share/ca-certificates/azcert.crt - sudo dpkg-reconfigure ca-certificates - cd .. sed -i -e '/^services:/a \ - name: ci-azurite\ image: mcr.microsoft.com/azure-storage/azurite\ volumes:\ - ./azurite:/etc/pulp\ - command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml + command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ pulp_scenario_settings: {"flatpak_index": true}\ pulp_scenario_env: {}\ @@ -127,9 +121,7 @@ if [ "${PULP_API_ROOT:-}" ]; then fi pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" --username "admin" --password "password" -if [[ "$TEST" != "docs" ]]; then - cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" -fi +cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml ansible-playbook start_container.yaml @@ -149,34 +141,20 @@ sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certi # Hack: adding pulp CA to certifi.where() CERTIFI=$(python -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERTIFI" > /dev/null -if [[ "$TEST" = "azure" ]]; then - cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a "$CERTIFI" > /dev/null -fi # Hack: adding pulp CA to default CA file CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)') -cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null +cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERT" > /dev/null # Updating certs sudo update-ca-certificates echo ::endgroup:: -# Add our azcert.crt certificate to the container image along with the certificates from certifi -# so that we can use HTTPS with our fake Azure CI. certifi is self-contained and doesn't allow -# extension or modification of the trust store, so we do a weird and hacky thing (above) where we just -# overwrite or append to certifi's trust store behind it's back. -# -# We do this for both the CI host and the CI image. if [[ "$TEST" = "azure" ]]; then - AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())') - PULPCERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') - cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a "$PULPCERTIFI" > /dev/null - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /etc/pki/tls/cert.pem > /dev/null - AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://ci-azurite:10000/devstoreaccount1;' + AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING fi echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +cmd_prefix bash -c "pip3 list" echo ::endgroup:: diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh deleted file mode 100755 index 2fa3a0bf0..000000000 --- a/.github/workflows/scripts/install_ruby_client.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -source .github/workflows/scripts/utils.sh - -PULP_URL="${PULP_URL:-https://pulp}" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "container" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - -pushd ../pulp-openapi-generator -rm -rf pulp_container-client - -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=container" - USE_LOCAL_API_JSON=1 ./generate.sh pulp_container ruby "$VERSION" -else - ./generate.sh pulp_container ruby "$VERSION" -fi - -pushd pulp_container-client -gem build pulp_container_client -gem install --both "./pulp_container_client-$VERSION.gem" -tar cvf ../../pulp_container/container-ruby-client.tar "./pulp_container_client-$VERSION.gem" -popd -popd diff --git a/.github/workflows/scripts/post_docs_test.sh b/.github/workflows/scripts/post_docs_test.sh deleted file mode 100755 index d008aee9c..000000000 --- a/.github/workflows/scripts/post_docs_test.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env sh - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -export BASE_ADDR=https://pulp.example.com:443 - -cd docs/_scripts/ -bash ./docs_check.sh diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index f4d6e910d..f3509a61d 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -24,10 +24,8 @@ RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https:/ if [ "$RESPONSE" == "200" ]; then echo "pulp_container client $VERSION has already been released. Skipping." - exit +else + twine upload -u __token__ -p "$PYPI_API_TOKEN" \ + "dist/pulp_container_client-$VERSION-py3-none-any.whl" \ + "dist/pulp_container-client-$VERSION.tar.gz" fi - -twine upload -u __token__ -p "$PYPI_API_TOKEN" \ -"dist/pulp_container_client-$VERSION-py3-none-any.whl" \ -"dist/pulp_container-client-$VERSION.tar.gz" \ -; diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh deleted file mode 100755 index 281042405..000000000 --- a/.github/workflows/scripts/publish_docs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -mkdir ~/.ssh -touch ~/.ssh/pulp-infra -chmod 600 ~/.ssh/pulp-infra -echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra - -echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts -chmod 644 ~/.ssh/known_hosts - -export PYTHONUNBUFFERED=1 -export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export WORKSPACE=$PWD - -# start the ssh agent -eval "$(ssh-agent -s)" -ssh-add ~/.ssh/pulp-infra - -python3 .github/workflows/scripts/docs-publisher.py --build-type "$1" --branch "$2" - -if [[ "$GITHUB_WORKFLOW" == "Container changelog update" ]]; then - # Do not build bindings docs on changelog update - exit -fi - -mkdir -p ../container-bindings -tar -xvf container-python-client-docs.tar --directory ../container-bindings -pushd ../container-bindings - -# publish to docs.pulpproject.org/pulp_container_client -rsync -avzh site/ doc_builder_pulp_container@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_container_client/ - -# publish to docs.pulpproject.org/pulp_container_client/en/{release} -rsync -avzh site/ doc_builder_pulp_container@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_container_client/en/"$2" -popd diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index 267ee53e6..bf6071bc5 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -28,6 +28,6 @@ then fi twine upload -u __token__ -p "$PYPI_API_TOKEN" \ -"dist/pulp_container-$VERSION-py3-none-any.whl" \ -"dist/pulp-container-$VERSION.tar.gz" \ +dist/pulp?container-"$VERSION"-py3-none-any.whl \ +dist/pulp?container-"$VERSION".tar.gz \ ; diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh index 9525f229d..a08353cdb 100755 --- a/.github/workflows/scripts/release.sh +++ b/.github/workflows/scripts/release.sh @@ -10,7 +10,8 @@ then exit 1 fi -NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +# The tail is a necessary workaround to remove the warning from the output. +NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)" echo "Release ${NEW_VERSION}" if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] @@ -20,7 +21,7 @@ then fi towncrier build --yes --version "${NEW_VERSION}" -bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty -bump2version patch --commit +bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump-my-version bump patch --commit git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index e1c51b407..aac3938b9 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -16,10 +16,9 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. source .github/workflows/scripts/utils.sh export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh -export POST_DOCS_TEST=$PWD/.github/workflows/scripts/post_docs_test.sh export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh -# Needed for both starting the service and building the docs. +# Needed for starting the service # Gets set in .github/settings.yml, but doesn't seem to inherited by # this script. export DJANGO_SETTINGS_MODULE=pulpcore.app.settings @@ -27,22 +26,6 @@ export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py export PULP_URL="https://pulp" -if [[ "$TEST" = "docs" ]]; then - if [[ "$GITHUB_WORKFLOW" == "Container CI" ]]; then - towncrier build --yes --version 4.0.0.ci - fi - # Legacy Docs Build - cd docs - make PULP_URL="$PULP_URL" diagrams html - tar -cvf docs.tar ./_build - cd .. - - if [ -f "$POST_DOCS_TEST" ]; then - source "$POST_DOCS_TEST" - fi - exit -fi - REPORTED_STATUS="$(pulp status)" echo "machine pulp @@ -52,75 +35,76 @@ password password # Some commands like ansible-galaxy specifically require 600 cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -# Generate and install binding +# Generate bindings +################### + +echo "::group::Generate bindings" + +touch bindings_requirements.txt pushd ../pulp-openapi-generator -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then # Use app_label to generate api.json and package to produce the proper package name. - if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] + # Workaround: Domains are not supported by the published bindings. + # Sadly: Different pulpcore-versions aren't either... + # * In the 'pulp' scenario we use the published/prebuilt bindings, so we can test it. + # * In other scenarios we generate new bindings from server spec, so we have a more + # reliable client. + if [ "$TEST" = "pulp" ] then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done + BUILT_CLIENTS=" container " else - # Sadly: Different pulpcore-versions aren't either... - for item in $(jq -r '.versions[]| select(.component!="container")| tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done + BUILT_CLIENTS="" fi -else - # Infer the client name from the package name by replacing "-" with "_". - # Use the component to infer the package name on older versions of pulpcore. - if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] - then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - else - # Sadly: Different pulpcore-versions aren't either... - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="container")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - fi -fi + for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") + do + COMPONENT="$(jq -r '.component' <<<"${ITEM}")" + VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")" + # On older status endpoints, the module was not provided, but the package should be accurate + # there, because we did not merge plugins into pulpcore back then. + MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")" + PACKAGE="${MODULE%%.*}" + cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json" + if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]] + then + rm -rf "./${PACKAGE}-client" + ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" + pushd "${PACKAGE}-client" + python setup.py sdist bdist_wheel --python-tag py3 + popd + else + if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] + then + ls -lR "${PACKAGE}-client/" + echo "Error: Client bindings for ${COMPONENT} not found." + echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing." + exit 1 + fi + fi + echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp_container/bindings_requirements.txt" + done popd -# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. -echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt +echo "::endgroup::" + +echo "::group::Debug bindings diffs" + +# Bindings diff for container +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "container-api.json" > "build-api.json" +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "../pulp-openapi-generator/container-api.json" > "test-api.json" +jsondiff --indent 2 build-api.json test-api.json || true +echo "::endgroup::" + +# Install test requirements +########################### + +# Add a safeguard to make sure the proper versions of the clients are installed. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" @@ -146,12 +130,12 @@ if [ -f "$FUNC_TEST_SCRIPT" ]; then else if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] then - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m 'not parallel' --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m 'not parallel' --nightly" else - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m 'not parallel'" - fi + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m 'not parallel'" + fi fi pushd ../pulp-cli pip install -r test_requirements.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 853d535f0..cf850049f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,10 @@ name: "Test" on: workflow_call: + inputs: + matrix_env: + required: true + type: string defaults: run: @@ -20,12 +24,7 @@ jobs: strategy: fail-fast: false matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - TEST: lowerbounds + env: ${{ fromJSON(inputs.matrix_env) }} steps: - uses: "actions/checkout@v4" @@ -39,19 +38,40 @@ jobs: repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - - uses: "actions/download-artifact@v3" + - name: "Download plugin package" + uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_container/dist/" + - name: "Download API specs" + uses: "actions/download-artifact@v4" + with: + name: "api_spec" + path: "pulp_container/" + + - name: "Download client packages" + uses: "actions/download-artifact@v4" + with: + name: "python-client.tar" + path: "pulp_container" + + - name: "Unpack client packages" + working-directory: "pulp-openapi-generator" + run: | + mkdir -p "pulp_container-client" + pushd "pulp_container-client" + tar xvf "../../pulp_container/container-python-client.tar" + popd + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_container/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: @@ -68,10 +88,6 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - uses: ruby/setup-ruby@v1 - if: ${{ env.TEST == 'pulp' }} - with: - ruby-version: "2.6" - name: "Install" run: | @@ -94,16 +110,6 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - if: "${{ env.TEST == 'pulp' }}" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - name: "Script" run: | .github/workflows/scripts/script.sh @@ -119,43 +125,13 @@ jobs: docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt - name: "Upload Deprecations" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: "deprecations" + name: "deprecations-${{ matrix.env.TEST }}" path: "pulp_container/deprecations-${{ matrix.env.TEST }}.txt" if-no-files-found: "error" retention-days: 5 - - name: Upload python client packages - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v3 - with: - name: "python-client.tar" - path: "pulp_container/container-python-client.tar" - if-no-files-found: "error" - retention-days: 5 - - - name: Upload python client docs - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v3 - with: - name: "python-client-docs.tar" - path: "pulp_container/container-python-client-docs.tar" - if-no-files-found: "error" - retention-days: 5 - - name: Upload Ruby client - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v3 - with: - name: "ruby-client.tar" - path: "pulp_container/container-ruby-client.tar" - if-no-files-found: "error" - retention-days: 5 - - name: Upload built docs - if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v3 - with: - name: "docs.tar" - path: "pulp_container/docs/docs.tar" + overwrite: true - name: "Logs" if: always() @@ -167,4 +143,5 @@ jobs: docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + docker exec pulp bash -c "pip3 list" || true +... diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index 5a0d759cf..d69e3998a 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -19,7 +19,7 @@ jobs: update_backport_labels: runs-on: "ubuntu-latest" steps: - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 7ba15fa1f..a42891e41 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -11,8 +11,8 @@ name: "Container CI Update" on: schedule: # * is a special character in YAML so you have to quote this string - # runs at 2:30 UTC every Sunday - - cron: '30 2 * * 0' + # runs at 3:30 UTC every Sunday + - cron: '30 3 * * 0' workflow_dispatch: jobs: @@ -29,14 +29,14 @@ jobs: repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install gitpython requests packaging jinja2 pyyaml + pip install gitpython packaging -r plugin_template/requirements.txt echo ::endgroup:: - name: "Configure Git with pulpbot name and email" @@ -62,40 +62,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch main" - body: "" branch: "update-ci/main" base: "main" - commit-message: | - Update CI files - - [noissue] - delete-branch: true - - uses: "actions/checkout@v4" - with: - fetch-depth: 0 - path: "pulp_container" - ref: "2.18" - - - name: "Run update" - working-directory: "pulp_container" - run: | - ../plugin_template/scripts/update_ci.sh - - - name: "Create Pull Request for CI files" - uses: "peter-evans/create-pull-request@v6" - with: - token: "${{ secrets.RELEASE_TOKEN }}" - path: "pulp_container" - committer: "pulpbot " - author: "pulpbot " - title: "Update CI files for branch 2.18" - body: "" - branch: "update-ci/2.18" - base: "2.18" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -106,7 +74,7 @@ jobs: - name: "Run update" working-directory: "pulp_container" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" @@ -116,13 +84,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 2.10" - body: "" branch: "update-ci/2.10" base: "2.10" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -133,7 +96,7 @@ jobs: - name: "Run update" working-directory: "pulp_container" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" @@ -143,13 +106,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 2.14" - body: "" branch: "update-ci/2.14" base: "2.14" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -160,7 +118,7 @@ jobs: - name: "Run update" working-directory: "pulp_container" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" @@ -170,13 +128,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 2.15" - body: "" branch: "update-ci/2.15" base: "2.15" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -187,7 +140,7 @@ jobs: - name: "Run update" working-directory: "pulp_container" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" @@ -197,11 +150,29 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 2.16" - body: "" branch: "update-ci/2.16" base: "2.16" - commit-message: | - Update CI files + delete-branch: true + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + path: "pulp_container" + ref: "2.18" - [noissue] + - name: "Run update" + working-directory: "pulp_container" + run: | + ../plugin_template/scripts/update_ci.sh --release + + - name: "Create Pull Request for CI files" + uses: "peter-evans/create-pull-request@v6" + with: + token: "${{ secrets.RELEASE_TOKEN }}" + path: "pulp_container" + committer: "pulpbot " + author: "pulpbot " + title: "Update CI files for branch 2.18" + branch: "update-ci/2.18" + base: "2.18" delete-branch: true +... diff --git a/CHANGES/.TEMPLATE.rst b/CHANGES/.TEMPLATE.md similarity index 50% rename from CHANGES/.TEMPLATE.rst rename to CHANGES/.TEMPLATE.md index 49c2305d7..2308193b9 100644 --- a/CHANGES/.TEMPLATE.rst +++ b/CHANGES/.TEMPLATE.md @@ -1,47 +1,39 @@ -{% if render_title %} -{% if versiondata.name %} -{{ versiondata.name }} {{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.name + versiondata.version + versiondata.date)|length + 4)}} -{% else %} -{{ versiondata.version }} ({{ versiondata.date }}) -{{ top_underline * ((versiondata.version + versiondata.date)|length + 3)}} -{% endif %} -{% endif %} +{# TOWNCRIER TEMPLATE #} {% for section, _ in sections.items() %} -{% set underline = underlines[0] %}{% if section %}{{section}} -{{ underline * section|length }}{% set underline = underlines[1] %} +{%- set section_slug = "-" + section|replace(" ", "-")|replace("_", "-")|lower %} +{%- if section %} +### {{section}} {: #{{versiondata.version}}{{section_slug}} } +{% else %} +{%- set section_slug = "" %} {% endif %} - {% if sections[section] %} {% for category, val in definitions.items() if category in sections[section]%} -{{ definitions[category]['name'] }} -{{ underline * definitions[category]['name']|length }} + +#### {{ definitions[category]['name'] }} {: #{{versiondata.version}}{{section_slug}}-{{category}} } {% if definitions[category]['showcontent'] %} {% for text, values in sections[section][category].items() %} - {{ text }} +{% if values %} {{ values|join(',\n ') }} +{% endif %} {% endfor %} - {% else %} - {{ sections[section][category]['']|join(', ') }} - {% endif %} {% if sections[section][category]|length == 0 %} -No significant changes. +No significant changes. {% else %} {% endif %} - {% endfor %} {% else %} -No significant changes. - +No significant changes. {% endif %} {% endfor %} ----- +--- diff --git a/doc_requirements.txt b/doc_requirements.txt index 119618d07..be646a8b2 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,18 +1,8 @@ # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --docs pulp_container' to update this file. +# './plugin-template --github pulp_container' to update this file. # # For more info visit https://github.com/pulp/plugin_template --r requirements.txt towncrier - -# Legacy docs -plantuml -sphinx~=7.1.2 -sphinx-rtd-theme==1.3.0 -sphinxcontrib-jquery -sphinxcontrib-openapi -mistune<4.0.0 -Jinja2<3.2 -# Extra requirements +pulp-docs @ git+https://github.com/pulp/pulp-docs@main diff --git a/lint_requirements.txt b/lint_requirements.txt index 71e843f26..959a5fc2d 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,9 +5,10 @@ # # For more info visit https://github.com/pulp/plugin_template -# python packages handy for developers, but not required by pulp black==24.3.0 +bump-my-version check-manifest flake8 flake8-black +packaging yamllint diff --git a/pyproject.toml b/pyproject.toml index 9e25e48ff..4712a7a7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,3 +49,34 @@ ignore = [ "lint_requirements.txt", ".flake8", ] + +[tool.bumpversion] +# This section is managed by the plugin template. Do not edit manually. + +current_version = "2.19.4.dev" +commit = false +tag = false +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(\\.(?P[a-z]+))?" +serialize = [ + "{major}.{minor}.{patch}.{release}", + "{major}.{minor}.{patch}", +] + +[tool.bumpversion.parts.release] +# This section is managed by the plugin template. Do not edit manually. + +optional_value = "prod" +values = [ + "dev", + "prod", +] + +[[tool.bumpversion.files]] +# This section is managed by the plugin template. Do not edit manually. + +filename = "./pulp_container/app/__init__.py" +search = "version = \"{current_version}\"" +replace = "version = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "./setup.py" \ No newline at end of file diff --git a/releasing.md b/releasing.md new file mode 100644 index 000000000..e85cf12b4 --- /dev/null +++ b/releasing.md @@ -0,0 +1,29 @@ +[//]: # "WARNING: DO NOT EDIT!" +[//]: # "" +[//]: # "This file was generated by plugin_template, and is managed by it. Please use" +[//]: # "'./plugin-template --github pulp_container' to update this file." +[//]: # "" +[//]: # "For more info visit https://github.com/pulp/plugin_template" +# Releasing (For Internal Use) + +This document outlines the steps to perform a release. + +### Determine if a Release is Required +- Make sure to have GitPython python package installed +- Run the release checker script: + ``` + python3 .ci/scripts/check_release.py + ``` + +### Release a New Y-version (e.g., 3.23.0) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_container/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. +- Look for the "Bump minor version" pull request and merge it. +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_container/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. + +### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_container/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. + +## Final Steps +- Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulp_container/actions/workflows/publish.yml) workflow succeeds). +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). +- [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). diff --git a/template_config.yml b/template_config.yml index 5a3e46cd5..0b5b85f0e 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-326-ge5addc7 +# generated with plugin_template@2021.08.26-405-g3845bbc api_root: /pulp/ black: true @@ -12,7 +12,6 @@ check_stray_pulpcore_imports: true ci_base_image: ghcr.io/pulp/pulp-ci-centos ci_env: {} ci_trigger: '{pull_request: {branches: [''*'']}}' -ci_update_docs: true cli_package: pulp-cli cli_repo: https://github.com/pulp/pulp-cli.git core_import_allowed: [] @@ -20,18 +19,13 @@ deploy_client_to_pypi: true deploy_client_to_rubygems: true deploy_to_pypi: true disabled_redis_runners: [] -doc_requirements_from_pulpcore: false docker_fixtures: false -docs_test: true -extra_docs_requirements: [] flake8: true flake8_ignore: [] github_org: pulp -issue_tracker: github -kanban: true latest_release_branch: '2.18' lint_requirements: true -noissue_marker: '[noissue]' +os_required_packages: [] parallel_test_workers: 8 plugin_app_label: container plugin_default_branch: main @@ -41,7 +35,6 @@ plugins: name: pulp_container post_job_template: null pre_job_template: null -publish_docs_to_pulpprojectdotorg: true pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} @@ -86,6 +79,4 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true -use_legacy_docs: true -use_unified_docs: false