🚸 Alternative workflow for running Python tests individually (#52)

* added parallel python test workflow * ♻️ adjust the individual Python testing workflow Signed-off-by: burgholzer <[email protected]> * ✨ allow running Python testing individually Signed-off-by: burgholzer <[email protected]> * 🩹 fix dependency on dist job Signed-off-by: burgholzer <[email protected]> * 🩹 properly name coverage report Signed-off-by: burgholzer <[email protected]> * 🩹 ensure coverage data is uploaded Signed-off-by: burgholzer <[email protected]> --------- Signed-off-by: burgholzer <[email protected]> Co-authored-by: burgholzer <[email protected]>
cda-tum · Dec 8, 2024 · 6022c5a · 6022c5a
1 parent 6f84e6d
commit 6022c5a
Show file tree

Hide file tree

Showing 2 changed files with 131 additions and 0 deletions.
diff --git a/.github/workflows/reusable-python-ci.yml b/.github/workflows/reusable-python-ci.yml
@@ -3,6 +3,14 @@ name: 🐍 • CI
 on:
   workflow_call:
     inputs:
+      skip-testing-latest-python:
+        description: "Whether to skip testing on the latest Python version. This only has an effect if the Python tests are run individually."
+        default: false
+        type: boolean
+      run-tests-individually:
+        description: "Whether to run the Python tests individually or combined"
+        default: false
+        type: boolean
       setup-z3:
         description: "Whether to set up Z3"
         default: false
@@ -38,9 +46,18 @@ jobs:
       # set up mold as linker for faster C++ builds
       - name: Set up mold as linker
         uses: rui314/setup-mold@v1
+      # run the build-and-inspect-python-package action (outputs supported Python versions)
       - uses: hynek/build-and-inspect-python-package@v2
+        id: baipp
+      # reduce the list of Python versions by one if the latest Python version is to be skipped
+      - name: 🐍 Conditionally reduce the list of considered Python versions
+        run: echo "supported-python-versions=$(echo '${{ steps.baipp.outputs.supported_python_classifiers_json_array }}' | jq -rc 'if ${{ inputs.skip-testing-latest-python }} then .[:-1] else . end')" >> $GITHUB_OUTPUT
+        id: supported-python-versions
+    outputs:
+      python-versions: ${{ steps.supported-python-versions.outputs.supported-python-versions }}
 
   python-tests:
+    if: ${{ !inputs.run-tests-individually }}
     name: 🐍 ${{ matrix.runs-on }}
     strategy:
       fail-fast: false
@@ -52,7 +69,26 @@ jobs:
       setup-z3: ${{ inputs.setup-z3 }}
       z3-version: ${{ inputs.z3-version }}
 
+  python-tests-individual:
+    if: ${{ inputs.run-tests-individually }}
+    needs: [dist]
+    name: 🐍 ${{ matrix.session }} ${{ matrix.python-version }} ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: [ubuntu-latest, macos-13, macos-14, windows-latest]
+        python-version: ${{ fromJson(needs.dist.outputs.python-versions) }}
+        session: ["minimums", "tests"]
+    uses: ./.github/workflows/reusable-python-tests-individual.yml
+    with:
+      runs-on: ${{ matrix.runs-on }}
+      python-version: ${{ matrix.python-version }}
+      session: ${{ matrix.session }}
+      setup-z3: ${{ inputs.setup-z3 }}
+      z3-version: ${{ inputs.z3-version }}
+
   python-coverage-upload:
+    if: ${{ !inputs.run-tests-individually }}
     name: 📈
     needs: [python-tests]
     runs-on: ubuntu-latest
@@ -74,3 +110,27 @@ jobs:
           fail_ci_if_error: true
           flags: python
           use_oidc: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork) }}
+
+  python-coverage-upload-individual:
+    if: ${{ inputs.run-tests-individually }}
+    name: 📈
+    needs: [python-tests-individual]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read # Required for the `actions/checkout` action
+      id-token: write # Required for the `codecov/codecov-action` action
+    steps:
+      # check out the repository (mostly for the codecov config)
+      - uses: actions/checkout@v4
+      # download coverage reports from all jobs
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-*
+          path: coverage-reports
+          merge-multiple: true
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          fail_ci_if_error: true
+          flags: python
+          use_oidc: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork) }}
diff --git a/.github/workflows/reusable-python-tests-individual.yml b/.github/workflows/reusable-python-tests-individual.yml
@@ -0,0 +1,71 @@
+name: 🐍 • Tests
+on:
+  workflow_call:
+    inputs:
+      runs-on:
+        description: "The platform to run the tests on"
+        required: true
+        type: string
+      python-version:
+        description: "The Python version to use"
+        required: true
+        type: string
+      session:
+        description: "The nox session to run (typically 'tests' or 'minimums')"
+        required: true
+        type: string
+      setup-z3:
+        description: "Whether to set up Z3"
+        default: false
+        type: boolean
+      z3-version:
+        description: "The version of Z3 to set up"
+        default: "4.13.0"
+        type: string
+
+jobs:
+  python-tests:
+    name: 🐍 ${{ inputs.session }} ${{ inputs.python-version }} ${{ inputs.runs-on }}
+    runs-on: ${{ inputs.runs-on }}
+    env:
+      FORCE_COLOR: 3
+      GITHUB_TOKEN: ${{ github.token }}
+    steps:
+      # check out the repository (including submodules and all history)
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      # set up MSVC development environment (Windows only)
+      - uses: ilammy/msvc-dev-cmd@v1
+      # optionally set up Z3
+      - if: ${{ inputs.setup-z3 }}
+        name: Setup Z3
+        uses: cda-tum/setup-z3@v1
+        with:
+          version: ${{ inputs.z3-version }}
+      # set up ccache for faster C++ builds
+      - name: Setup ccache
+        uses: Chocobo1/setup-ccache-action@v1
+        with:
+          prepend_symlinks_to_path: false
+          windows_compile_environment: msvc
+          override_cache_key: python-tests-${{ inputs.runs-on }}-${{ inputs.python-version }}-${{ inputs.session }}
+      # set up mold as linker for faster C++ builds (Linux only)
+      - name: Set up mold as linker (Linux only)
+        uses: rui314/setup-mold@v1
+      # set up uv for faster Python package management
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+          enable-cache: true
+      # run the nox session (assumes a corresponding nox session exists) with coverage
+      - name: Test on 🐍 ${{ inputs.python-version }}
+        run: uvx nox -s ${{ inputs.session }}-${{ inputs.python-version }} --verbose -- --cov --cov-report=xml:coverage-${{ inputs.session }}-${{ inputs.python-version }}-${{ inputs.runs-on }}.xml
+      # upload the report as an artifact to GitHub so that it can later be uploaded to Codecov
+      - name: Upload 🐍 coverage report for the ${{ inputs.session }} session on 🐍 ${{ inputs.python-version }} running ${{ inputs.runs-on }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ inputs.session }}-${{ inputs.python-version }}-${{ inputs.runs-on }}
+          path: coverage-*