ci-tests-full.yml

name: CI testing - complete

# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
on:  # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches: [master, "release/*"]
  pull_request:
    branches: [master, "release/*"]
    types: [opened, reopened, ready_for_review, synchronize]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
  cancel-in-progress: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}

defaults:
  run:
    shell: bash

jobs:
  check-diff:
    if: github.event.pull_request.draft == false
    uses: ./.github/workflows/focus-diff.yml

  pytest:
    runs-on: ${{ matrix.os }}
    needs: check-diff
    strategy:
      fail-fast: false
      matrix:
        os: ['ubuntu-20.04']
        python-version: ['3.8']
        pytorch-version: ['1.9.1', '1.10.2', '1.11.0', '1.12.1']
        include:
          - {os: 'ubuntu-22.04', python-version: '3.8', pytorch-version: '1.13.0'}
          - {os: 'ubuntu-22.04', python-version: '3.10', pytorch-version: '1.13.0'}
          - {os: 'macOS-11', python-version: '3.8', pytorch-version: '1.13.0'}
          - {os: 'macOS-11', python-version: '3.9', pytorch-version: '1.13.0'}
          - {os: 'windows-2022', python-version: '3.8', pytorch-version: '1.13.0'}
          - {os: 'windows-2022', python-version: '3.9', pytorch-version: '1.13.0'}
          # the oldest configurations
          - {os: 'ubuntu-20.04', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'}
          - {os: 'ubuntu-20.04', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'}
          - {os: 'macOS-11', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'}
          - {os: 'macOS-11', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'}
          - {os: 'windows-2019', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'}
          - {os: 'windows-2019', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'}
    env:
      PYTEST_ARTEFACT: test-results-${{ matrix.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}.xml
      PYTORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html
      FREEZE_REQUIREMENTS: 1

    # Timeout: https://stackoverflow.com/a/59076067/4521646
    # seems that MacOS jobs take much more than orger OS
    timeout-minutes: 95

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
    - name: Setup macOS
      if: ${{ runner.os == 'macOS' }}
      run: |
        echo 'UNITTEST_TIMEOUT=--timeout=120' >> $GITHUB_ENV
        brew install gcc libomp ffmpeg # https://github.com/pytorch/pytorch/issues/20030
    - name: Setup Linux
      if: ${{ runner.os == 'Linux' }}
      run: |
        echo 'UNITTEST_TIMEOUT=--timeout=120' >> $GITHUB_ENV
        sudo apt update --fix-missing
        sudo apt install -y ffmpeg
    - name: Setup Windows
      if: ${{ runner.os == 'windows' }}
      run: |
        choco install ffmpeg

    - name: Set PyTorch version
      if: inputs.requires != 'oldest'
      run: |
        pip install packaging -q
        python ./requirements/adjust-versions.py requirements.txt ${{ matrix.pytorch-version }}

    - name: full chashing
      uses: ./.github/actions/caching
      with:
        requires: ${{ matrix.requires }}
        python-version: ${{ matrix.python-version }}

    - name: Install pkg
      run: |
        pip --version
        pip install -e . -U --find-links $PYTORCH_URL
        pip install -r requirements/doctest.txt -U
        pip list

    # todo: copy this to install checks
    - name: DocTests
      working-directory: ./src
      # NOTE: run coverage on tests does not propagate failure status for Win, https://github.com/nedbat/coveragepy/issues/1003
      run: python -m pytest torchmetrics --reruns 3 --reruns-delay 1

    - name: Freeze PIL (hotfix)
      # import of PILLOW_VERSION which they recently removed in v9.0 in favor of __version__
      run: pip install "Pillow<9.0"  # It messes with torchvision

    - name: Install all dependencies
      run: |
        pip install cython  # needed for installing `pycocotools` in latest config
        python ./requirements/adjust-versions.py requirements.txt
        for fpath in `ls requirements/*.txt`; do
            python ./requirements/adjust-versions.py $fpath
        done
        pip install --requirement requirements/devel.txt -U --find-links $PYTORCH_URL
        pip list

    - name: Sanity check
      id: info
      run: |
        python -c "from torch import __version__ as ver; ver = ver.split('+')[0] ; assert ver == '${{ matrix.pytorch-version }}', ver"
        python -c 'import torch ; print("TORCH=" + str(torch.__version__))' >> $GITHUB_OUTPUT

    - name: Unitests
      uses: ./.github/actions/unittesting
      if: ${{ needs.check-diff.outputs.test-dirs != '' }}
      with:
        requires: ${{ matrix.requires }}
        python-version: ${{ matrix.python-version }}
        pytorch-version: ${{ steps.info.outputs.TORCH }}
        dirs: ${{ needs.check-diff.outputs.test-dirs }}
        test-timeout: ${{ env.UNITTEST_TIMEOUT }}

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: tests/coverage.xml
        flags: cpu,pip,unitests,${{ runner.os }},python${{ matrix.python-version }},torch${{ steps.info.outputs.TORCH }}
        env_vars: OS,PYTHON
        name: codecov-umbrella
        fail_ci_if_error: false