diff --git a/.github/workflows/code_scan.yml b/.github/workflows/code_scan.yml index 119f9362..1b806bf3 100644 --- a/.github/workflows/code_scan.yml +++ b/.github/workflows/code_scan.yml @@ -1,6 +1,9 @@ name: Security Code Scan on: + schedule: + # every UTC 7PM from Mon to Fri + - cron: "0 19 * * 1-5" push: branches: - releases/* diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..5de85eb1 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,41 @@ +name: E2E test + +on: + schedule: + # every UTC 7PM from Mon to Fri + - cron: "0 19 * * 1-5" + workflow_dispatch: # run on request (no need for PR) + +# Declare default permissions as read only. +permissions: read-all + +jobs: + Pre-Merge-Checks: + uses: ./.github/workflows/pre_merge.yml + + E2E-Test: + runs-on: ubuntu-22.04 + needs: Pre-Merge-Checks + timeout-minutes: 120 + # This is what will cancel the job concurrency + concurrency: + group: ${{ github.workflow }}-E2E-${{ github.ref }} + cancel-in-progress: true + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Install Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: "3.10" + - name: Install tox + run: python -m pip install tox==4.4.6 + - name: Run E2E Test + run: tox -vv -e val-py310 -- tests/e2e --csv=.tox/val-py310/e2e-test.csv + - name: Upload artifacts + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: e2e-test-results + path: .tox/val-py310/*.csv + # Use always() to always run this step to publish test results when there are test failures + if: ${{ always() }} diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml index eaf3ed28..1027194f 100644 --- a/.github/workflows/pre_merge.yml +++ b/.github/workflows/pre_merge.yml @@ -4,7 +4,7 @@ on: push: branches: - develop - - releases/** + - releases/* pull_request: types: - opened @@ -61,22 +61,22 @@ jobs: - name: Install tox run: python -m pip install tox==4.4.6 - name: Run unit test - run: tox -vv -e pytest-${{ matrix.tox-env }} -- tests/unit --csv=.tox/pytest-${{ matrix.tox-env }}/unit-test.csv - --cov=openvino_xai --cov-report term --cov-report xml:.tox/pytest-${{ matrix.tox-env }}/unit-test-coverage.xml + run: tox -vv -e dev-${{ matrix.tox-env }} -- tests/unit --csv=.tox/dev-${{ matrix.tox-env }}/unit-test.csv + --cov=openvino_xai --cov-report term --cov-report xml:.tox/dev-${{ matrix.tox-env }}/unit-test-coverage.xml - name: Upload artifacts uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: unit-test-results-${{ matrix.tox-env }} path: | - .tox/pytest-${{ matrix.tox-env }}/*.csv - .tox/pytest-${{ matrix.tox-env }}/*.xml + .tox/dev-${{ matrix.tox-env }}/*.csv + .tox/dev-${{ matrix.tox-env }}/*.xml # Use always() to always run this step to publish test results when there are test failures if: ${{ always() }} - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: - files: .tox/pytest-${{ matrix.tox-env }}/unit-test-coverage.xml - flags: pytest-${{ matrix.tox-env }} + files: .tox/dev-${{ matrix.tox-env }}/unit-test-coverage.xml + flags: dev-${{ matrix.tox-env }} Integration-Test: runs-on: ubuntu-22.04 @@ -96,11 +96,11 @@ jobs: - name: Install tox run: python -m pip install tox==4.4.6 - name: Run Integration Test - run: tox -vv -e pytest-py310 -- tests/integration --csv=.tox/pytest-py310/intg-test.csv + run: tox -vv -e dev-py310 -- tests/integration --csv=.tox/dev-py310/intg-test.csv - name: Upload artifacts uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: intg-test-results - path: .tox/pytest-py310/*.csv + path: .tox/dev-py310/*.csv # Use always() to always run this step to publish test results when there are test failures if: ${{ always() }} diff --git a/.gitignore b/.gitignore index 95613474..95c2c8de 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ otx_models/ .mypy_cache .ruff_cache .coverage +.tox *.jpg *.jpeg diff --git a/pyproject.toml b/pyproject.toml index 4ac86b64..7a20280b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dev = [ "pre-commit==3.7.0", "addict", ] -dev_timm = [ +val = [ "timm==0.9.5", "onnx==1.14.1", ] diff --git a/tests/integration/test_classification_timm.py b/tests/e2e/test_classification_timm.py similarity index 100% rename from tests/integration/test_classification_timm.py rename to tests/e2e/test_classification_timm.py diff --git a/tox.ini b/tox.ini index 4c265e21..2e439416 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,12 @@ skip_install = true commands = pre-commit run --all-files -[testenv:pytest-{py310, py311}] -extras = dev,dev_timm +[testenv:dev-{py310, py311}] +extras = dev +commands = + pytest -ra --showlocals {posargs:tests/} + +[testenv:val-{py310, py311}] +extras = dev,val commands = pytest -ra --showlocals {posargs:tests/}