Add daily E2E test workflow (#6)

* Add daily E2E test workflow

* Move timm tests to e2e

.github/workflows/code_scan.yml

@@ -1,6 +1,9 @@
 name: Security Code Scan
 
 on:
+  schedule:
+    # every UTC 7PM from Mon to Fri
+    - cron: "0 19 * * 1-5"
   push:
     branches:
       - releases/*

.github/workflows/e2e.yml

@@ -0,0 +1,41 @@
+name: E2E test
+
+on:
+  schedule:
+    # every UTC 7PM from Mon to Fri
+    - cron: "0 19 * * 1-5"
+  workflow_dispatch: # run on request (no need for PR)
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  Pre-Merge-Checks:
+    uses: ./.github/workflows/pre_merge.yml
+
+  E2E-Test:
+    runs-on: ubuntu-22.04
+    needs: Pre-Merge-Checks
+    timeout-minutes: 120
+    # This is what will cancel the job concurrency
+    concurrency:
+      group: ${{ github.workflow }}-E2E-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Install Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: "3.10"
+      - name: Install tox
+        run: python -m pip install tox==4.4.6
+      - name: Run E2E Test
+        run: tox -vv -e val-py310 -- tests/e2e --csv=.tox/val-py310/e2e-test.csv
+      - name: Upload artifacts
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: e2e-test-results
+          path: .tox/val-py310/*.csv
+        # Use always() to always run this step to publish test results when there are test failures
+        if: ${{ always() }}

.github/workflows/pre_merge.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - develop
-      - releases/**
+      - releases/*
   pull_request:
     types:
       - opened
@@ -61,22 +61,22 @@ jobs:
       - name: Install tox
         run: python -m pip install tox==4.4.6
       - name: Run unit test
-        run: tox -vv -e pytest-${{ matrix.tox-env }} -- tests/unit --csv=.tox/pytest-${{ matrix.tox-env }}/unit-test.csv
-          --cov=openvino_xai --cov-report term --cov-report xml:.tox/pytest-${{ matrix.tox-env }}/unit-test-coverage.xml
+        run: tox -vv -e dev-${{ matrix.tox-env }} -- tests/unit --csv=.tox/dev-${{ matrix.tox-env }}/unit-test.csv
+          --cov=openvino_xai --cov-report term --cov-report xml:.tox/dev-${{ matrix.tox-env }}/unit-test-coverage.xml
       - name: Upload artifacts
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: unit-test-results-${{ matrix.tox-env }}
           path: |
-            .tox/pytest-${{ matrix.tox-env }}/*.csv
-            .tox/pytest-${{ matrix.tox-env }}/*.xml
+            .tox/dev-${{ matrix.tox-env }}/*.csv
+            .tox/dev-${{ matrix.tox-env }}/*.xml
         # Use always() to always run this step to publish test results when there are test failures
         if: ${{ always() }}
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
         with:
-          files: .tox/pytest-${{ matrix.tox-env }}/unit-test-coverage.xml
-          flags: pytest-${{ matrix.tox-env }}
+          files: .tox/dev-${{ matrix.tox-env }}/unit-test-coverage.xml
+          flags: dev-${{ matrix.tox-env }}
 
   Integration-Test:
     runs-on: ubuntu-22.04
@@ -96,11 +96,11 @@ jobs:
       - name: Install tox
         run: python -m pip install tox==4.4.6
       - name: Run Integration Test
-        run: tox -vv -e pytest-py310 -- tests/integration --csv=.tox/pytest-py310/intg-test.csv
+        run: tox -vv -e dev-py310 -- tests/integration --csv=.tox/dev-py310/intg-test.csv
       - name: Upload artifacts
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: intg-test-results
-          path: .tox/pytest-py310/*.csv
+          path: .tox/dev-py310/*.csv
         # Use always() to always run this step to publish test results when there are test failures
         if: ${{ always() }}

.gitignore

@@ -33,6 +33,7 @@ otx_models/
 .mypy_cache
 .ruff_cache
 .coverage
+.tox
 
 *.jpg
 *.jpeg

pyproject.toml

@@ -39,7 +39,7 @@ dev = [
   "pre-commit==3.7.0",
   "addict",
 ]
-dev_timm = [
+val = [
   "timm==0.9.5",
   "onnx==1.14.1",
 ]

tox.ini

@@ -16,7 +16,12 @@ skip_install = true
 commands =
     pre-commit run --all-files
 
-[testenv:pytest-{py310, py311}]
-extras = dev,dev_timm
+[testenv:dev-{py310, py311}]
+extras = dev
+commands =
+    pytest -ra --showlocals {posargs:tests/}
+
+[testenv:val-{py310, py311}]
+extras = dev,val
 commands =
     pytest -ra --showlocals {posargs:tests/}