chore: use tox-uv to speed up dep install in ci (#887)

ComposioHQ · Dec 12, 2024 · 3f02820 · 3f02820
1 parent e0cf0f4
commit 3f02820
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 46 deletions.
diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml
@@ -59,11 +59,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get autoremove
-          sudo apt-get autoclean
-          pip install --user --upgrade setuptools
-          pip install tox==4.6.3
+          pip install 'tox>=4.21,<5' tox-uv
       - name: Code style check
         run: |
           tox -e black-diff
@@ -79,8 +75,6 @@ jobs:
       run:
         working-directory: ./python
     continue-on-error: False
-    needs:
-      - lock_check
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -100,32 +94,65 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install Dependencies
         run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get autoremove
-          sudo apt-get autoclean
-          pip install --user --upgrade setuptools
-          pip install tox==4.6.3
+          pip install 'tox>=4.21,<5' tox-uv
       - name: Unittests
         run: |
           export COMPOSIO_API_KEY=${{ secrets.COMPOSIO_API_KEY_STAGING }}
           export COMPOSIO_BASE_URL=${{ secrets.COMPOSIO_BASE_URL_STAGING }}
           export FLY_API_TOKEN=${{ secrets.FLY_API_TOKEN }}
           export E2B_API_KEY=${{ secrets.E2B_API_KEY_STAGING }}
 
-          tox -e test -- -m 'not e2e'
+          tox -e test -- -m 'not e2e and not swe'
       - if: matrix.python-version == '3.10'
         name: Upload test results to Codecov
         uses: codecov/test-results-action@v1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  swe:
+    defaults:
+      run:
+        working-directory: ./python
+    continue-on-error: False
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10"]
+    timeout-minutes: 20
+    concurrency:
+      group: 'test:swe:${{ matrix.python-version }}:${{ github.ref }}'
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Dependencies
+        run: |
+          pip install 'tox>=4.21,<5' tox-uv
+      - name: Unittests
+        run: |
+          export COMPOSIO_API_KEY=${{ secrets.COMPOSIO_API_KEY_STAGING }}
+          export COMPOSIO_BASE_URL=${{ secrets.COMPOSIO_BASE_URL_STAGING }}
+          export FLY_API_TOKEN=${{ secrets.FLY_API_TOKEN }}
+          export E2B_API_KEY=${{ secrets.E2B_API_KEY_STAGING }}
+
+          tox -e test -- -m 'swe'
+      - name: Upload test results to Codecov
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
   e2e:
     defaults:
       run:
         working-directory: ./python
     continue-on-error: False
-    needs:
-      - lock_check
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -154,11 +181,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install Dependencies
         run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get autoremove
-          sudo apt-get autoclean
-          pip install --user --upgrade setuptools
-          pip install tox==4.6.3
+          pip install 'tox>=4.21,<5' tox-uv
       - name: Build docker images
         run: |
           cd dockerfiles

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
@@ -85,10 +85,6 @@ jobs:
           COMPOSIO_BASE_URL: ${{ inputs.base_url || secrets.COMPOSIO_BASE_URL || 'https://backend.composio.dev/api' }}
           # TODO(@kaavee): Add Anthropic API key
         run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get autoremove
-          sudo apt-get autoclean
-
           python -m pip install --upgrade pip pipenv pytest
           python -m pip install plugins/${{matrix.package}}
           python -m pip install '.[all]'

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -31,10 +31,6 @@ jobs:
           COMPOSIO_BASE_URL: https://backend.composio.dev/api
           COMPOSIO_API_KEY: ${{secrets.COMPOSIO_API_KEY_RELEASE}}
         run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get autoremove
-          sudo apt-get autoclean
-
           # Setup release
           python -m pip install --upgrade pip
           python -m pip install .
@@ -91,10 +87,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Build ${{ matrix.package }}
         run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get autoremove
-          sudo apt-get autoclean
-          python -m pip install --upgrade pip
           pip install twine build
 
           # Build dist
@@ -128,10 +120,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Build
         run: |
-          sudo apt-get update --fix-missing
-          sudo apt-get autoremove
-          sudo apt-get autoclean
-          python -m pip install --upgrade pip
           pip install twine build
 
           # Build dist

diff --git a/python/pytest.ini b/python/pytest.ini
@@ -1,3 +1,4 @@
 [pytest]
 markers =
-    e2e: marks test which depend on external service as end-to-end 
+    e2e: marks tests which depend on external service as end-to-end 
+    swe: marks swebench tests which run on the remote docker workspace
diff --git a/python/swe/tests/test_benchmark.py b/python/swe/tests/test_benchmark.py
@@ -24,6 +24,7 @@ def mock_load_dataset():
         yield mock
 
 
+@pytest.mark.swe
 class TestIntegration:
     image_name = "composio/swe:testing"
 

diff --git a/python/tox.ini b/python/tox.ini
@@ -117,21 +117,21 @@ deps =
     tqdm==4.66.4
 commands =
     ; Install swekit
-    pip install -e swe/ --no-deps
+    uv pip install -e swe/ --no-deps
 
     ; TODO: Extract plugin tests separately
     ; Installing separately because of the dependency conflicts
-    pip install plugins/langchain --no-deps
+    uv pip install plugins/langchain --no-deps
 
     pytest -vvv -rfE --doctest-modules composio/ tests/ swe/tests --junitxml=junit.xml --cov=composio --cov=examples --cov=swe --cov-report=html --cov-report=xml --cov-report=term --cov-report=term-missing --cov-config=.coveragerc {posargs}
 
-    ; pip3 install plugins/autogen
-    ; pip3 install plugins/claude
-    ; pip3 install plugins/crew_ai
-    ; pip3 install plugins/griptape
-    ; pip3 install plugins/julep
-    ; pip3 install plugins/lyzr
-    ; pip3 install plugins/openai
+    ; uv pip install plugins/autogen
+    ; uv pip install plugins/claude
+    ; uv pip install plugins/crew_ai
+    ; uv pip install plugins/griptape
+    ; uv pip install plugins/julep
+    ; uv pip install plugins/lyzr
+    ; uv pip install plugins/openai
 
 ; Linter config