Merge pull request #2 from pola-rs/main

Keep up with the latest polars
lisphilar · Apr 26, 2024 · d1b479f · d1b479f
2 parents 2c9c3af + 9b0503a
commit d1b479f
Show file tree

Hide file tree

Showing 250 changed files with 5,866 additions and 5,132 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -41,39 +41,14 @@ jobs:
 
       - name: Create virtual environment
         run: |
-          python -m venv .venv
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv venv
           echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
+          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
 
       - name: Install Python dependencies
         working-directory: py-polars
-        run: pip install -r requirements-dev.txt
-
-      - name: Load benchmark data from cache
-        id: cache-data
-        uses: actions/cache/restore@v4
-        with:
-          path: py-polars/tests/benchmark/data/G1_1e7_1e2_5_0.csv
-          key: benchmark-data
-
-      - name: Set up R
-        if: steps.cache-data.outputs.cache-hit != 'true'
-        uses: r-lib/actions/setup-r@v2
-        with:
-          r-version: '4.3.3'
-
-      - name: Generate data
-        if: steps.cache-data.outputs.cache-hit != 'true'
-        working-directory: py-polars/tests/benchmark/data
-        run: |
-          Rscript -e 'install.packages("data.table", repos="https://cloud.r-project.org")'
-          Rscript groupby-datagen.R 1e7 1e2 5 0
-
-      - name: Save benchmark data in cache
-        if: github.ref_name == 'main'
-        uses: actions/cache/save@v4
-        with:
-          path: py-polars/tests/benchmark/data/G1_1e7_1e2_5_0.csv
-          key: ${{ steps.cache-data.outputs.cache-primary-key }}
+        run: uv pip install --compile-bytecode -r requirements-dev.txt
 
       - name: Set up Rust
         run: rustup show

diff --git a/.github/workflows/docs-global.yml b/.github/workflows/docs-global.yml
@@ -63,13 +63,15 @@ jobs:
 
       - name: Create virtual environment
         run: |
-          python -m venv .venv
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv venv
           echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
+          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
 
-      - name: Install dependencies
+      - name: Install Python dependencies
         run: |
-          pip install -r py-polars/requirements-dev.txt
-          pip install -r docs/requirements.txt
+          uv pip install -r py-polars/requirements-dev.txt
+          uv pip install -r docs/requirements.txt
 
       - name: Set up Rust
         run: rustup show

diff --git a/.github/workflows/docs-python.yml b/.github/workflows/docs-python.yml
@@ -32,17 +32,21 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
-          cache: pip
-          cache-dependency-path: py-polars/docs/requirements-docs.txt
+
+      - name: Create virtual environment
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv venv
+          echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
+          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
 
       - name: Install Python dependencies
         working-directory: py-polars/docs
-        run: pip install -r requirements-docs.txt
+        run: uv pip install -r requirements-docs.txt
 
       - name: Build Python documentation
         working-directory: py-polars/docs
         env:
-          SPHINXOPTS: -W --jobs=auto
           POLARS_VERSION: ${{ github.event.client_payload.tag || 'main' }}
         run: make html
 

diff --git a/.github/workflows/lint-global.yml b/.github/workflows/lint-global.yml
@@ -15,4 +15,4 @@ jobs:
       - name: Lint Markdown and TOML
         uses: dprint/[email protected]
       - name: Spell Check with Typos
-        uses: crate-ci/typos@v1.19.0
+        uses: crate-ci/typos@v1.20.10
diff --git a/.github/workflows/lint-python.yml b/.github/workflows/lint-python.yml
@@ -40,9 +40,6 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ['3.8', '3.12']
-    defaults:
-      run:
-        working-directory: py-polars
 
     steps:
       - uses: actions/checkout@v4
@@ -52,11 +49,20 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
+      - name: Create virtual environment
         run: |
-          pip install -r requirements-dev.txt
-          pip install -r requirements-lint.txt
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv venv
+          echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
+          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
+
+      - name: Install Python dependencies
+        working-directory: py-polars
+        run: |
+          uv pip install -r requirements-dev.txt
+          uv pip install -r requirements-lint.txt
 
       # Allow untyped calls for older Python versions
       - name: Run mypy
+        working-directory: py-polars
         run: mypy ${{ (matrix.python-version == '3.8') && '--allow-untyped-calls' || '' }}
diff --git a/.github/workflows/codecov.yml → .github/workflows/test-coverage.yml b/.github/workflows/codecov.yml → .github/workflows/test-coverage.yml
@@ -5,14 +5,14 @@ on:
     paths:
       - '**.rs'
       - '**.py'
-      - .github/workflows/codecov.yml
+      - .github/workflows/test-coverage.yml
   push:
     branches:
       - main
     paths:
       - '**.rs'
       - '**.py'
-      - .github/workflows/codecov.yml
+      - .github/workflows/test-coverage.yml
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -25,13 +25,15 @@ defaults:
 env:
   RUSTFLAGS: '-C instrument-coverage --cfg=coverage --cfg=coverage_nightly --cfg=trybuild_no_target'
   RUST_BACKTRACE: 1
-  LLVM_PROFILE_FILE: '/Users/runner/work/polars/polars/target/polars-%p-%3m.profraw'
+  LLVM_PROFILE_FILE: ${{ github.workspace }}/target/polars-%p-%3m.profraw
   CARGO_LLVM_COV: 1
   CARGO_LLVM_COV_SHOW_ENV: 1
-  CARGO_LLVM_COV_TARGET_DIR: '/Users/runner/work/polars/polars/target'
+  CARGO_LLVM_COV_TARGET_DIR: ${{ github.workspace }}/target
 
 jobs:
   coverage-rust:
+    # Running under ubuntu doesn't seem to work:
+    # https://github.com/pola-rs/polars/issues/14255
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
@@ -72,27 +74,36 @@ jobs:
         run: cargo llvm-cov report --lcov --output-path coverage-rust.lcov
 
       - name: Upload coverage report
-        uses: actions/upload-artifact@v4
+        uses: codecov/codecov-action@v4
         with:
-          name: coverage-rust
-          path: coverage-rust.lcov
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage-rust.lcov
+          root_dir: ${{ github.workspace }}
+          flags: rust
+          fail_ci_if_error: true
 
   coverage-python:
+    # Running under ubuntu doesn't seem to work:
+    # https://github.com/pola-rs/polars/issues/14255
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Create virtual environment
         run: |
-          python -m venv .venv
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv venv
           echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
+          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
 
-      - name: Install dependencies
-        run: pip install -r py-polars/requirements-dev.txt
+      - name: Install Python dependencies
+        working-directory: py-polars
+        run: uv pip install --compile-bytecode -r requirements-dev.txt
 
       - name: Set up Rust
         run: rustup component add llvm-tools-preview
@@ -113,39 +124,33 @@ jobs:
 
       - name: Run Python tests
         working-directory: py-polars
-        run: pytest --cov -n auto --dist loadgroup -m "not benchmark and not docs" --cov-report xml:main.xml
+        run: pytest --cov -n auto --dist loadgroup -m "not release and not benchmark and not docs" --cov-report xml:main.xml
         continue-on-error: true
 
       - name: Run Python tests - async reader
         working-directory: py-polars
         env:
           POLARS_FORCE_ASYNC: 1
-        run: pytest --cov -m "not benchmark and not docs" tests/unit/io/ --cov-report xml:async.xml
+        run: pytest --cov -n auto --dist loadgroup -m "not release and not benchmark and not docs" tests/unit/io/ --cov-report xml:async.xml
         continue-on-error: true
 
       - name: Report Rust coverage
         run: cargo llvm-cov report --lcov --output-path coverage-python.lcov
 
-      - name: Upload coverage reports
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage-python
-          path: |
-            coverage-python.lcov
-            py-polars/main.xml
-            py-polars/async.xml
-
-  upload-coverage:
-    needs: [coverage-rust, coverage-python]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download coverage files
-        uses: actions/download-artifact@v4
+      - name: Upload coverage reports - Python
+        uses: codecov/codecov-action@v4
         with:
-          merge-multiple: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: py-polars/main.xml,py-polars/async.xml
+          root_dir: ${{ github.workspace }}
+          flags: python
+          fail_ci_if_error: true
 
-      - name: Upload coverage information
+      - name: Upload coverage report - Rust
         uses: codecov/codecov-action@v4
         with:
-          files: coverage-rust.lcov,coverage-python.lcov,main.xml,async.xml
           token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage-python.lcov
+          root_dir: ${{ github.workspace }}
+          flags: rust
+          fail_ci_if_error: true
diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml
@@ -60,9 +60,12 @@ jobs:
         run: |
           python -m venv .venv
           echo "$GITHUB_WORKSPACE/py-polars/.venv/$BIN" >> $GITHUB_PATH
+          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/py-polars/.venv" >> $GITHUB_ENV
 
-      - name: Install dependencies
-        run: pip install -r requirements-dev.txt
+      - name: Install Python dependencies
+        run: |
+          pip install uv
+          uv pip install --compile-bytecode -r requirements-dev.txt
 
       - name: Set up Rust
         run: rustup show
@@ -89,13 +92,13 @@ jobs:
           # Currently skipped due to performance issues in coverage:
           # https://github.com/nedbat/coveragepy/issues/1665
           COV: ${{ !(matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12') && '--cov' || '--no-cov' }}
-        run: pytest $COV -n auto --dist loadgroup -m "not benchmark and not docs"
+        run: pytest $COV -n auto --dist loadgroup -m "not release and not benchmark and not docs"
 
       - name: Run tests async reader tests
         if: github.ref_name != 'main' && matrix.os != 'windows-latest'
         env:
           POLARS_FORCE_ASYNC: 1
-        run: pytest -m "not benchmark and not docs" tests/unit/io/
+        run: pytest -m "not release and not benchmark and not docs" tests/unit/io/
 
       - name: Check import without optional dependencies
         if: github.ref_name != 'main' && matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest'