diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3c33bfa942..c1e6627e0b 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -5,104 +5,64 @@ on: branches: - master pull_request: - branches: - - master - - docs schedule: - cron: "0 2 * * 1-5" -# to make bash use ~/.bashrc, see https://github.com/marketplace/actions/setup-miniconda#example-5-custom-installer -defaults: - run: - shell: bash -l {0} - concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +defaults: + run: + shell: bash + jobs: main: - env: - CACHE_NUMBER_COMPILE: 6 - MAMBA_ROOT_PREFIX: /dev/shm/mambaroot runs-on: ${{ matrix.os }} strategy: fail-fast: false - max-parallel: 12 matrix: - os: [ubuntu-latest, windows-latest, macOS-12] - python-version: [3.8, 3.9, "3.10", "3.11", "3.12"] + os: [ubuntu-latest, windows-latest, macos-13] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - name: disk space - if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'ubuntu-latest') }} - run: | - df -h - - uses: maxim-lobanov/setup-xcode@v1 - # alternative would be to upgrade tapi to 1100.0.11, we can possibly remove this in the future - if: matrix.os == 'macOS-12' - with: - xcode-version: "13.2.1" - uses: actions/checkout@v2 with: submodules: true - - uses: actions/checkout@v2 - if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'ubuntu-latest') }} - with: - repository: vaexio/vaex-enterprise - token: ${{ secrets.PAT_PULL_ENTERPRISE }} - path: vaex-enterprise - - name: install micromamba - uses: mamba-org/setup-micromamba@v1 + + - uses: actions/setup-python@v5 with: - micromamba-version: 1.5.8-0 - environment-name: vaex-dev - environment-file: ci/conda-env.yml - create-args: >- - python=${{ matrix.python-version }} - - name: Extra non-windows installs + python-version: ${{ matrix.python-version }} + + - run: python -m pip install --upgrade pip setuptools wheel uv + + # - uses: actions/checkout@v2 + # if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'ubuntu-latest') }} + # with: + # repository: vaexio/vaex-enterprise + # token: ${{ secrets.PAT_PULL_ENTERPRISE }} + # path: vaex-enterprise + + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v2 + + - name: Setup PCRE (Nix-only) if: matrix.os != 'windows-latest' - run: | - echo "not needed for now" - # - name: Extra windows installs - # if: matrix.os == 'windows-latest' - # run: | - # # not sure why, but the conda env is strange (old dask) if we put this constraint in the conda solver - # # also might solve scipy issues - # micromamba update scipy ipython dask - # micromamba install scipy=1.7.1 - # - name: free disk space - # if: matrix.os == 'ubuntu-latest' - # run: | - # sudo swapoff -a - # sudo rm -f /swapfile - # sudo apt clean - # docker rmi $(docker image ls -aq) - # df -h + run: sudo -E bash bin/install_pcre.sh + - name: Install OpenMP runtime (Mac-only) - if: matrix.os == 'macOS-12' - run: | - brew install libomp - - name: Cache compiled binaries - # this fails for this combination, leading to binaries filled with 0's - if: matrix.os != 'macOS-12' - id: cache-compiled-binaries - uses: actions/cache@v2 - with: - path: | - packages/vaex-core/build/lib*/vaex/vaexfast* - packages/vaex-core/build/lib*/vaex/super* - key: ${{ runner.OS }}-${{ matrix.python-version }}-${{ hashFiles('packages/vaex-core/src/*') }}-${{ env.CACHE_NUMBER_COMPILE }} - - name: Fix cache timestamp - run: | - touch packages/vaex-core/build/lib*/vaex/vaexfast* || echo "ok to fail" - touch packages/vaex-core/build/lib*/vaex/super* || echo "ok to fail" - ls -alh packages/vaex-core/build/lib*/vaex/ || echo "ok to fail" + if: startswith(matrix.os, 'macos') + run: brew install libomp + + - name: Copy dll (Windows-only) + if: (matrix.os == 'windows-latest') + uses: ./ci/actions/windll + - name: Install vaex - run: | - pip install myst_parser - pip install -e . -v - # ./ci/03-install-vaex.sh micromamba + run: uv pip install -v --system .[ci] + + - run: uv pip freeze + # - name: Install vaex-enterprise # if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'ubuntu-latest') }} # run: | @@ -110,26 +70,7 @@ jobs: # micromamba install -c conda-forge distributed # pip install ray # pip install -e . - - name: Copy dll - if: (matrix.os == 'windows-latest') - uses: ./ci/actions/windll - - name: Build Python packages - run: | - mkdir dist - (cd packages/vaex-core && python setup.py bdist_wheel); cp packages/vaex-core/dist/* dist - (cd packages/vaex-graphql && python setup.py bdist_wheel); cp packages/vaex-graphql/dist/* dist - (cd packages/vaex-jupyter && python setup.py bdist_wheel); cp packages/vaex-jupyter/dist/* dist - (cd packages/vaex-ml && python setup.py bdist_wheel); cp packages/vaex-ml/dist/* dist - (cd packages/vaex-viz && python setup.py bdist_wheel); cp packages/vaex-viz/dist/* dist - (cd packages/vaex-astro && python setup.py bdist_wheel); cp packages/vaex-astro/dist/* dist - (cd packages/vaex-hdf5 && python setup.py bdist_wheel); cp packages/vaex-hdf5/dist/* dist - (cd packages/vaex-server && python setup.py bdist_wheel); cp packages/vaex-server/dist/* dist - (cd packages/vaex-meta && python setup.py bdist_wheel); cp packages/vaex-server/dist/* dist - - name: Upload builds - uses: actions/upload-artifact@v2 - with: - name: dist-${{ github.run_number }}-${{ matrix.python-version }}-${{ matrix.os }} - path: ./dist + # - name: Lint with flake8 # run: | # echo "soon" @@ -138,32 +79,30 @@ jobs: # # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide # # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Install pytest-asyncio correct version - if: matrix.python-version != '3.10' - run: | - # 3.10 doesn't seem to work - pip install "pytest-asyncio<0.14" + - name: Test with pytest run: | ./ci/04-run-test-suite.sh + - name: Test with pytest (with cache on) - if: matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest' + if: matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' run: | VAEX_CACHE_RESULTS=1 ./ci/04-run-test-suite.sh + - name: Check ml spec - # no catboost for py39 and py37, xgboost fails on 3.6 - if: matrix.python-version != '3.9' && matrix.os != 'windows-latest' run: | python -m vaex.ml.spec packages/vaex-ml/vaex/ml/spec_new.json diff packages/vaex-ml/vaex/ml/spec_new.json packages/vaex-ml/vaex/ml/spec.json + # - name: Test with pytest (vaex-enterprise) # if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'ubuntu-latest') }} # run: | # py.test vaex-enterprise/tests --timeout=1000 + - name: Test notebooks if: matrix.os != 'windows-latest' - run: | - ./ci/05-run-notebooks.sh + run: ./ci/05-run-notebooks.sh + - name: Authenticate Google Cloud Platform if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && !((matrix.os == 'windows-latest')) }} uses: google-github-actions/setup-gcloud@v0 @@ -171,74 +110,17 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID_VAEX }} service_account_key: ${{ secrets.GCP_SA_KEY_VAEX }} export_default_credentials: true + - name: Test vaex-contrib # do not run in a PR from someone else, skip windows - if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && !((matrix.os == 'windows-latest')) }} + if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os != 'windows-latest') }} env: PROJECT_ID: ${{ secrets.GCP_PROJECT_ID_VAEX }} - run: | - pip install -e 'packages/vaex-contrib[gcp]' - ./ci/06-run-contrib-tests.sh - - install: - runs-on: ${{ matrix.os }} - needs: [main] - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest, macOS-12] - python-version: [3.8, 3.9] - - steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 - with: - name: dist-${{ github.run_number }}-${{ matrix.python-version }}-${{ matrix.os }} - path: ./dist - - - name: Install micromamba - uses: mamba-org/setup-micromamba@v1 - with: - micromamba-version: 1.5.6-0 - environment-name: vaex-test - environment-file: ci/conda-base-minimal.yml - create-args: >- - python=${{ matrix.python-version }} - pcre - rich - # it shouldn't be needed to use pcre, maybe we should make the wheels similar to how we make distributions - - name: Cache pip - uses: actions/cache@v2 - env: - # Increase this value to reset cache - CACHE_NUMBER: 0 - with: - path: | - ~/.cache/pip - key: - pip-${{ runner.os }}-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }} - - - name: Install core packages - run: | - pip install -vv dist/vaex*core* dist/vaex*hdf5* - - - name: Test core import - run: | - python -c "import vaex; df = vaex.example()" - - - name: Install all packages - run: | - pip install -vv dist/* - - - name: Test import - run: | - python -c "import vaex; df = vaex.example()" + run: ./ci/06-run-contrib-tests.sh - name: Test comand line run: | vaex convert ~/.vaex/data/helmi-dezeeuw-2000-FeH-v2-10percent.hdf5 test.parquet - pip install rich VAEX_PROGRESS_TYPE=rich vaex convert ~/.vaex/data/helmi-dezeeuw-2000-FeH-v2-10percent.hdf5 test.parquet vaex settings yaml vaex settings md @@ -246,7 +128,7 @@ jobs: - name: Test server if: matrix.os != 'windows-latest' run: | - vaex server --add-example --port 9999& + vaex server --add-example --port 9999 & - name: Wait for Vaex server if: matrix.os != 'windows-latest' diff --git a/.github/workflows/wheel-universal.yml b/.github/workflows/wheel-universal.yml index 3be4f23408..48fb563c30 100644 --- a/.github/workflows/wheel-universal.yml +++ b/.github/workflows/wheel-universal.yml @@ -43,7 +43,7 @@ jobs: - name: Build vaex (meta) run: | - (cp README.md packages/vaex-meta/ && cd packages/vaex-meta && python setup.py sdist bdist_wheel); cp packages/vaex-meta/dist/* dist + (cp README.md packages/vaex/ && cd packages/vaex && python setup.py sdist bdist_wheel); cp packages/vaex/dist/* dist - name: Publish a Python distribution to PyPI if: startsWith(github.ref, 'refs/tags/meta') diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 471f074fe8..4341992e1f 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -33,4 +33,7 @@ submodules: # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - - requirements: requirements_rtd.txt + - method: pip + path: . + extra_requirements: + - ci diff --git a/.releash.py b/.releash.py index 1aa9745f3e..5d52b0c6b9 100644 --- a/.releash.py +++ b/.releash.py @@ -10,7 +10,7 @@ core.version_source = version_core core.version_targets.append(VersionTarget(core, '{path}/vaex/core/_version.py')) core.version_targets.append(VersionTargetReplace(core, [ - 'packages/vaex-meta/setup.py', + 'packages/vaex/setup.py', ])) @@ -32,7 +32,7 @@ def add_version_replace(package): def add_meta_version_replace(package): # always bump the meta package package.version_targets.append(VersionTargetReplace(package, [ - 'packages/vaex-meta/setup.py', + 'packages/vaex/setup.py', ], pattern='{name}(?P[^0-9]*)' + str(package.version_source), )) add_version_replace(core) @@ -43,7 +43,7 @@ def add_meta_version_replace(package): #core.release_targets.append(gitpush) core.release_targets.append(ReleaseTargetCondaForge(core, '../feedstocks/vaex-core-feedstock')) -packages = ['vaex-core', 'vaex-meta', 'vaex-viz', 'vaex-hdf5', 'vaex-server', 'vaex-astro', 'vaex-jupyter', 'vaex-ml', 'vaex-graphql', 'vaex-contrib'] +packages = ['vaex-core', 'vaex', 'vaex-viz', 'vaex-hdf5', 'vaex-server', 'vaex-astro', 'vaex-jupyter', 'vaex-ml', 'vaex-graphql', 'vaex-contrib'] names = [k[5:] for k in packages[1:]] for name in names: diff --git a/asv.conf.json b/asv.conf.json deleted file mode 100644 index c0496bfffc..0000000000 --- a/asv.conf.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "version": 1, - "project": "vaex", - "project_url": "https://vaex.io/", - "repo": ".", - "environment_type": "conda", - "show_commit_url": "http://github.com/vaexio/vaex/commit/", - "env_dir": ".asv/env", - "results_dir": ".asv/results", - "html_dir": ".asv/html", - "conda_channels": [ - "conda-forge", - "defaults" - ], - "pythons": [ - "3.7" - ], - "matrix": { - "setuptools": [ - "42.0.2" - ], - "certifi": [ - "2019.6.16" - ], - "pip": [], - "numpy": [], - "scipy": [], - "pyqt": [], - "matplotlib": [], - "pyopengl": [], - "h5py": [], - "numexpr": [], - "astropy": [], - "tornado": [], - "cython": [], - "runipy": [], - "pandas": [], - "pytest": [], - "numba": [], - "pyarrow": [ - "0.15.1" - ], - "graphviz": [], - "python-graphviz": [], - "pcre": [], - "catboost": [], - "libcxx": [ - "9.0.0" - ], - "notebook": [], - "scikit-learn": [], - "lightgbm": [], - "py-xgboost": [], - "bqplot": [ - "0.12.6" - ], - "ipyleaflet": [ - "0.12.3" - ], - "ipympl": [ - "0.5.6" - ], - "ipyvolume": [ - "0.5.2" - ], - "graphene": [ - "2.1.8" - ], - "graphene-tornado": [ - "2.6.1" - ], - "pip+python-dateutil": [ - "2.8.0" - ], - "pip+aplus": [], - "pip+futures": [ - "3.1.1" - ], - "pip+future": [ - "0.18.2" - ], - "pip+pyyaml": [], - "pip+cachetools": [], - "pip+progressbar2": [], - "pip+psutil": [ - "5.6.7" - ], - "pip+pillow": [], - "pip+requests": [], - "pip+s3fs": [ - "0.2.2" - ], - "pip+ipyvuetify": [] - }, - "install_timeout": 1800, - "install_command": [ - "in-dir={env_dir} python -mpip install {wheel_file}", - "in-dir={env_dir} pip install {build_dir} --verbose" - ], - "build_command": [ - "in-dir={build_dir}/packages/vaex-core python setup.py build", - "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}/packages/vaex-core" - ], - "build_cache_size": 5 -} diff --git a/ci/04-run-test-suite.sh b/ci/04-run-test-suite.sh index 504d1a5a13..0c0e90efa2 100755 --- a/ci/04-run-test-suite.sh +++ b/ci/04-run-test-suite.sh @@ -3,17 +3,16 @@ set -e if [ -f ${HOME}/.bashrc ]; then source ${HOME}/.bashrc -else +elif [ -f ${HOME}/.bash_profile ]; then source ${HOME}/.bash_profile fi export VAEX_SERVER_OVERRIDE='{"dataframe.vaex.io":"dataframe-dev.vaex.io"}' -pytest tests\ - packages/vaex-core/vaex/datatype_test.py\ - packages/vaex-core/vaex/file/\ - packages/vaex-core/vaex/test/dataset.py::TestDataset\ - --doctest-modules\ - packages/vaex-core/vaex/datatype.py\ - packages/vaex-core/vaex/utils.py\ - packages/vaex-core/vaex/struct.py\ - packages/vaex-core/vaex/groupby.py\ - --timeout=1000 +python -m pytest --pyargs --doctest-modules --timeout=1000\ + tests\ + vaex.datatype_test\ + vaex.file\ + vaex.test.dataset::TestDataset\ + vaex.datatype\ + vaex.utils\ + vaex.struct\ + vaex.groupby diff --git a/ci/conda-base-minimal.yml b/ci/conda-base-minimal.yml deleted file mode 100644 index d80a0e2f94..0000000000 --- a/ci/conda-base-minimal.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: vaex-dev -channels: - - conda-forge -dependencies: -- numpy<1.21 -- pandas -- scipy diff --git a/ci/conda-env-extra.yml b/ci/conda-env-extra.yml deleted file mode 100644 index d8e29b3e7f..0000000000 --- a/ci/conda-env-extra.yml +++ /dev/null @@ -1 +0,0 @@ -catboost diff --git a/ci/conda-env-nightlies.yml b/ci/conda-env-nightlies.yml deleted file mode 100644 index 19d8363071..0000000000 --- a/ci/conda-env-nightlies.yml +++ /dev/null @@ -1 +0,0 @@ -pyarrow diff --git a/ci/conda-env-notebooks.yml b/ci/conda-env-notebooks.yml deleted file mode 100644 index 66c0b11ef2..0000000000 --- a/ci/conda-env-notebooks.yml +++ /dev/null @@ -1,2 +0,0 @@ -plotly -ipyvolume==0.6.0a6 diff --git a/ci/conda-env.yml b/ci/conda-env.yml deleted file mode 100644 index 10b0f18fca..0000000000 --- a/ci/conda-env.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: vaex-dev -channels: - - msys2 - - conda-forge - - numba/label/dev -dependencies: -- aplus -- astropy -- blake3 -- cython -- cachetools -- catboost -- dask<2024.2.0 -- diskcache -- filelock -- fsspec<2022.2.0 -- gcsfs -- geopandas -- graphviz -- joblib<1.3 # 1.3 is broken for py36 -- h5py -- httpx # for testing with starlette/fastapi -- ipyvolume=0.6.0a6 -- matplotlib-base -- nest-asyncio<1.5.2 -- notebook -- numba -- numpy<1.26.4 -- pandas -- pcre -- pip==20.3.4 -- plotly -- python-annoy -- pyarrow>=5.0.0 -- pyqt -- pytest -- pytest-asyncio -- pytest-mpl -- pytest-timeout -- python-graphviz -- py-xgboost -- pyyaml -- rich -- s3fs>=2021.8.0 -- scikit-learn -- scipy -- tabulate -- tornado -- uvicorn<0.16 -- xarray<2022.6.0 -# currently not using this, since the test that requires this is flakey -# - myst-parser<0.18 # 0.18 breaks our test, missing main -- pytz # for some reason, pip has trouble resolving this, and the libraries below -- python-utils -- progressbar2 -- zipp<3.16.0 -- pip: - - lightgbm>=4.0.0 \ No newline at end of file diff --git a/dodo.py b/dodo.py index 2dccf0c355..7e71a921e4 100644 --- a/dodo.py +++ b/dodo.py @@ -23,7 +23,7 @@ def action(targets): return { 'actions': [action], 'targets': ["binder/requirements.txt"], - 'file_dep': ['packages/vaex-meta/vaex/meta/_version.py'] + 'file_dep': ['packages/vaex/vaex/meta/_version.py'] } @@ -35,6 +35,6 @@ def action(targets): return { 'actions': [action], - 'targets': ["packages/vaex-meta/README.md"], + 'targets': ["packages/vaex/README.md"], 'file_dep': ['README.md'] } diff --git a/misc/conda/aplus/bld.bat b/misc/conda/aplus/bld.bat deleted file mode 100644 index 87b1481d74..0000000000 --- a/misc/conda/aplus/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/misc/conda/aplus/build.sh b/misc/conda/aplus/build.sh deleted file mode 100644 index 4d7fc032b8..0000000000 --- a/misc/conda/aplus/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install - -# Add more build steps here, if they are necessary. - -# See -# http://docs.continuum.io/conda/build.html -# for a list of environment variables that are set during the build process. diff --git a/misc/conda/aplus/meta.yaml b/misc/conda/aplus/meta.yaml deleted file mode 100644 index 66682292f2..0000000000 --- a/misc/conda/aplus/meta.yaml +++ /dev/null @@ -1,60 +0,0 @@ -package: - name: aplus - version: "0.11.0" - -source: - fn: aplus-0.11.0.tar.gz - url: https://files.pythonhosted.org/packages/9a/25/58f13028bc43c2d219e32402995275ed577cb02da7e43876a4020727c66a/aplus-0.11.0.tar.gz - md5: cb3009b753154aea6bbf3c6088b2a509 -# patches: - # List any patch files here - # - fix.patch - -# build: - # noarch_python: True - # preserve_egg_dir: True - # entry_points: - # Put any entry points (scripts to be generated automatically) here. The - # syntax is module:function. For example - # - # - aplus = aplus:main - # - # Would create an entry point called aplus that calls aplus.main() - - - # If this is a new build for the same version, increment the build - # number. If you do not include this key, it defaults to 0. - # number: 1 - -requirements: - build: - - python - - run: - - python - -test: - # Python imports - imports: - - aplus - - # commands: - # You can put test commands to be run here. Use this to test that the - # entry points work. - - - # You can also put a file called run_test.py in the recipe that will be run - # at test time. - - # requires: - # Put any additional test requirements here. For example - # - nose - -about: - #home: The package home page - license: UNKNOWN - summary: 'UNKNOWN' - -# See -# http://docs.continuum.io/conda/build.html for -# more information about meta.yaml diff --git a/misc/conda/attrdict/bld.bat b/misc/conda/attrdict/bld.bat deleted file mode 100644 index 87b1481d74..0000000000 --- a/misc/conda/attrdict/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/misc/conda/attrdict/build.sh b/misc/conda/attrdict/build.sh deleted file mode 100644 index 4d7fc032b8..0000000000 --- a/misc/conda/attrdict/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install - -# Add more build steps here, if they are necessary. - -# See -# http://docs.continuum.io/conda/build.html -# for a list of environment variables that are set during the build process. diff --git a/misc/conda/attrdict/meta.yaml b/misc/conda/attrdict/meta.yaml deleted file mode 100644 index 09ad147433..0000000000 --- a/misc/conda/attrdict/meta.yaml +++ /dev/null @@ -1,65 +0,0 @@ -package: - name: attrdict - version: "2.0.0" - -source: - fn: attrdict-2.0.0.tar.gz - url: https://files.pythonhosted.org/packages/35/bb/bac3e42ae04bc082c28cd8186bfb5b50fb240a4f7419f876c683125ccc8b/attrdict-2.0.0.tar.gz - md5: 8a7c1a4e737fe9e2b2b8844c0f7746f8 -# patches: - # List any patch files here - # - fix.patch - -# build: - # noarch_python: True - # preserve_egg_dir: True - # entry_points: - # Put any entry points (scripts to be generated automatically) here. The - # syntax is module:function. For example - # - # - attrdict = attrdict:main - # - # Would create an entry point called attrdict that calls attrdict.main() - - - # If this is a new build for the same version, increment the build - # number. If you do not include this key, it defaults to 0. - # number: 1 - -requirements: - build: - - python - - setuptools - - six - - run: - - python - - six - -test: - # Python imports - imports: - - attrdict - - # commands: - # You can put test commands to be run here. Use this to test that the - # entry points work. - - - # You can also put a file called run_test.py in the recipe that will be run - # at test time. - - requires: - - coverage - - nose >=1.0 - # Put any additional test requirements here. For example - # - nose - -about: - home: https://github.com/bcj/AttrDict - license: MIT License - summary: 'A dict with attribute-style access' - -# See -# http://docs.continuum.io/conda/build.html for -# more information about meta.yaml diff --git a/misc/conda/cachetools/bld.bat b/misc/conda/cachetools/bld.bat deleted file mode 100644 index 87b1481d74..0000000000 --- a/misc/conda/cachetools/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/misc/conda/cachetools/build.sh b/misc/conda/cachetools/build.sh deleted file mode 100644 index 4d7fc032b8..0000000000 --- a/misc/conda/cachetools/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install - -# Add more build steps here, if they are necessary. - -# See -# http://docs.continuum.io/conda/build.html -# for a list of environment variables that are set during the build process. diff --git a/misc/conda/cachetools/meta.yaml b/misc/conda/cachetools/meta.yaml deleted file mode 100644 index f4520bc9d2..0000000000 --- a/misc/conda/cachetools/meta.yaml +++ /dev/null @@ -1,61 +0,0 @@ -package: - name: cachetools - version: "1.1.6" - -source: - fn: cachetools-1.1.6.tar.gz - url: https://files.pythonhosted.org/packages/ba/00/b0ec69e21142cd838b2383a7881cf18368e35847cb66f908c8f25bcbaafc/cachetools-1.1.6.tar.gz - md5: 387d7f34effd9335ae55bd0762e77bfa -# patches: - # List any patch files here - # - fix.patch - -# build: - # noarch_python: True - # preserve_egg_dir: True - # entry_points: - # Put any entry points (scripts to be generated automatically) here. The - # syntax is module:function. For example - # - # - cachetools = cachetools:main - # - # Would create an entry point called cachetools that calls cachetools.main() - - - # If this is a new build for the same version, increment the build - # number. If you do not include this key, it defaults to 0. - # number: 1 - -requirements: - build: - - python - - setuptools - - run: - - python - -test: - # Python imports - imports: - - cachetools - - # commands: - # You can put test commands to be run here. Use this to test that the - # entry points work. - - - # You can also put a file called run_test.py in the recipe that will be run - # at test time. - - # requires: - # Put any additional test requirements here. For example - # - nose - -about: - home: https://github.com/tkem/cachetools - license: MIT License - summary: 'Extensible memoizing collections and decorators' - -# See -# http://docs.continuum.io/conda/build.html for -# more information about meta.yaml diff --git a/misc/conda/jprops/bld.bat b/misc/conda/jprops/bld.bat deleted file mode 100644 index 87b1481d74..0000000000 --- a/misc/conda/jprops/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/misc/conda/jprops/build.sh b/misc/conda/jprops/build.sh deleted file mode 100644 index 4d7fc032b8..0000000000 --- a/misc/conda/jprops/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install - -# Add more build steps here, if they are necessary. - -# See -# http://docs.continuum.io/conda/build.html -# for a list of environment variables that are set during the build process. diff --git a/misc/conda/jprops/meta.yaml b/misc/conda/jprops/meta.yaml deleted file mode 100644 index f868524422..0000000000 --- a/misc/conda/jprops/meta.yaml +++ /dev/null @@ -1,60 +0,0 @@ -package: - name: jprops - version: "1.0" - -source: - fn: jprops-1.0.tar.gz - url: https://files.pythonhosted.org/packages/1f/57/42e0088e3607b2d157bb039ba5c19536010f3d505871573cb28020e1e1d6/jprops-1.0.tar.gz - md5: d150084738d03d8046418a0f79879821 -# patches: - # List any patch files here - # - fix.patch - -# build: - # noarch_python: True - # preserve_egg_dir: True - # entry_points: - # Put any entry points (scripts to be generated automatically) here. The - # syntax is module:function. For example - # - # - jprops = jprops:main - # - # Would create an entry point called jprops that calls jprops.main() - - - # If this is a new build for the same version, increment the build - # number. If you do not include this key, it defaults to 0. - # number: 1 - -requirements: - build: - - python - - setuptools - - run: - - python - -# test: - # Python imports - # imports: - - # commands: - # You can put test commands to be run here. Use this to test that the - # entry points work. - - - # You can also put a file called run_test.py in the recipe that will be run - # at test time. - - # requires: - # Put any additional test requirements here. For example - # - nose - -about: - home: http://mgood.github.com/jprops/ - license: BSD - summary: 'Parser for Java .properties files' - -# See -# http://docs.continuum.io/conda/build.html for -# more information about meta.yaml diff --git a/misc/conda/kapteyn/bld.bat b/misc/conda/kapteyn/bld.bat deleted file mode 100644 index 87b1481d74..0000000000 --- a/misc/conda/kapteyn/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/misc/conda/kapteyn/build.sh b/misc/conda/kapteyn/build.sh deleted file mode 100644 index 4d7fc032b8..0000000000 --- a/misc/conda/kapteyn/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install - -# Add more build steps here, if they are necessary. - -# See -# http://docs.continuum.io/conda/build.html -# for a list of environment variables that are set during the build process. diff --git a/misc/conda/kapteyn/meta.yaml b/misc/conda/kapteyn/meta.yaml deleted file mode 100644 index 03e176960e..0000000000 --- a/misc/conda/kapteyn/meta.yaml +++ /dev/null @@ -1,64 +0,0 @@ -package: - name: kapteyn - version: "2.3" - -source: - fn: kapteyn-2.3.tar.gz - url: https://files.pythonhosted.org/packages/0a/c3/4ceffdeb87918ade3dc9649e100d3b0be0fdc38b5105324b17686a3e0e48/kapteyn-2.3.tar.gz - md5: 9abb373995d1b683936c48485a41c2b4 -# patches: - # List any patch files here - # - fix.patch - -# build: - # noarch_python: True - # preserve_egg_dir: True - # entry_points: - # Put any entry points (scripts to be generated automatically) here. The - # syntax is module:function. For example - # - # - kapteyn = kapteyn:main - # - # Would create an entry point called kapteyn that calls kapteyn.main() - - - # If this is a new build for the same version, increment the build - # number. If you do not include this key, it defaults to 0. - # number: 1 - -requirements: - build: - - python - - setuptools - - six - - numpy - - cython - run: - - python - - numpy - - six -test: - # Python imports - imports: - - kapteyn - - kapteyn.wcs - # commands: - # You can put test commands to be run here. Use this to test that the - # entry points work. - - - # You can also put a file called run_test.py in the recipe that will be run - # at test time. - - # requires: - # Put any additional test requirements here. For example - # - nose - -about: - home: http://www.astro.rug.nl/software/kapteyn/ - license: BSD License or GNU Library or Lesser General Public License (LGPL) - summary: 'Kapteyn Package: Python modules for astronomical applications' - -# See -# http://docs.continuum.io/conda/build.html for -# more information about meta.yaml diff --git a/misc/conda/layeredconfig/bld.bat b/misc/conda/layeredconfig/bld.bat deleted file mode 100644 index 87b1481d74..0000000000 --- a/misc/conda/layeredconfig/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/misc/conda/layeredconfig/build.sh b/misc/conda/layeredconfig/build.sh deleted file mode 100644 index 4d7fc032b8..0000000000 --- a/misc/conda/layeredconfig/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install - -# Add more build steps here, if they are necessary. - -# See -# http://docs.continuum.io/conda/build.html -# for a list of environment variables that are set during the build process. diff --git a/misc/conda/layeredconfig/meta.yaml b/misc/conda/layeredconfig/meta.yaml deleted file mode 100644 index 6bbaefb55c..0000000000 --- a/misc/conda/layeredconfig/meta.yaml +++ /dev/null @@ -1,65 +0,0 @@ -package: - name: layeredconfig - version: "0.1.0" - -source: - fn: layeredconfig-0.1.0.tar.gz - url: https://files.pythonhosted.org/packages/d8/a3/9818f77bda2e9c1a490f353d6f9bd5a1d301e20c7b7172600b0a69829291/layeredconfig-0.1.0.tar.gz - md5: c19b10f505fa39f4656ac64072c5619d -# patches: - # List any patch files here - # - fix.patch - -# build: - # noarch_python: True - # preserve_egg_dir: True - # entry_points: - # Put any entry points (scripts to be generated automatically) here. The - # syntax is module:function. For example - # - # - layeredconfig = layeredconfig:main - # - # Would create an entry point called layeredconfig that calls layeredconfig.main() - - - # If this is a new build for the same version, increment the build - # number. If you do not include this key, it defaults to 0. - # number: 1 - -requirements: - build: - - python - - setuptools - - six - - pyyaml - - run: - - python - - six - - pyyaml - - requests -test: - # Python imports - imports: - - layeredconfig - - # commands: - # You can put test commands to be run here. Use this to test that the - # entry points work. - - - # You can also put a file called run_test.py in the recipe that will be run - # at test time. - - # requires: - # Put any additional test requirements here. For example - # - nose - -about: - home: https://github.com/staffanm/layeredconfig - license: BSD License - summary: 'Manages configuration coming from config files, environment variables, command line arguments, code defaults or other sources' - -# See -# http://docs.continuum.io/conda/build.html for -# more information about meta.yaml diff --git a/misc/conda/progressbar2/bld.bat b/misc/conda/progressbar2/bld.bat deleted file mode 100644 index 87b1481d74..0000000000 --- a/misc/conda/progressbar2/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/misc/conda/progressbar2/build.sh b/misc/conda/progressbar2/build.sh deleted file mode 100644 index 4d7fc032b8..0000000000 --- a/misc/conda/progressbar2/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install - -# Add more build steps here, if they are necessary. - -# See -# http://docs.continuum.io/conda/build.html -# for a list of environment variables that are set during the build process. diff --git a/misc/conda/progressbar2/meta.yaml b/misc/conda/progressbar2/meta.yaml deleted file mode 100644 index 9680ebbda1..0000000000 --- a/misc/conda/progressbar2/meta.yaml +++ /dev/null @@ -1,61 +0,0 @@ -package: - name: progressbar2 - version: "3.6.0" - -source: - fn: progressbar2-3.6.0.tar.gz - url: https://files.pythonhosted.org/packages/c0/06/b85e78d16a802ed3d7b100762f45a377bb57ee71cb66b3d8b0aec98f4039/progressbar2-3.6.0.tar.gz - md5: cb839d723cf5747fdd159050dd8b612f -# patches: - # List any patch files here - # - fix.patch - -# build: - # noarch_python: True - # preserve_egg_dir: True - # entry_points: - # Put any entry points (scripts to be generated automatically) here. The - # syntax is module:function. For example - # - # - progressbar2 = progressbar2:main - # - # Would create an entry point called progressbar2 that calls progressbar2.main() - - - # If this is a new build for the same version, increment the build - # number. If you do not include this key, it defaults to 0. - # number: 1 - -requirements: - build: - - python - - setuptools - - run: - - python - -test: - # Python imports - imports: - - progressbar - - # commands: - # You can put test commands to be run here. Use this to test that the - # entry points work. - - - # You can also put a file called run_test.py in the recipe that will be run - # at test time. - - # requires: - # Put any additional test requirements here. For example - # - nose - -about: - home: https://github.com/WoLpH/python-progressbar - license: BSD License - summary: 'A Python Progressbar library to provide visual (yet text based) progress to\nlong running operations.' - -# See -# http://docs.continuum.io/conda/build.html for -# more information about meta.yaml diff --git a/misc/conda/vaex/bld.bat b/misc/conda/vaex/bld.bat deleted file mode 100644 index 87b1481d74..0000000000 --- a/misc/conda/vaex/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/misc/conda/vaex/build.sh b/misc/conda/vaex/build.sh deleted file mode 100644 index 2d466c4513..0000000000 --- a/misc/conda/vaex/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install --single-version-externally-managed --record=record.txt - -# Add more build steps here, if they are necessary. - -# See -# http://docs.continuum.io/conda/build.html -# for a list of environment variables that are set during the build process. diff --git a/misc/conda/vaex/meta.yaml b/misc/conda/vaex/meta.yaml deleted file mode 100644 index 97b47c7a37..0000000000 --- a/misc/conda/vaex/meta.yaml +++ /dev/null @@ -1,117 +0,0 @@ -package: - name: vaex - version: "1.0.0b2" - -source: - fn: vaex-1.0.0b2.tar.gz - url: https://pypi.python.org/packages/c5/f2/4956c264012df7cd936b8b5deba675f30ec830c707f55345e9ecba1eec36/vaex-1.0.0b2.tar.gz - md5: 012070a8a6ffe1de8c2bbfadf8958ba7 -# patches: - # List any patch files here - # - fix.patch - -build: - script: python setup.py install --single-version-externally-managed --record record.txt - osx_is_app: True - # noarch_python: True - # preserve_egg_dir: True - entry_points: - # Put any entry points (scripts to be generated automatically) here. The - # syntax is module:function. For example - # - # - vaex = vaex:main - # - # Would create an entry point called vaex that calls vaex.main() - - - vaex=vaex.ui.main:main - - # If this is a new build for the same version, increment the build - # number. If you do not include this key, it defaults to 0. - # number: 1 - - -requirements: - build: - - python - - pyopengl >=3.1.0 - - numpy >=1.7.2 - - scipy >=0.1 - - astropy >=0.3 - - h5py >=2.2.1 - - jprops - - matplotlib >=1.3.1 - - numexpr - - psutil >=1.2.1 - - aplus - - tornado >4.1 - - futures - - future >=0.15.2 - - pyyaml - - cachetools - - progressbar2 - - attrdict - - pandas - - pillow - - run: - - python - - pyopengl >=3.1.0 - - numpy >=1.7.2 - - scipy >=0.1 - - astropy >=0.3 - - h5py >=2.2.1 - - jprops - - matplotlib >=1.3.1 - - numexpr - - psutil >=1.2.1 - - aplus - - tornado >4.1 - - futures - - future >=0.15.2 - - pyyaml - - cachetools - - progressbar2 - - attrdict - - pandas - - pillow - - python.app # [osx] - -test: - # Python imports - imports: - - vaex - - vaex.file - - vaex.misc - - vaex.test - - vaex.ui - - vaex.ui.icons - - vaex.ui.plugin - - commands: - # You can put test commands to be run here. Use this to test that the - # entry points work. - - #- vaex --help - - # You can also put a file called run_test.py in the recipe that will be run - # at test time. - - # requires: - # Put any additional test requirements here. For example - # - nose -app: - entry: vaex - summary: 'Vaex is a graphical tool to visualize and explore large tabular datasets.' - icon: vaex32.png - type: desk - - -about: - home: https://www.astro.rug.nl/~breddels/vaex - license: MIT -# license_file: LICENSE - summary: 'Vaex is a graphical tool and library to visualize and explore large tabular datasets.' - -# See -# http://docs.continuum.io/conda/build.html for -# more information about meta.yaml diff --git a/packages/vaex-astro/setup.py b/packages/vaex-astro/setup.py index 26006116e3..8ae1e3ade4 100644 --- a/packages/vaex-astro/setup.py +++ b/packages/vaex-astro/setup.py @@ -12,7 +12,7 @@ license = 'MIT' version = version.__version__ url = 'https://www.github.com/maartenbreddels/vaex' -install_requires_astro = ['vaex-core>=4.5.0,<5', 'astropy'] +install_requires_astro = ['vaex-core~=4.5', 'astropy'] setup( name=name + '-astro', diff --git a/packages/vaex-contrib/setup.py b/packages/vaex-contrib/setup.py index 61edc62354..4121424de4 100644 --- a/packages/vaex-contrib/setup.py +++ b/packages/vaex-contrib/setup.py @@ -10,7 +10,7 @@ license = 'MIT' version = version.__version__ url = 'https://www.github.com/vaexio/vaex' -install_requires_contrib = ['vaex-core>=4.0.0,<5'] +install_requires_contrib = ['vaex-core~=4.0'] setup(name=name + '-contrib', version=version, @@ -18,7 +18,7 @@ url=url, install_requires=install_requires_contrib, extras_require={ - 'gcp': ['google-cloud-bigquery', 'google-cloud-bigquery-storage'], + 'all': ['google-cloud-bigquery', 'google-cloud-bigquery-storage'], }, license=license, packages=['vaex.contrib', 'vaex.contrib.io'], diff --git a/packages/vaex-core/setup.py b/packages/vaex-core/setup.py index cba3be7472..669875ffa9 100644 --- a/packages/vaex-core/setup.py +++ b/packages/vaex-core/setup.py @@ -5,38 +5,58 @@ from setuptools import Extension import platform -use_skbuild = len(os.environ.get('VAEX_BUILD_SKBUILD', '')) > 0 +use_skbuild = len(os.environ.get("VAEX_BUILD_SKBUILD", "")) > 0 if use_skbuild: from skbuild import setup import skbuild.command.build_ext -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +on_rtd = os.environ.get("READTHEDOCS", None) == "True" dirname = os.path.dirname(__file__) path_version = os.path.join(dirname, "vaex/core/_version.py") -version = SourceFileLoader('version', path_version).load_module() +version = SourceFileLoader("version", path_version).load_module() -name = 'vaex' +name = "vaex" author = "Maarten A. Breddels" author_email = "maartenbreddels@gmail.com" -license = 'MIT' +license = "MIT" version = version.__version__ -url = 'https://www.github.com/maartenbreddels/vaex' -# TODO: can we do without requests and progressbar2? +url = "https://www.github.com/maartenbreddels/vaex" # TODO: after python2 supports frops, future and futures can also be dropped -# TODO: would be nice to have astropy only as dep in vaex-astro -setup_requires = ["numpy~=1.16"] -install_requires_core = ["numpy~=1.16", "aplus", "tabulate>=0.8.3", - "future>=0.15.2", "pyyaml", "progressbar2", - "requests", "six", "cloudpickle", "pandas", "dask!=2022.4.0", - "nest-asyncio>=1.3.3", "pyarrow>=5.0.0", "frozendict!=2.2.0", - "blake3", "filelock", "pydantic>=1.8.0", "rich", - ] -if sys.version_info[0] == 2: - install_requires_core.append("futures>=2.2.0") -install_requires_viz = ["matplotlib>=1.3.1", ] -install_requires_astro = ["kapteyn"] +setup_requires = ["numpy~=1.17"] +install_requires_core = [ + "numpy~=1.17", + "aplus", + "tabulate>=0.8.3", + "dask!=2022.4.0", + "future>=0.15.2", + "pyyaml", + "six", + "cloudpickle", + "pandas>=1.0,<3", + "nest_asyncio>=1.3.3", + "pyarrow>=5.0.0", + "frozendict!=2.2.0", + "blake3", + "filelock", + "pydantic>=1.8.0", + "rich", +] +extras_require_core = { + "all": [ + "gcsfs>=0.6.2", + "s3fs", + "ipyvolume", + "diskcache", + "fsspec", + "h5py", + "httpx", + "aplus", + "psutil", + "graphviz", + ], +} if "MACOSX_DEPLOYMENT_TARGET" not in os.environ: os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.9" @@ -45,24 +65,27 @@ # MB: I like these options during development, the second if for ccache # extra_dev_options = ['-fmax-errors=4', '-fdiagnostics-color', '-pedantic-errors'] + class get_numpy_include(object): """Helper class to determine the numpy include path The purpose of this class is to postpone importing numpy until it is actually installed, so that the ``get_include()`` - method can be invoked. """ + method can be invoked.""" def __init__(self): pass def __str__(self): import numpy as np + return np.get_include() + class get_pybind_include(object): """Helper class to determine the pybind11 include path The purpose of this class is to postpone importing pybind11 until it is actually installed, so that the ``get_include()`` - method can be invoked. """ + method can be invoked.""" def __init__(self, user=False): self.user = user @@ -71,7 +94,7 @@ def __str__(self): # this trick does not work anymore it seems, we now just vendor it # import pybind11 # return pybind11.get_include(self.user) - return 'vendor/pybind11/include' + return "vendor/pybind11/include" USE_ABSL = False @@ -79,71 +102,102 @@ def __str__(self): define_macros = [] if USE_ABSL: - define_macros += [('VAEX_USE_ABSL', None)] + define_macros += [("VAEX_USE_ABSL", None)] if USE_TSL: - define_macros += [('VAEX_USE_TSL', None)] + define_macros += [("VAEX_USE_TSL", None)] dll_files = [] -if platform.system().lower() == 'windows': +if platform.system().lower() == "windows": extra_compile_args = ["/EHsc"] - dll_files = ['pcre.dll', 'pcrecpp.dll', 'vcruntime140_1.dll'] + dll_files = ["pcre.dll", "pcrecpp.dll", "vcruntime140_1.dll"] else: # TODO: maybe enable these flags for non-wheel/conda builds? ["-mtune=native", "-march=native"] - extra_compile_args = ["-std=c++11", "-O3", "-funroll-loops", "-Werror=return-type", "-Wno-unused-parameter"] + extra_compile_args = [ + "-std=c++11", + "-O3", + "-funroll-loops", + "-Werror=return-type", + "-Wno-unused-parameter", + ] extra_compile_args.append("-g") extra_compile_args += extra_dev_options -if sys.platform == 'darwin': +if sys.platform == "darwin": extra_compile_args.append("-mmacosx-version-min=10.9") # on windows (Conda-forge builds), the dirname is an absolute path -extension_vaexfast = Extension("vaex.vaexfast", [os.path.relpath(os.path.join(dirname, "src/vaexfast.cpp"))], - include_dirs=[get_numpy_include()], - extra_compile_args=extra_compile_args) -extension_strings = Extension("vaex.superstrings", [ - os.path.relpath(os.path.join(dirname, "src/strings.cpp")), - os.path.relpath(os.path.join(dirname, "src/string_utils.cpp")), +extension_vaexfast = Extension( + "vaex.vaexfast", + [os.path.relpath(os.path.join(dirname, "src/vaexfast.cpp"))], + include_dirs=[get_numpy_include()], + extra_compile_args=extra_compile_args, +) +extension_strings = Extension( + "vaex.superstrings", + [ + os.path.relpath(os.path.join(dirname, "src/strings.cpp")), + os.path.relpath(os.path.join(dirname, "src/string_utils.cpp")), ], include_dirs=[ get_numpy_include(), get_pybind_include(), get_pybind_include(user=True), - 'vendor/string-view-lite/include', - 'vendor/boost', - os.path.join(sys.prefix, 'include'), - os.path.join(sys.prefix, 'Library', 'include'), # windows - os.path.join(dirname, 'vendor', 'pcre', 'Library', 'include') # windows pcre from conda-forge + "vendor/string-view-lite/include", + "vendor/boost", + os.path.join(sys.prefix, "include"), + os.path.join(sys.prefix, "Library", "include"), # windows + os.path.join( + dirname, "vendor", "pcre", "Library", "include" + ), # windows pcre from conda-forge ], library_dirs=[ - os.path.join(sys.prefix, 'lib'), - os.path.join(sys.prefix, 'Library', 'lib'), # windows - os.path.join(dirname, 'vendor', 'pcre', 'Library', 'lib'), # windows pcre from conda-forge + os.path.join(sys.prefix, "lib"), + os.path.join(sys.prefix, "Library", "lib"), # windows + os.path.join( + dirname, "vendor", "pcre", "Library", "lib" + ), # windows pcre from conda-forge ], extra_compile_args=extra_compile_args, - libraries=['pcre', 'pcrecpp'] + libraries=["pcre", "pcrecpp"], ) -extension_superutils = Extension("vaex.superutils", [ +extension_superutils = Extension( + "vaex.superutils", + [ os.path.relpath(os.path.join(dirname, "src/hash_string.cpp")), os.path.relpath(os.path.join(dirname, "src/hash_primitives_pot.cpp")), os.path.relpath(os.path.join(dirname, "src/hash_object.cpp")), os.path.relpath(os.path.join(dirname, "src/hash_primitives_prime.cpp")), os.path.relpath(os.path.join(dirname, "src/superutils.cpp")), os.path.relpath(os.path.join(dirname, "src/string_utils.cpp")), - ] + ([os.path.relpath(os.path.join(dirname, "vendor/abseil-cpp/absl/container/internal/raw_hash_set.cc"))] if USE_ABSL else []), + ] + + ( + [ + os.path.relpath( + os.path.join( + dirname, "vendor/abseil-cpp/absl/container/internal/raw_hash_set.cc" + ) + ) + ] + if USE_ABSL + else [] + ), include_dirs=[ - get_numpy_include(), get_pybind_include(), + get_numpy_include(), + get_pybind_include(), get_pybind_include(user=True), - 'vendor/abseil-cpp', - 'vendor/flat_hash_map', - 'vendor/sparse-map/include', - 'vendor/hopscotch-map/include', - 'vendor/string-view-lite/include', + "vendor/abseil-cpp", + "vendor/flat_hash_map", + "vendor/sparse-map/include", + "vendor/hopscotch-map/include", + "vendor/string-view-lite/include", ], extra_compile_args=extra_compile_args, define_macros=define_macros, - ) +) -extension_superagg = Extension("vaex.superagg", [ +extension_superagg = Extension( + "vaex.superagg", + [ os.path.relpath(os.path.join(dirname, "src/agg_nunique_string.cpp")), os.path.relpath(os.path.join(dirname, "src/agg_minmax.cpp")), os.path.relpath(os.path.join(dirname, "src/agg_nunique.cpp")), @@ -159,69 +213,93 @@ def __str__(self): os.path.relpath(os.path.join(dirname, "src/string_utils.cpp")), ], include_dirs=[ - get_numpy_include(), get_pybind_include(), + get_numpy_include(), + get_pybind_include(), get_pybind_include(user=True), - 'vendor/flat_hash_map', - 'vendor/sparse-map/include', - 'vendor/hopscotch-map/include', - 'vendor/string-view-lite/include' + "vendor/flat_hash_map", + "vendor/sparse-map/include", + "vendor/hopscotch-map/include", + "vendor/string-view-lite/include", ], extra_compile_args=extra_compile_args, define_macros=define_macros, - ) +) -setup(name=name + '-core', - version=version, - description='Core of vaex', - url=url, - author=author, - author_email=author_email, - setup_requires=setup_requires, - install_requires=install_requires_core, - license=license, - package_data={'vaex': dll_files + ['test/files/*.fits', 'test/files/*.vot', 'test/files/*.hdf5']}, - packages=['vaex', 'vaex.arrow', 'vaex.core', 'vaex.file', 'vaex.test', 'vaex.ext', 'vaex.misc', 'vaex.datasets'], - include_package_data=True, - ext_modules=([extension_vaexfast] if on_rtd else [extension_vaexfast, extension_strings, extension_superutils, extension_superagg]) if not use_skbuild else [], - zip_safe=False, - python_requires=">=3.8", - classifiers=[ - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - ], - extras_require={ - 'all': ["gcsfs>=0.6.2", "s3fs"] - }, - entry_points={ - 'console_scripts': ['vaex = vaex.__main__:main'], - 'gui_scripts': ['vaexgui = vaex.__main__:main'], # sometimes in osx, you need to run with this - 'vaex.dataframe.accessor': [ - 'geo = vaex.geo:DataFrameAccessorGeo', - 'struct = vaex.struct:DataFrameAccessorStruct', - ], - 'vaex.dataset.opener': [ - 'csv = vaex.csv:DatasetCsvLazy', - 'arrow = vaex.arrow.opener:ArrowOpener', - 'parquet = vaex.arrow.opener:ParquetOpener', - 'feather = vaex.arrow.opener:FeatherOpener', - ], - 'vaex.memory.tracker': [ - 'default = vaex.memory:MemoryTracker' - ], - 'vaex.progressbar': [ - 'vaex = vaex.progress:simple', - 'simple = vaex.progress:simple', - 'widget = vaex.progress:widget', - 'rich = vaex.progress:rich', - ], - 'vaex.file.scheme': [ - 's3 = vaex.file.s3', - 'fsspec+s3 = vaex.file.s3fs', - 'arrow+s3 = vaex.file.s3arrow', - 'gs = vaex.file.gcs', - 'fsspec+gs = vaex.file.gcs', - ] - } - ) +setup( + name=name + "-core", + version=version, + description="Core of vaex", + url=url, + author=author, + author_email=author_email, + setup_requires=setup_requires, + install_requires=install_requires_core, + license=license, + package_data={ + "vaex": dll_files + + ["test/files/*.fits", "test/files/*.vot", "test/files/*.hdf5"] + }, + packages=[ + "vaex", + "vaex.arrow", + "vaex.core", + "vaex.file", + "vaex.test", + "vaex.ext", + "vaex.misc", + "vaex.datasets", + ], + include_package_data=True, + ext_modules=( + [extension_vaexfast] + if on_rtd + else [ + extension_vaexfast, + extension_strings, + extension_superutils, + extension_superagg, + ] + ) + if not use_skbuild + else [], + zip_safe=False, + python_requires=">=3.8", + classifiers=[ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + ], + extras_require=extras_require_core, + entry_points={ + "console_scripts": ["vaex = vaex.__main__:main"], + "gui_scripts": [ + "vaexgui = vaex.__main__:main" + ], # sometimes in osx, you need to run with this + "vaex.dataframe.accessor": [ + "geo = vaex.geo:DataFrameAccessorGeo", + "struct = vaex.struct:DataFrameAccessorStruct", + ], + "vaex.dataset.opener": [ + "csv = vaex.csv:DatasetCsvLazy", + "arrow = vaex.arrow.opener:ArrowOpener", + "parquet = vaex.arrow.opener:ParquetOpener", + "feather = vaex.arrow.opener:FeatherOpener", + ], + "vaex.memory.tracker": ["default = vaex.memory:MemoryTracker"], + "vaex.progressbar": [ + "vaex = vaex.progress:simple", + "simple = vaex.progress:simple", + "widget = vaex.progress:widget", + "rich = vaex.progress:rich", + ], + "vaex.file.scheme": [ + "s3 = vaex.file.s3", + "fsspec+s3 = vaex.file.s3fs", + "arrow+s3 = vaex.file.s3arrow", + "gs = vaex.file.gcs", + "fsspec+gs = vaex.file.gcs", + ], + }, +) diff --git a/packages/vaex-core/vaex/functions.py b/packages/vaex-core/vaex/functions.py index fcfc69e6ee..5da4b05da7 100644 --- a/packages/vaex-core/vaex/functions.py +++ b/packages/vaex-core/vaex/functions.py @@ -605,7 +605,7 @@ def dt_weekofyear(x): 1 6 2 46 """ - return _to_pandas_series(x).dt.weekofyear.values + return _to_pandas_series(x).dt.isocalendar().week.values @register_function(scope='dt', as_property=True) def dt_hour(x): diff --git a/packages/vaex-graphql/setup.py b/packages/vaex-graphql/setup.py index 8a6e8a0860..ab3fcc5e5e 100644 --- a/packages/vaex-graphql/setup.py +++ b/packages/vaex-graphql/setup.py @@ -12,7 +12,7 @@ license = 'MIT' version = version.__version__ url = 'https://www.github.com/vaexio/vaex' -install_requires_graphql = ['vaex-core>=4.0.0,<5', 'graphene>=2.1.8,<3', 'graphene>=2.1.8,<3', 'graphene-tornado>=2.5.1,<3'] +install_requires_graphql = ['vaex-core~=4.0', 'graphene-tornado~=3.0.0b2'] setup( name=name + '-graphql', diff --git a/packages/vaex-hdf5/setup.py b/packages/vaex-hdf5/setup.py index b21f664df5..5588b9e9b2 100644 --- a/packages/vaex-hdf5/setup.py +++ b/packages/vaex-hdf5/setup.py @@ -13,7 +13,7 @@ license = 'MIT' version = version.__version__ url = 'https://www.github.com/maartenbreddels/vaex' -install_requires_hdf5 = ["vaex-core>=4.0.0,<5", "h5py>=2.9"] +install_requires_hdf5 = ["vaex-core~=4.0", "h5py>=2.9"] setup(name=name + '-hdf5', version=version, diff --git a/packages/vaex-jupyter/setup.py b/packages/vaex-jupyter/setup.py index 467f4b375b..d6ca98eb6a 100644 --- a/packages/vaex-jupyter/setup.py +++ b/packages/vaex-jupyter/setup.py @@ -14,7 +14,7 @@ license = 'MIT' version = version.__version__ url = 'https://www.github.com/maartenbreddels/vaex' -install_requires_jupyter = ['vaex-core>=4.7.0,<5', 'vaex-viz', 'bqplot>=0.10.1', 'ipyvolume>=0.4', 'ipyleaflet', 'ipympl', 'ipyvuetify>=1.2.2,<2', 'xarray'] +install_requires_jupyter = ['vaex-core~=4.7', 'vaex-viz', 'bqplot>=0.10.1', 'ipyvolume>=0.4', 'ipyleaflet', 'ipympl', 'ipyvuetify>=1.2.2,<2', 'xarray'] setup(name=name + '-jupyter', version=version, diff --git a/packages/vaex-ml/setup.py b/packages/vaex-ml/setup.py index 07e76a82ea..04ed2e7725 100644 --- a/packages/vaex-ml/setup.py +++ b/packages/vaex-ml/setup.py @@ -12,7 +12,18 @@ license = 'MIT' version = version.__version__ url = 'https://www.github.com/vaexio/vaex' -install_requires_ml = ['vaex-core>=4.8.0,<5', 'numba', 'traitlets', 'jinja2'] +install_requires_ml = [ + 'vaex-core~=4.8', + 'numba', + 'traitlets', + 'jinja2', + 'annoy', + 'scikit-learn', + 'xgboost', + 'lightgbm~=4.0', + 'catboost', +] +extras_require_ml = {'all': ['tensorflow~=2.1']} setup(name=name + '-ml', version=version, @@ -21,7 +32,7 @@ author=author, author_email=author_email, install_requires=install_requires_ml, - extras_require={'all': ['tensorflow>=2.1.0', 'tensorflow-io>=0.12.0']}, + extras_require=extras_require_ml, license=license, packages=['vaex.ml', 'vaex.ml.incubator'], include_package_data=True, diff --git a/packages/vaex-ml/vaex/ml/tensorflow.py b/packages/vaex-ml/vaex/ml/tensorflow.py index 8256a91d24..43d4337883 100644 --- a/packages/vaex-ml/vaex/ml/tensorflow.py +++ b/packages/vaex-ml/vaex/ml/tensorflow.py @@ -1,6 +1,6 @@ import base64 import tempfile -import shutil +import os.path import numpy as np @@ -177,13 +177,12 @@ def transform(self, df): def state_get(self): state = super(KerasModel, self).state_get() - with tempfile.TemporaryDirectory() as directory: - self.model.save(directory) - zip_path = tempfile.mktemp(".zip") - shutil.make_archive(zip_path[:-4], 'zip', directory) - with open(zip_path, 'rb') as f: + filepath = os.path.join(directory, 'model.keras') + self.model.save(filepath) + with open(filepath, 'rb') as f: data = f.read() + # store model as raw zip bytes base64 encoded state['model'] = base64.encodebytes(data).decode('ascii') return state @@ -194,8 +193,7 @@ def state_set(self, state, trusted=True): data = base64.decodebytes(model_data.encode('ascii')) with tempfile.TemporaryDirectory() as directory: - zip_path = tempfile.mktemp('.zip') - with open(zip_path, 'wb') as f: + filepath = os.path.join(directory, 'model.keras') + with open(filepath, 'wb') as f: f.write(data) - shutil.unpack_archive(zip_path, directory) - self.model = K.models.load_model(directory, custom_objects=self.custom_objects) + self.model = K.models.load_model(filepath, custom_objects=self.custom_objects) diff --git a/packages/vaex-ml/vaex/ml/transformations.py b/packages/vaex-ml/vaex/ml/transformations.py index cca5eaac6a..90b70c8008 100644 --- a/packages/vaex-ml/vaex/ml/transformations.py +++ b/packages/vaex-ml/vaex/ml/transformations.py @@ -432,7 +432,7 @@ def transform(self, df): column_name = self.prefix + feature + '_' + str_value if value is None: copy[column_name] = copy.func.where(copy[feature].ismissing(), self.one, self.zero, dtype=dtype) - elif isinstance(value, np.float) and np.isnan(value): + elif isinstance(value, float) and np.isnan(value): copy[column_name] = copy.func.where(copy[feature].isnan(), self.one, self.zero, dtype=dtype) else: copy[column_name] = copy.func.where(copy[feature] == value, self.one, self.zero, dtype=dtype) diff --git a/packages/vaex-server/setup.py b/packages/vaex-server/setup.py index 37916d450c..a54fbc1355 100644 --- a/packages/vaex-server/setup.py +++ b/packages/vaex-server/setup.py @@ -14,7 +14,7 @@ license = 'MIT' version = version.__version__ url = 'https://www.github.com/maartenbreddels/vaex' -install_requires_server = ['vaex-core>=4.7.0,<5', 'tornado>4.1', 'cachetools', 'fastapi', 'uvicorn[standard]'] +install_requires_server = ['vaex-core~=4.7', 'tornado>4.1', 'cachetools', 'fastapi', 'uvicorn[standard]'] setup(name=name + '-server', version=version, diff --git a/packages/vaex-ui/setup.py b/packages/vaex-ui/setup.py index 401942eb6f..5e378b3145 100644 --- a/packages/vaex-ui/setup.py +++ b/packages/vaex-ui/setup.py @@ -13,7 +13,7 @@ license = 'MIT' version = version.__version__ url = 'https://www.github.com/maartenbreddels/vaex' -install_requires_ui = ["vaex-core>=0.6.1", "PyOpenGL", "scipy", "matplotlib", "vaex-hdf5", "vaex-astro"] +install_requires_ui = ["vaex-core>=0.6.1", "PyOpenGL", "scipy", "matplotlib", "vaex-hdf5", "vaex-astro", "PyQt5"] setup(name=name + '-ui', version=version, diff --git a/packages/vaex-meta/LICENSE.txt b/packages/vaex/LICENSE.txt similarity index 100% rename from packages/vaex-meta/LICENSE.txt rename to packages/vaex/LICENSE.txt diff --git a/packages/vaex-meta/MANIFEST.in b/packages/vaex/MANIFEST.in similarity index 100% rename from packages/vaex-meta/MANIFEST.in rename to packages/vaex/MANIFEST.in diff --git a/packages/vaex-meta/README.md b/packages/vaex/README.md similarity index 100% rename from packages/vaex-meta/README.md rename to packages/vaex/README.md diff --git a/packages/vaex-meta/setup.py b/packages/vaex/setup.py similarity index 100% rename from packages/vaex-meta/setup.py rename to packages/vaex/setup.py diff --git a/packages/vaex-meta/vaex/meta/__init__.py b/packages/vaex/vaex/meta/__init__.py similarity index 100% rename from packages/vaex-meta/vaex/meta/__init__.py rename to packages/vaex/vaex/meta/__init__.py diff --git a/packages/vaex-meta/vaex/meta/_version.py b/packages/vaex/vaex/meta/_version.py similarity index 100% rename from packages/vaex-meta/vaex/meta/_version.py rename to packages/vaex/vaex/meta/_version.py diff --git a/requirements-ml.txt b/requirements-ml.txt deleted file mode 100644 index b4ab89ed63..0000000000 --- a/requirements-ml.txt +++ /dev/null @@ -1,7 +0,0 @@ -annoy -scikit-learn -xgboost -lightgbm>4 -catboost -# tensorflow==2.1.0 -# tensorflow-io==0.12.0 diff --git a/requirements_rtd.txt b/requirements_rtd.txt deleted file mode 100644 index edff735a07..0000000000 --- a/requirements_rtd.txt +++ /dev/null @@ -1,28 +0,0 @@ --e . -sphinx -sphinx-book-theme -sphinx-sitemap -sphinxcontrib-googleanalytics -sphinxext-rediraffe -future -pyyaml -astropy -psutil>=1.2.1 -numpy>=1.13 -aplus -tornado>=5.0 -futures==2.2.0 -cachetools -progressbar2 -jupyter -nbconvert>=5.1.1 -nbsphinx>=0.2.18 -ipython -jupyter-sphinx>=0.1.2 -pip>=10.0 -annoy -lightgbm -xgboost -sphinx_gallery -catboost -myst-parser diff --git a/setup.py b/setup.py index 6f6c02be06..4dbfba4ab0 100644 --- a/setup.py +++ b/setup.py @@ -1,94 +1,56 @@ -from setuptools import setup -from setuptools.command.develop import develop -from setuptools.command.install import install -import pip -import os -import sys -import contextlib - - -@contextlib.contextmanager -def cwd(path): - curdir = os.getcwd() - try: - os.chdir(path) - yield - finally: - os.chdir(curdir) - -# inspired by https://blog.shazam.com/python-microlibs-5be9461ad979 - - -packages = ['vaex-core', 'vaex-viz', 'vaex-hdf5', 'vaex-server', 'vaex-astro', 'vaex-jupyter', 'vaex-ml', 'vaex-meta', 'vaex-graphql', 'vaex-contrib'] -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +# only for local development / install from source, e.g. +# pip install https://github.com/vaexio/vaex/archive/refs/heads/master.zip +from pathlib import Path -class DevelopCmd(develop): - def run(self): - relative = os.path.abspath(os.path.join('packages', 'vaex-core', 'vaex')) - for package in packages: - with cwd(os.path.join('packages', package)): - err = os.system(f'{sys.executable} -m pip install -e .') - if err: - raise RuntimeError(f'Oops, failed to install {package}') - # we need to make symbolic links from vaex-core/vaex/ to vaex-/vaex/ - # otherwise development install do not work - if package not in ['vaex-core']: - name = package.split('-')[1] - - source = os.path.abspath(os.path.join('packages', package, 'vaex', name)) - rel_source = os.path.relpath(source, relative) - with cwd(relative): - print('symlinking', source, name, rel_source) - if os.path.exists(name) and os.readlink(name) == rel_source: - print('symlink ok') - else: - if os.path.exists(name): - print('old symlink', os.readlink(name)) - os.remove(name) - os.symlink(rel_source, name) - - -class InstallCmd(install): - """ Add custom steps for the install command """ - def run(self): - for package in packages: - with cwd(os.path.join('packages', package)): - os.system(f'{sys.executable} -m pip install --no-deps .') - for package in packages: - with cwd(os.path.join('packages', package)): - os.system(f'{sys.executable} -m pip install --upgrade .') +from setuptools import setup +packages = [ + "vaex-core", + "vaex-viz", + "vaex-hdf5", + "vaex-server", + "vaex-astro", + "vaex-jupyter", + "vaex-ml", + "vaex-graphql", + "vaex-contrib", + "vaex", +] setup( - name='vaex-meta', + name="vaex-meta", version="0.1.0", description="Convenience setup.py for when installing from the git repo", classifiers=[ - 'Private :: Do Not Upload to pypi server', + "Private :: Do Not Upload to pypi server", ], packages=[], install_requires=[ - 'pip', + f"{package}[all] @ {(Path(__file__).parent / 'packages' / package).as_uri()}" + for package in packages ], extras_require={ - 'dev': [ - 'pytest', - 'gcsfs', - 's3fs', - 'graphviz', - 'myst_parser', - 'rich', - # For generating the documentation - 'sphinx', - 'nbsphinx', - 'sphinx_gallery', - 'sphinx_sitemap', - 'sphinx_book_theme', + "ci": [ + # readthedocs + "sphinx", + "sphinx_book_theme", + "sphinx_sitemap", + "sphinxcontrib_googleanalytics", + "sphinxext_rediraffe", + "sphinx_gallery", + "nbsphinx", + "jupyter_sphinx", + "myst_parser", + # tests + "pytest", + "pytest-asyncio", + "pytest-mpl", + "pytest-timeout", + # ipynb tests + "nbconvert", + "jupyterlab", + "plotly", ] }, - cmdclass={ - 'install': InstallCmd, - 'develop': DevelopCmd, - }, ) diff --git a/tests/agg_test.py b/tests/agg_test.py index 9c94e18ec8..9681639db0 100644 --- a/tests/agg_test.py +++ b/tests/agg_test.py @@ -247,8 +247,8 @@ def test_minmax_all_dfs(df): def test_minmax_mixed_types(): - x = np.array([1, 0], dtype=np.int) - y = np.array([0.5, 1.5], dtype=np.float) + x = np.array([1, 0], dtype=int) + y = np.array([0.5, 1.5], dtype=float) df = vaex.from_arrays(x=x, y=y) with pytest.raises(TypeError): df.minmax(['x', 'y']) diff --git a/tests/datetime_test.py b/tests/datetime_test.py index 78a55f3fa7..af9397cb36 100644 --- a/tests/datetime_test.py +++ b/tests/datetime_test.py @@ -25,7 +25,7 @@ def test_datetime_operations(): assert df.date.dt.year.tolist() == pandas_df.date.dt.year.values.tolist() assert df.date.dt.is_leap_year.tolist() == pandas_df.date.dt.is_leap_year.values.tolist() assert any(df.date.dt.is_leap_year.tolist()) - assert df.date.dt.weekofyear.tolist() == pandas_df.date.dt.weekofyear.values.tolist() + assert df.date.dt.weekofyear.tolist() == pandas_df.date.dt.isocalendar().week.values.tolist() assert df.date.dt.dayofyear.tolist() == pandas_df.date.dt.dayofyear.values.tolist() assert df.date.dt.dayofweek.tolist() == pandas_df.date.dt.dayofweek.values.tolist() assert df.date.dt.floor('H').tolist() == pandas_df.date.dt.floor('H').values.tolist() diff --git a/tests/export_test.py b/tests/export_test.py index 41c40f6299..c310f77b0f 100644 --- a/tests/export_test.py +++ b/tests/export_test.py @@ -194,7 +194,7 @@ def test_export_unicode_column_name_hdf5(tmpdir): # assert df[0:2].s.tolist() == ['x', 'x'] # assert df[-3:-1].s.tolist() == ['y', 'y'] -@pytest.mark.parametrize("dtypes", [{}, {'name': np.object, 'age': 'Int64', 'weight': np.float}]) +@pytest.mark.parametrize("dtypes", [{}, {'name': object, 'age': int, 'weight': float}]) def test_multi_file_naive_read_convert_export(tmpdir, dtypes): current_dir = os.path.dirname(__file__) @@ -236,7 +236,7 @@ def test_export_csv(df_local, tmpdir): assert '123456' in vaex.open(path) -@pytest.mark.parametrize("dtypes", [{}, {'name': np.object, 'age': 'Int64', 'weight': np.float}]) +@pytest.mark.parametrize("dtypes", [{}, {'name': object, 'age': int, 'weight': float}]) def test_export_generates_same_hdf5_shasum(tmpdir, dtypes): current_dir = os.path.dirname(__file__) diff --git a/tests/from_pandas_test.py b/tests/from_pandas_test.py index edc60662c6..ab70500263 100644 --- a/tests/from_pandas_test.py +++ b/tests/from_pandas_test.py @@ -20,7 +20,7 @@ def test_from_pandas(): 'int_missing': pd.Series([1, None, 5, 1, 10], dtype='Int64'), 'datetime_1': [pd.NaT, datetime.datetime(2019, 1, 1, 1, 1, 1), datetime.datetime(2019, 1, 1, 1, 1, 1), datetime.datetime(2019, 1, 1, 1, 1, 1), datetime.datetime(2019, 1, 1, 1, 1, 1)], 'datetime_2': [pd.NaT, None, pd.NaT, pd.NaT, pd.NaT], - 'datetime_3': [pd.Timedelta('1M'), pd.Timedelta('1D'), pd.Timedelta('100M'), pd.Timedelta('2D'), pd.Timedelta('1H')], + 'datetime_3': [pd.Timedelta(days=30.5), pd.Timedelta(hours=24), pd.Timedelta(days=30.5 * 100), pd.Timedelta(days=2), pd.Timedelta(hours=1)], 'datetime_4': [pd.Timestamp('2001-1-1 2:2:11'), pd.Timestamp('2001-12'), pd.Timestamp('2001-10-1'), pd.Timestamp('2001-03-1 2:2:11'), pd.Timestamp('2001-1-1 2:2:11')], 'datetime_5': [datetime.date(2010, 1, 1), datetime.date(2010, 1, 1), datetime.date(2010, 1, 1), datetime.date(2010, 1, 1), datetime.date(2010, 1, 1)], 'datetime_6': [datetime.time(21, 1, 1), datetime.time(21, 1, 1), datetime.time(21, 1, 1), datetime.time(21, 1, 1), datetime.time(21, 1, 1)], diff --git a/tests/ml/tensorflow_test.py b/tests/ml/tensorflow_test.py index 66222d24d6..d4026edac0 100644 --- a/tests/ml/tensorflow_test.py +++ b/tests/ml/tensorflow_test.py @@ -33,7 +33,7 @@ def test_keras_model_classification(tmpdir, df_iris): nn_model.add(K.layers.Dense(3, activation='softmax')) nn_model.compile(optimizer=K.optimizers.RMSprop(learning_rate=0.01), loss=K.losses.categorical_crossentropy, - metrics='accuracy') + metrics=['accuracy']) X = df[features].values y = df[targets].values diff --git a/tests/strings_test.py b/tests/strings_test.py index d1c2c0d99b..746bb64be9 100644 --- a/tests/strings_test.py +++ b/tests/strings_test.py @@ -12,8 +12,8 @@ @pytest.mark.skipif(vaex.utils.osname == 'windows', reason="windows' snprintf seems buggy") def test_format(): - num1 = np.array([1, 2, 3], dtype=np.int32) - num2 = np.array([1.1, 2.2, 3.3], dtype=np.float32) + num1 = np.array([1, 2, 3], dtype=int) + num2 = np.array([1.1, 2.2, 3.3], dtype=float) text = ['Here', 'we', 'go'] df = vaex.from_arrays(num1=num1, num2=num2, text=text) @@ -77,7 +77,7 @@ def test_concat(): ds = ds1.concat(ds2) assert len(ds) == len(ds1) + len(ds2) assert ds.data_type('names') == pa.string() - assert ds.data_type('names') != np.object + assert ds.data_type('names') != object def test_string_count_stat():