From 5f4be91f21daba5f0e6eec792bd33376721d134b Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 6 Jul 2013 00:13:32 -0400 Subject: [PATCH] BLD: add requirements files and update install.sh This commit adds 5 requirements files: one for each Travis build. The corresponding sections in ci/install.sh are also removed in favor of these files. This allows pandas to have more fine-grained control over versions of hard and soft deps when using travis. --- .gitignore | 3 +- .travis.yml | 4 +- ci/install.sh | 75 ++++--------------- ci/requirements-2.6.txt | 4 + ci/requirements-2.7.txt | 15 ++++ ci/requirements-2.7_LOCALE.txt | 14 ++++ ci/requirements-3.2.txt | 12 +++ ci/requirements-3.3.txt | 12 +++ ci/speedpack/Vagrantfile | 1 + ci/speedpack/build.sh | 110 +++++++++++----------------- ci/speedpack/wheelhouse/placeholder | 0 11 files changed, 122 insertions(+), 128 deletions(-) create mode 100644 ci/requirements-2.6.txt create mode 100644 ci/requirements-2.7.txt create mode 100644 ci/requirements-2.7_LOCALE.txt create mode 100644 ci/requirements-3.2.txt create mode 100644 ci/requirements-3.3.txt delete mode 100644 ci/speedpack/wheelhouse/placeholder diff --git a/.gitignore b/.gitignore index 9a0794373cfdc..3da165e07c77c 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,5 @@ pandas/io/*.json .idea/pandas.iml .build_cache_dir .vagrant -*.whl \ No newline at end of file +*.whl +**/wheelhouse/* diff --git a/.travis.yml b/.travis.yml index ff292767011cf..3d3d90eb47129 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,13 +10,13 @@ env: matrix: include: - python: 2.7 - env: NOSE_ARGS="slow and not network" LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true + env: NOSE_ARGS="slow and not network" LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true JOB_TAG=_LOCALE - python: 2.7 env: NOSE_ARGS="not slow" FULL_DEPS=true - python: 3.2 env: NOSE_ARGS="not slow" FULL_DEPS=true - python: 3.3 - env: NOSE_ARGS="not slow" + env: NOSE_ARGS="not slow" FULL_DEPS=true # allow importing from site-packages, # so apt-get python-x works for system pythons diff --git a/ci/install.sh b/ci/install.sh index 8145e2e74e893..5c681a707ce49 100755 --- a/ci/install.sh +++ b/ci/install.sh @@ -15,85 +15,42 @@ echo "inside $0" -# Install Dependencie +# Install Dependencies # as of pip 1.4rc2, wheel files are still being broken regularly, this is a known good # commit. should revert to pypi when a final release is out pip install -I git+https://github.com/pypa/pip@42102e9deaea99db08b681d06906c2945f6f95e2#egg=pip -pip Install -I https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz +pv="${TRAVIS_PYTHON_VERSION:0:1}" +[ "$pv" == "2" ] && pv="" +[ "$pv" == "2" ] && DISTRIBUTE_VERSION="==0.6.35" + +pip install -I distribute${DISTRIBUTE_VERSION} pip install wheel # comment this line to disable the fetching of wheel files -#PIP_ARGS+=" -I --use-wheel --find-links=https://cache27-pypandas.rhcloud.com/" +PIP_ARGS+=" -I --use-wheel --find-links=http://cache27diy-cpycloud.rhcloud.com/${TRAVIS_PYTHON_VERSION}${JOB_TAG}/" # Force virtualenv to accpet system_site_packages rm -f $VIRTUAL_ENV/lib/python$TRAVIS_PYTHON_VERSION/no-global-site-packages.txt -if [ x"$LOCALE_OVERRIDE" != x"" ]; then + +if [ -n "$LOCALE_OVERRIDE" ]; then # make sure the locale is available # probably useless, since you would need to relogin sudo locale-gen "$LOCALE_OVERRIDE" -fi; - -#scipy is not included in the cached venv -if [ x"$FULL_DEPS" == x"true" ] ; then - # for pytables gets the lib as well - time sudo apt-get $APT_ARGS install libhdf5-serial-dev - - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then - time sudo apt-get $APT_ARGS install python3-bs4 - elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then - time sudo apt-get $APT_ARGS install python-bs4 - fi - - if [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then - time sudo apt-get $APT_ARGS install python3-scipy - elif [ ${TRAVIS_PYTHON_VERSION} == "2.7" ]; then - time sudo apt-get $APT_ARGS install python-scipy - fi fi -# Hard Deps -for dep in nose 'python-dateutil' 'pytz>=2013a' 'cython==0.19.1'; do - time pip install $PIP_ARGS $dep -done - -if [ ${TRAVIS_PYTHON_VERSION} == "3.3" ]; then # should be >=3,3 - time pip install $PIP_ARGS numpy==1.7.1 -elif [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then - # sudo apt-get $APT_ARGS install python3-numpy; # 1.6.2 or precise - time pip install $PIP_ARGS numpy==1.6.1 -else - time pip install $PIP_ARGS numpy==1.6.1 -fi +time pip install $PIP_ARGS -r ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.txt # Optional Deps if [ x"$FULL_DEPS" == x"true" ]; then echo "Installing FULL_DEPS" + # for pytables gets the lib as well + time sudo apt-get $APT_ARGS install libhdf5-serial-dev + time sudo apt-get $APT_ARGS install python${pv}-bs4 + time sudo apt-get $APT_ARGS install python${pv}-scipy + + time sudo apt-get $APT_ARGS remove python${pv}-lxml - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then - time pip install $PIP_ARGS xlwt - time pip install $PIP_ARGS bottleneck==0.6.0 - time pip install $PIP_ARGS numexpr==2.1 - time pip install $PIP_ARGS tables==2.3.1 - else - time pip install $PIP_ARGS numexpr==2.1 - time pip install $PIP_ARGS tables==3.0.0 - fi - - time pip install $PIP_ARGS matplotlib==1.2.1 - time pip install $PIP_ARGS openpyxl - time pip install $PIP_ARGS xlrd>=0.9.0 - time pip install $PIP_ARGS 'http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r=' - time pip install $PIP_ARGS patsy - time pip install $PIP_ARGS html5lib - - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then - time sudo apt-get $APT_ARGS remove python3-lxml - elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then - time sudo apt-get $APT_ARGS remove python-lxml - fi - - pip install $PIP_ARGS lxml # fool statsmodels into thinking pandas was already installed # so it won't refuse to install itself. diff --git a/ci/requirements-2.6.txt b/ci/requirements-2.6.txt new file mode 100644 index 0000000000000..3873f56fa6070 --- /dev/null +++ b/ci/requirements-2.6.txt @@ -0,0 +1,4 @@ +numpy==1.6.1 +cython==0.19.1 +python-dateutil==2.1 +pytz==2013b diff --git a/ci/requirements-2.7.txt b/ci/requirements-2.7.txt new file mode 100644 index 0000000000000..797066a0f1699 --- /dev/null +++ b/ci/requirements-2.7.txt @@ -0,0 +1,15 @@ +python-dateutil==2.1 +pytz==2013b +xlwt==0.7.5 +numpy==1.7.1 +cython==0.19.1 +bottleneck==0.6.0 +numexpr==2.1 +tables==2.3.1 +matplotlib==1.2.1 +openpyxl==1.6.2 +xlrd==0.9.2 +patsy==0.1.0 +html5lib==1.0b2 +lxml==3.2.1 +http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r= diff --git a/ci/requirements-2.7_LOCALE.txt b/ci/requirements-2.7_LOCALE.txt new file mode 100644 index 0000000000000..9c9306bdf1872 --- /dev/null +++ b/ci/requirements-2.7_LOCALE.txt @@ -0,0 +1,14 @@ +python-dateutil +pytz==2013b +xlwt==0.7.5 +openpyxl==1.6.2 +xlrd==0.9.2 +numpy==1.6.1 +cython==0.19.1 +bottleneck==0.6.0 +numexpr==2.1 +tables==2.3.1 +matplotlib==1.2.1 +patsy==0.1.0 +html5lib==1.0b2 +lxml==3.2.1 diff --git a/ci/requirements-3.2.txt b/ci/requirements-3.2.txt new file mode 100644 index 0000000000000..c72ccb06f5167 --- /dev/null +++ b/ci/requirements-3.2.txt @@ -0,0 +1,12 @@ +python-dateutil==2.1 +pytz==2013b +openpyxl==1.6.2 +xlrd==0.9.2 +html5lib==1.0b2 +numpy==1.6.2 +cython==0.19.1 +numexpr==2.1 +tables==3.0.0 +matplotlib==1.2.1 +patsy==0.1.0 +lxml==3.2.1 diff --git a/ci/requirements-3.3.txt b/ci/requirements-3.3.txt new file mode 100644 index 0000000000000..c00c51f4ab7d2 --- /dev/null +++ b/ci/requirements-3.3.txt @@ -0,0 +1,12 @@ +python-dateutil==2.1 +pytz==2013b +openpyxl==1.6.2 +xlrd==0.9.2 +html5lib==1.0b2 +numpy==1.7.1 +cython==0.19.1 +numexpr==2.1 +tables==3.0.0 +matplotlib==1.2.1 +patsy==0.1.0 +lxml==3.2.1 diff --git a/ci/speedpack/Vagrantfile b/ci/speedpack/Vagrantfile index d4d718548e7c8..ec939b7c0a937 100644 --- a/ci/speedpack/Vagrantfile +++ b/ci/speedpack/Vagrantfile @@ -7,6 +7,7 @@ Vagrant.configure("2") do |config| # config.vbguest.auto_update = true # config.vbguest.no_remote = true + config.vm.synced_folder File.expand_path("..", Dir.pwd), "/reqf" config.vm.synced_folder "wheelhouse", "/wheelhouse" config.vm.provider :virtualbox do |vb| diff --git a/ci/speedpack/build.sh b/ci/speedpack/build.sh index 93a7e83b97161..39994fb3f30d6 100755 --- a/ci/speedpack/build.sh +++ b/ci/speedpack/build.sh @@ -6,93 +6,71 @@ # # Runtime can be up to an hour or more. -echo "Running build.sh..." -set -x - -WHEEL_DIR=/wheelhouse -VERSIONS="2.6 2.7 3.2 3.3" -SCRIPT_FILE="/tmp/run.sh" -PARALLEL=false +echo "Building wheels..." -export PIP_ARGS=" --download-cache /tmp -w $WHEEL_DIR --use-wheel --find-links=$WHEEL_DIR" +# print a trace for everything; RTFM +set -x +# install and update some basics apt-get update apt-get install python-software-properties git -y apt-add-repository ppa:fkrull/deadsnakes -y apt-get update +# install some deps and virtualenv apt-get install python-pip libfreetype6-dev libpng12-dev -y pip install virtualenv apt-get install libhdf5-serial-dev g++ -y +apt-get build-dep python-lxml -y +export PYTHONIOENCODING='utf-8' -function generate_wheels { - VER=$1 - set -x +function generate_wheels() { + # get the requirements file + local reqfile="$1" - if [ x"$VIRTUAL_ENV" != x"" ]; then - deactivate - fi + # get the python version + local TAG=$(echo $reqfile | grep -Po "(\d\.?[\d\-](_\w+)?)") - cd ~/ - sudo rm -Rf venv-$VER - virtualenv -p python$VER venv-$VER - source venv-$VER/bin/activate + # base dir for wheel dirs + local WHEELSTREET=/wheelhouse + local WHEELHOUSE="$WHEELSTREET/$TAG" - pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip - pip install -I --download-cache /tmp https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz - pip install -I --download-cache /tmp wheel + local PY_VER="${TAG:0:3}" + local PY_MAJOR="${PY_VER:0:1}" + local PIP_ARGS="--use-wheel --find-links=$WHEELHOUSE --download-cache /tmp" - export INCLUDE_PATH=/usr/include/python$VER/ - export C_INCLUDE_PATH=/usr/include/python$VER/ - pip wheel $PIP_ARGS cython==0.19.1 - pip install --use-wheel --find-links=$WHEEL_DIR cython==0.19.1 + # install the python version if not installed + apt-get install python$PY_VER python$PY_VER-dev -y - pip wheel $PIP_ARGS numpy==1.6.1 - pip wheel $PIP_ARGS numpy==1.7.1 - pip install --use-wheel --find-links=$WHEEL_DIR numpy==1.7.1 - pip wheel $PIP_ARGS bottleneck==0.6.0 + # create a new virtualenv + rm -Rf /tmp/venv + virtualenv -p python$PY_VER /tmp/venv + source /tmp/venv/bin/activate - pip wheel $PIP_ARGS numexpr==1.4.2 - pip install --use-wheel --find-links=$WHEEL_DIR numexpr==1.4.2 - pip wheel $PIP_ARGS tables==2.3.1 - pip wheel $PIP_ARGS tables==2.4.0 + # install pip setuptools + pip install -I --download-cache /tmp 'git+https://github.com/pypa/pip@42102e9d#egg=pip' + DISTRIBUTE_VERSION= + if [ "${PY_MAJOR}" == "2" ]; then + DISTRIBUTE_VERSION="==0.6.35" + fi + pip install -I --download-cache /tmp distribute${DISTRIBUTE_VERSION} + pip install -I --download-cache /tmp wheel - pip uninstall numexpr -y - pip wheel $PIP_ARGS numexpr==2.1 - pip install --use-wheel --find-links=$WHEEL_DIR numexpr==2.1 - pip wheel $PIP_ARGS tables==3.0.0 - pip uninstall numexpr -y + # make the dir if it doesn't exist + mkdir -p $WHEELHOUSE - pip wheel $PIP_ARGS matplotlib==1.2.1 + # put the requirements file in the wheelhouse + cp $reqfile $WHEELHOUSE + + # install and build the wheels + cat $reqfile | while read N; do + pip wheel $PIP_ARGS --wheel-dir=$WHEELHOUSE $N + pip install $PIP_ARGS --no-index $N + done } -for VER in $VERSIONS ; do - apt-get install python$VER python$VER-dev -y +for reqfile in $(ls -1 /reqf/requirements-*.*); do + generate_wheels "$reqfile" done - -if $PARALLEL; then - echo '#!/bin/bash' > $SCRIPT_FILE - echo "export WHEEL_DIR=$WHEEL_DIR" >> $SCRIPT_FILE - echo "export PIP_ARGS='$PIP_ARGS'">> $SCRIPT_FILE - - declare -f generate_wheels >> $SCRIPT_FILE - echo 'generate_wheels $1' >> $SCRIPT_FILE - chmod u+x $SCRIPT_FILE - - pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip - pip install --download-cache /tmp --no-install wheel - pip install --download-cache /tmp --no-install https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz - - for VER in 2.6 2.7 3.2 3.3 ; do - $SCRIPT_FILE $VER & - done - - wait - -else - for VER in 2.6 2.7 3.2 3.3 ; do - generate_wheels $VER - done -fi diff --git a/ci/speedpack/wheelhouse/placeholder b/ci/speedpack/wheelhouse/placeholder deleted file mode 100644 index e69de29bb2d1d..0000000000000