BLD: add requirements files and update install.sh

This commit adds 5 requirements files: one for each Travis build. The
corresponding sections in ci/install.sh are also removed in favor of these
files.

This allows pandas to have more fine-grained control over versions of
hard and soft deps when using travis.

.gitignore

@@ -36,4 +36,5 @@ pandas/io/*.json
 .idea/pandas.iml
 .build_cache_dir
 .vagrant
-*.whl
+*.whl
+**/wheelhouse/*

.travis.yml

@@ -10,13 +10,13 @@ env:
 matrix:
   include:
     - python: 2.7
-      env: NOSE_ARGS="slow and not network"  LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true
+      env: NOSE_ARGS="slow and not network"  LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true JOB_TAG=_LOCALE
     - python: 2.7
       env: NOSE_ARGS="not slow" FULL_DEPS=true
     - python: 3.2
       env: NOSE_ARGS="not slow" FULL_DEPS=true
     - python: 3.3
-      env: NOSE_ARGS="not slow"
+      env: NOSE_ARGS="not slow" FULL_DEPS=true
 
 # allow importing from site-packages,
 # so apt-get python-x works for system pythons

ci/install.sh

@@ -15,85 +15,42 @@
 
 echo "inside $0"
 
-# Install Dependencie
+# Install Dependencies
 # as of pip 1.4rc2, wheel files are still being broken regularly, this is a known good
 # commit. should revert to pypi when a final release is out
 pip install -I git+https://github.com/pypa/pip@42102e9deaea99db08b681d06906c2945f6f95e2#egg=pip
-pip Install -I https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz
+pv="${TRAVIS_PYTHON_VERSION:0:1}"
+[ "$pv" == "2" ] && pv=""
+[ "$pv" == "2" ] && DISTRIBUTE_VERSION="==0.6.35"
+
+pip install -I distribute${DISTRIBUTE_VERSION}
 pip install wheel
 
 # comment this line to disable the fetching of wheel files
-#PIP_ARGS+=" -I --use-wheel --find-links=https://cache27-pypandas.rhcloud.com/"
+PIP_ARGS+=" -I --use-wheel --find-links=http://cache27diy-cpycloud.rhcloud.com/${TRAVIS_PYTHON_VERSION}${JOB_TAG}/"
 
 # Force virtualenv to accpet system_site_packages
 rm -f $VIRTUAL_ENV/lib/python$TRAVIS_PYTHON_VERSION/no-global-site-packages.txt
 
-if [ x"$LOCALE_OVERRIDE" != x"" ]; then
+
+if [ -n "$LOCALE_OVERRIDE" ]; then
     # make sure the locale is available
     # probably useless, since you would need to relogin
     sudo locale-gen "$LOCALE_OVERRIDE"
-fi;
-
-#scipy is not included in the cached venv
-if [ x"$FULL_DEPS" == x"true" ] ; then
-   # for pytables gets the lib as well
-  time sudo apt-get $APT_ARGS install libhdf5-serial-dev
-
-   if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then
-     time sudo apt-get $APT_ARGS install python3-bs4
-   elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
-     time sudo apt-get $APT_ARGS install python-bs4
-   fi
-
-   if [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then
-       time sudo apt-get $APT_ARGS install python3-scipy
-   elif [ ${TRAVIS_PYTHON_VERSION} == "2.7" ]; then
-       time sudo apt-get $APT_ARGS install python-scipy
-   fi
 fi
 
-# Hard Deps
-for dep in nose 'python-dateutil' 'pytz>=2013a' 'cython==0.19.1'; do
-    time pip install $PIP_ARGS $dep
-done
-
-if [ ${TRAVIS_PYTHON_VERSION} == "3.3" ]; then # should be >=3,3
-    time pip install $PIP_ARGS numpy==1.7.1
-elif [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then
-    # sudo apt-get $APT_ARGS install python3-numpy; # 1.6.2 or precise
-    time pip install $PIP_ARGS numpy==1.6.1
-else
-    time pip install $PIP_ARGS numpy==1.6.1
-fi
+time pip install $PIP_ARGS -r ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.txt
 
 # Optional Deps
 if [ x"$FULL_DEPS" == x"true" ]; then
     echo "Installing FULL_DEPS"
+   # for pytables gets the lib as well
+    time sudo apt-get $APT_ARGS install libhdf5-serial-dev
+    time sudo apt-get $APT_ARGS install python${pv}-bs4
+    time sudo apt-get $APT_ARGS install python${pv}-scipy
+
+    time sudo apt-get $APT_ARGS remove python${pv}-lxml
 
-    if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
-        time pip install $PIP_ARGS xlwt
-        time pip install $PIP_ARGS bottleneck==0.6.0
-        time pip install $PIP_ARGS numexpr==2.1
-        time pip install $PIP_ARGS tables==2.3.1
-    else
-        time pip install $PIP_ARGS numexpr==2.1
-        time pip install $PIP_ARGS tables==3.0.0
-    fi
-
-    time pip install $PIP_ARGS matplotlib==1.2.1
-    time pip install $PIP_ARGS openpyxl
-    time pip install $PIP_ARGS xlrd>=0.9.0
-    time pip install $PIP_ARGS 'http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r='
-    time pip install $PIP_ARGS patsy
-    time pip install $PIP_ARGS html5lib
-
-    if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then
-      time sudo apt-get $APT_ARGS remove python3-lxml
-    elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
-      time sudo apt-get $APT_ARGS remove python-lxml
-    fi
-
-    pip install $PIP_ARGS lxml
     # fool statsmodels into thinking pandas was already installed
     # so it won't refuse to install itself.
 

ci/requirements-2.6.txt

@@ -0,0 +1,4 @@
+numpy==1.6.1
+cython==0.19.1
+python-dateutil==2.1
+pytz==2013b

ci/requirements-2.7.txt

@@ -0,0 +1,15 @@
+python-dateutil==2.1
+pytz==2013b
+xlwt==0.7.5
+numpy==1.7.1
+cython==0.19.1
+bottleneck==0.6.0
+numexpr==2.1
+tables==2.3.1
+matplotlib==1.2.1
+openpyxl==1.6.2
+xlrd==0.9.2
+patsy==0.1.0
+html5lib==1.0b2
+lxml==3.2.1
+http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r=

ci/requirements-2.7_LOCALE.txt

@@ -0,0 +1,14 @@
+python-dateutil
+pytz==2013b
+xlwt==0.7.5
+openpyxl==1.6.2
+xlrd==0.9.2
+numpy==1.6.1
+cython==0.19.1
+bottleneck==0.6.0
+numexpr==2.1
+tables==2.3.1
+matplotlib==1.2.1
+patsy==0.1.0
+html5lib==1.0b2
+lxml==3.2.1

ci/requirements-3.2.txt

@@ -0,0 +1,12 @@
+python-dateutil==2.1
+pytz==2013b
+openpyxl==1.6.2
+xlrd==0.9.2
+html5lib==1.0b2
+numpy==1.6.2
+cython==0.19.1
+numexpr==2.1
+tables==3.0.0
+matplotlib==1.2.1
+patsy==0.1.0
+lxml==3.2.1

ci/requirements-3.3.txt

@@ -0,0 +1,12 @@
+python-dateutil==2.1
+pytz==2013b
+openpyxl==1.6.2
+xlrd==0.9.2
+html5lib==1.0b2
+numpy==1.7.1
+cython==0.19.1
+numexpr==2.1
+tables==3.0.0
+matplotlib==1.2.1
+patsy==0.1.0
+lxml==3.2.1

ci/speedpack/Vagrantfile

@@ -7,6 +7,7 @@ Vagrant.configure("2") do |config|
 #  config.vbguest.auto_update = true
 #  config.vbguest.no_remote = true
 
+  config.vm.synced_folder File.expand_path("..", Dir.pwd), "/reqf"
   config.vm.synced_folder "wheelhouse", "/wheelhouse"
 
   config.vm.provider :virtualbox do |vb|

ci/speedpack/build.sh

@@ -6,93 +6,71 @@
 #
 # Runtime can be up to an hour or more.
 
-echo "Running build.sh..."
-set -x
-
-WHEEL_DIR=/wheelhouse
-VERSIONS="2.6 2.7 3.2 3.3"
-SCRIPT_FILE="/tmp/run.sh"
-PARALLEL=false
+echo "Building wheels..."
 
-export PIP_ARGS=" --download-cache /tmp -w $WHEEL_DIR --use-wheel --find-links=$WHEEL_DIR"
+# print a trace for everything; RTFM
+set -x
 
+# install and update some basics
 apt-get update
 apt-get install python-software-properties git -y
 apt-add-repository ppa:fkrull/deadsnakes -y
 apt-get update
 
+# install some deps and virtualenv
 apt-get install python-pip libfreetype6-dev libpng12-dev -y
 pip install virtualenv
 apt-get install libhdf5-serial-dev g++ -y
+apt-get build-dep python-lxml -y
 
+export PYTHONIOENCODING='utf-8'
 
-function generate_wheels {
-	VER=$1
-	set -x
+function generate_wheels() {
+    # get the requirements file
+    local reqfile="$1"
 
-	if [ x"$VIRTUAL_ENV" != x"" ]; then
-		deactivate
-	fi
+    # get the python version
+    local TAG=$(echo $reqfile |  grep -Po "(\d\.?[\d\-](_\w+)?)")
 
-	cd ~/
-	sudo rm -Rf venv-$VER
-	virtualenv -p python$VER venv-$VER
-	source venv-$VER/bin/activate
+    # base dir for wheel dirs
+    local WHEELSTREET=/wheelhouse
+    local WHEELHOUSE="$WHEELSTREET/$TAG"
 
-	pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip
-	pip install -I --download-cache  /tmp https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz
-	pip install -I --download-cache /tmp wheel
+    local PY_VER="${TAG:0:3}"
+    local PY_MAJOR="${PY_VER:0:1}"
+    local PIP_ARGS="--use-wheel --find-links=$WHEELHOUSE --download-cache /tmp"
 
-	export INCLUDE_PATH=/usr/include/python$VER/
-	export C_INCLUDE_PATH=/usr/include/python$VER/
-	pip wheel $PIP_ARGS cython==0.19.1
-	pip install --use-wheel --find-links=$WHEEL_DIR cython==0.19.1
+    # install the python version if not installed
+    apt-get install python$PY_VER python$PY_VER-dev -y
 
-	pip wheel $PIP_ARGS numpy==1.6.1
-	pip wheel $PIP_ARGS numpy==1.7.1
-	pip install --use-wheel --find-links=$WHEEL_DIR numpy==1.7.1
-	pip wheel $PIP_ARGS bottleneck==0.6.0
+    # create a new virtualenv
+    rm -Rf /tmp/venv
+    virtualenv -p python$PY_VER /tmp/venv
+    source /tmp/venv/bin/activate
 
-	pip wheel $PIP_ARGS numexpr==1.4.2
-	pip install --use-wheel --find-links=$WHEEL_DIR numexpr==1.4.2
-	pip wheel $PIP_ARGS tables==2.3.1
-	pip wheel $PIP_ARGS tables==2.4.0
+    # install pip setuptools
+    pip install -I --download-cache /tmp 'git+https://github.com/pypa/pip@42102e9d#egg=pip'
+    DISTRIBUTE_VERSION=
+    if [ "${PY_MAJOR}" == "2" ]; then
+        DISTRIBUTE_VERSION="==0.6.35"
+    fi
+    pip install -I --download-cache /tmp distribute${DISTRIBUTE_VERSION}
+    pip install -I --download-cache /tmp wheel
 
-	pip uninstall numexpr -y
-	pip wheel $PIP_ARGS numexpr==2.1
-	pip install --use-wheel --find-links=$WHEEL_DIR numexpr==2.1
-	pip wheel $PIP_ARGS tables==3.0.0
-	pip uninstall numexpr -y
+    # make the dir if it doesn't exist
+    mkdir -p $WHEELHOUSE
 
-	pip wheel $PIP_ARGS matplotlib==1.2.1
+    # put the requirements file in the wheelhouse
+    cp $reqfile $WHEELHOUSE
+
+    # install and build the wheels
+    cat $reqfile | while read N; do
+        pip wheel $PIP_ARGS --wheel-dir=$WHEELHOUSE $N
+        pip install $PIP_ARGS --no-index $N
+    done
 }
 
 
-for VER in $VERSIONS ; do
-	apt-get install python$VER python$VER-dev -y
+for reqfile in $(ls -1 /reqf/requirements-*.*); do
+    generate_wheels "$reqfile"
 done
-
-if $PARALLEL; then
-	echo '#!/bin/bash' > $SCRIPT_FILE
-	echo "export WHEEL_DIR=$WHEEL_DIR" >> $SCRIPT_FILE
-	echo "export PIP_ARGS='$PIP_ARGS'">> $SCRIPT_FILE
-
-	declare -f generate_wheels >>  $SCRIPT_FILE
-	echo 'generate_wheels $1' >> $SCRIPT_FILE
-	chmod u+x $SCRIPT_FILE
-
-	pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip
-	pip install --download-cache /tmp --no-install wheel
-	pip install --download-cache /tmp --no-install https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz
-
-	for VER in 2.6 2.7 3.2 3.3 ; do
-		$SCRIPT_FILE  $VER &
-	done
-
-	wait
-
-else
-	for VER in 2.6 2.7 3.2 3.3 ; do
-		generate_wheels $VER
-	done
-fi