diff --git a/.gitignore b/.gitignore index f12847a80edaf..9a0794373cfdc 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ pandas/io/*.json .idea/pandas.iml .build_cache_dir +.vagrant +*.whl \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 30f09deefd93a..ff292767011cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,13 +10,13 @@ env: matrix: include: - python: 2.7 - env: NOSE_ARGS="slow and not network" LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true UPLOAD=false + env: NOSE_ARGS="slow and not network" LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true - python: 2.7 - env: NOSE_ARGS="not slow" FULL_DEPS=true UPLOAD=true + env: NOSE_ARGS="not slow" FULL_DEPS=true - python: 3.2 - env: NOSE_ARGS="not slow" FULL_DEPS=true UPLOAD=true + env: NOSE_ARGS="not slow" FULL_DEPS=true - python: 3.3 - env: NOSE_ARGS="not slow" UPLOAD=true + env: NOSE_ARGS="not slow" # allow importing from site-packages, # so apt-get python-x works for system pythons diff --git a/ci/install.sh b/ci/install.sh index 60ea5643c6ad2..54de404854af8 100755 --- a/ci/install.sh +++ b/ci/install.sh @@ -14,11 +14,18 @@ # echo "inside $0" + # Install Dependencie -SITE_PKG_DIR=$VIRTUAL_ENV/lib/python$TRAVIS_PYTHON_VERSION/site-packages -echo "Using SITE_PKG_DIR: $SITE_PKG_DIR" +# as of pip 1.4rc2, wheel files are still being broken regularly, this is a known good +# commit. should revert to pypi when a final release is out +pip install -I git+https://github.com/pypa/pip@42102e9deaea99db08b681d06906c2945f6f95e2#egg=pip +pip Install -I https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz +pip install wheel + +# comment this line to disable the fetching of wheel files +PIP_ARGS+=" -I --use-wheel --find-links=https://cache27-pypandas.rhcloud.com/" -# workaround for travis ignoring system_site_packages in travis.yml +# Force virtualenv to accpet system_site_packages rm -f $VIRTUAL_ENV/lib/python$TRAVIS_PYTHON_VERSION/no-global-site-packages.txt if [ x"$LOCALE_OVERRIDE" != x"" ]; then @@ -30,75 +37,73 @@ fi; #scipy is not included in the cached venv if [ x"$FULL_DEPS" == x"true" ] ; then # for pytables gets the lib as well - sudo apt-get $APT_ARGS install libhdf5-serial-dev + time sudo apt-get $APT_ARGS install libhdf5-serial-dev if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then - sudo apt-get $APT_ARGS install python3-bs4 + time sudo apt-get $APT_ARGS install python3-bs4 elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then - sudo apt-get $APT_ARGS install python-bs4 + time sudo apt-get $APT_ARGS install python-bs4 fi if [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then - sudo apt-get $APT_ARGS install python3-scipy + time sudo apt-get $APT_ARGS install python3-scipy elif [ ${TRAVIS_PYTHON_VERSION} == "2.7" ]; then - sudo apt-get $APT_ARGS install python-scipy + time sudo apt-get $APT_ARGS install python-scipy fi fi # Hard Deps -pip install $PIP_ARGS nose python-dateutil pytz -pip install $PIP_ARGS cython +time pip install $PIP_ARGS nose python-dateutil pytz>=2013a +time pip install $PIP_ARGS cython==0.19.1 if [ ${TRAVIS_PYTHON_VERSION} == "3.3" ]; then # should be >=3,3 - pip install $PIP_ARGS numpy==1.7.0 + time pip install $PIP_ARGS numpy==1.7.1 elif [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then # sudo apt-get $APT_ARGS install python3-numpy; # 1.6.2 or precise - pip install $PIP_ARGS numpy==1.6.1 + time pip install $PIP_ARGS numpy==1.6.1 else - pip install $PIP_ARGS numpy==1.6.1 + time pip install $PIP_ARGS numpy==1.6.1 fi # Optional Deps if [ x"$FULL_DEPS" == x"true" ]; then echo "Installing FULL_DEPS" - pip install $PIP_ARGS cython if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then - pip install $PIP_ARGS xlwt - pip install $PIP_ARGS bottleneck - pip install $PIP_ARGS numexpr==2.0.1 - pip install $PIP_ARGS tables==2.3.1 + time pip install $PIP_ARGS xlwt + time pip install $PIP_ARGS bottleneck==0.6.0 + time pip install $PIP_ARGS numexpr==2.1 + time pip install $PIP_ARGS tables==2.3.1 else - pip install $PIP_ARGS numexpr - pip install $PIP_ARGS tables + time pip install $PIP_ARGS numexpr==2.1 + time pip install $PIP_ARGS tables==3.0.0 fi - pip install $PIP_ARGS matplotlib - pip install $PIP_ARGS openpyxl - pip install $PIP_ARGS xlrd>=0.9.0 - pip install $PIP_ARGS 'http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r=' - pip install $PIP_ARGS patsy - pip install $PIP_ARGS html5lib + time pip install $PIP_ARGS matplotlib==1.2.1 + time pip install $PIP_ARGS openpyxl + time pip install $PIP_ARGS xlrd>=0.9.0 + time pip install $PIP_ARGS 'http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r=' + time pip install $PIP_ARGS patsy + time pip install $PIP_ARGS html5lib if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then - sudo apt-get $APT_ARGS remove python3-lxml + time sudo apt-get $APT_ARGS remove python3-lxml elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then - sudo apt-get $APT_ARGS remove python-lxml + time sudo apt-get $APT_ARGS remove python-lxml fi - pip install $PIP_ARGS lxml # fool statsmodels into thinking pandas was already installed # so it won't refuse to install itself. mkdir $SITE_PKG_DIR/pandas touch $SITE_PKG_DIR/pandas/__init__.py echo "version='0.10.0-phony'" > $SITE_PKG_DIR/pandas/version.py - pip install $PIP_ARGS git+git://github.com/statsmodels/statsmodels@c9062e43b8a5f7385537ca95#egg=statsmodels + time pip install $PIP_ARGS git+git://github.com/statsmodels/statsmodels@c9062e43b8a5f7385537ca95#egg=statsmodels rm -Rf $SITE_PKG_DIR/pandas # scrub phoney pandas fi # build pandas -python setup.py build_ext install +time python setup.py build_ext install true diff --git a/ci/speedpack/Vagrantfile b/ci/speedpack/Vagrantfile new file mode 100644 index 0000000000000..d4d718548e7c8 --- /dev/null +++ b/ci/speedpack/Vagrantfile @@ -0,0 +1,21 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure("2") do |config| + config.vm.box = "precise64" + config.vm.box_url = "http://files.vagrantup.com/precise64.box" + +# config.vbguest.auto_update = true +# config.vbguest.no_remote = true + + config.vm.synced_folder "wheelhouse", "/wheelhouse" + + config.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--cpus", "4"] + vb.customize ["modifyvm", :id, "--memory", "2048"] + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"] + end + + config.vm.provision :shell, :path => "build.sh" + +end diff --git a/ci/speedpack/build.sh b/ci/speedpack/build.sh new file mode 100755 index 0000000000000..93a7e83b97161 --- /dev/null +++ b/ci/speedpack/build.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# This script is meant to run on a mint precise64 VM. +# The generated wheel files should be compatible +# with travis-ci as of 07/2013. +# +# Runtime can be up to an hour or more. + +echo "Running build.sh..." +set -x + +WHEEL_DIR=/wheelhouse +VERSIONS="2.6 2.7 3.2 3.3" +SCRIPT_FILE="/tmp/run.sh" +PARALLEL=false + +export PIP_ARGS=" --download-cache /tmp -w $WHEEL_DIR --use-wheel --find-links=$WHEEL_DIR" + +apt-get update +apt-get install python-software-properties git -y +apt-add-repository ppa:fkrull/deadsnakes -y +apt-get update + +apt-get install python-pip libfreetype6-dev libpng12-dev -y +pip install virtualenv +apt-get install libhdf5-serial-dev g++ -y + + +function generate_wheels { + VER=$1 + set -x + + if [ x"$VIRTUAL_ENV" != x"" ]; then + deactivate + fi + + cd ~/ + sudo rm -Rf venv-$VER + virtualenv -p python$VER venv-$VER + source venv-$VER/bin/activate + + pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip + pip install -I --download-cache /tmp https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz + pip install -I --download-cache /tmp wheel + + export INCLUDE_PATH=/usr/include/python$VER/ + export C_INCLUDE_PATH=/usr/include/python$VER/ + pip wheel $PIP_ARGS cython==0.19.1 + pip install --use-wheel --find-links=$WHEEL_DIR cython==0.19.1 + + pip wheel $PIP_ARGS numpy==1.6.1 + pip wheel $PIP_ARGS numpy==1.7.1 + pip install --use-wheel --find-links=$WHEEL_DIR numpy==1.7.1 + pip wheel $PIP_ARGS bottleneck==0.6.0 + + pip wheel $PIP_ARGS numexpr==1.4.2 + pip install --use-wheel --find-links=$WHEEL_DIR numexpr==1.4.2 + pip wheel $PIP_ARGS tables==2.3.1 + pip wheel $PIP_ARGS tables==2.4.0 + + pip uninstall numexpr -y + pip wheel $PIP_ARGS numexpr==2.1 + pip install --use-wheel --find-links=$WHEEL_DIR numexpr==2.1 + pip wheel $PIP_ARGS tables==3.0.0 + pip uninstall numexpr -y + + pip wheel $PIP_ARGS matplotlib==1.2.1 +} + + +for VER in $VERSIONS ; do + apt-get install python$VER python$VER-dev -y +done + +if $PARALLEL; then + echo '#!/bin/bash' > $SCRIPT_FILE + echo "export WHEEL_DIR=$WHEEL_DIR" >> $SCRIPT_FILE + echo "export PIP_ARGS='$PIP_ARGS'">> $SCRIPT_FILE + + declare -f generate_wheels >> $SCRIPT_FILE + echo 'generate_wheels $1' >> $SCRIPT_FILE + chmod u+x $SCRIPT_FILE + + pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip + pip install --download-cache /tmp --no-install wheel + pip install --download-cache /tmp --no-install https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz + + for VER in 2.6 2.7 3.2 3.3 ; do + $SCRIPT_FILE $VER & + done + + wait + +else + for VER in 2.6 2.7 3.2 3.3 ; do + generate_wheels $VER + done +fi diff --git a/ci/speedpack/nginx/nginx.conf.template b/ci/speedpack/nginx/nginx.conf.template new file mode 100644 index 0000000000000..e2cfeaf053d08 --- /dev/null +++ b/ci/speedpack/nginx/nginx.conf.template @@ -0,0 +1,48 @@ +#user nobody; +worker_processes 1; + +#error_log logs/error.log; +#error_log logs/error.log notice; +#error_log logs/error.log info; + +#pid logs/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include mime.types; + default_type application/octet-stream; + + #log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + # '$status $body_bytes_sent "$http_referer" ' + # '"$http_user_agent" "$http_x_forwarded_for"'; + + #access_log logs/access.log on; + + sendfile on; + #tcp_nopush on; + + #keepalive_timeout 0; + keepalive_timeout 65; + + #gzip on; + + server { + listen $OPENSHIFT_IP:$OPENSHIFT_PORT; + + access_log access.log ; + sendfile on; + + location / { + root ../../app-root/data/store/; + autoindex on; + } + + + } + +} diff --git a/ci/speedpack/wheelhouse/placeholder b/ci/speedpack/wheelhouse/placeholder new file mode 100644 index 0000000000000..e69de29bb2d1d