use docker containers for postgres and redis

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "GeoServer-Docker"]
+	path = GeoServer-Docker
+	url = https://github.com/thinkWhere/GeoServer-Docker

Dockerfile

@@ -1,31 +1,56 @@
 FROM python:2.7
 
-RUN mkdir -p /usr/src/app
-WORKDIR /usr/src/app
+RUN mkdir -p /projects/PublicMapping/DistrictBuilder
+WORKDIR /projects/PublicMapping/DistrictBuilder
 
 RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
 
-ENV PG_MAJOR 9.5
-ENV PG_VERSION 9.4+165+deb8u2
-
 RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' ${PG_MAJOR} > /etc/apt/sources.list.d/pgdg.list
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         gdal-bin \
         gettext \
-        postgresql-client=${PG_VERSION} \
+        postgresql-client \
+        libfreetype6-dev \
+        r-base-dev \
+        build-essential \
+        libgeos-dev \
+        default-jdk \
     && rm -rf /var/lib/apt/lists/*
 
-COPY requirements.txt /usr/src/app/
+COPY requirements.txt /projects/PublicMapping/DistrictBuilder
 RUN pip install --no-cache-dir \
         numpy==$(grep "numpy" requirements.txt | cut -d= -f3) \
         scipy==$(grep "scipy" requirements.txt | cut -d= -f3) \
     && pip install --no-cache-dir -r requirements.txt
 
-COPY . /usr/src/app
+RUN R CMD javareconf &&\
+    Rscript -e 'install.packages("R2HTML", dependencies=TRUE, repos="http://cran.us.r-project.org")' && \
+    Rscript -e 'install.packages("stringr", dependencies=TRUE, repos="http://cran.us.r-project.org")' && \
+    Rscript -e 'install.packages("ineq", dependencies=TRUE, repos="http://cran.us.r-project.org")' && \
+    Rscript -e 'install.packages("rgeos", repos="http://R-Forge.R-project.org", dependencies=TRUE)' && \
+    Rscript -e 'install.packages("BARD", dependencies=TRUE, repos="http://cran.us.r-project.org")'
+
+RUN useradd -G www-data --create-home --shell /bin/bash celery
+
+RUN mkdir -p /projects/PublicMapping/data && cd /projects/PublicMapping/data && \
+    wget --no-check-certificate -O VA_data.zip https://s3.amazonaws.com/districtbuilderdata/VA_data.zip && \
+    mkdir -p /projects/PublicMapping/local/data && cd /projects/PublicMapping/local/data && \
+    unzip /projects/PublicMapping/data/VA_data.zip
+
+COPY . /projects/PublicMapping/DistrictBuilder
+
+RUN wget --no-check-certificate -O /etc/init.d/celeryd \
+         https://raw.githubusercontent.com/celery/celery/master/extra/generic-init.d/celeryd && \
+         chmod a+x /etc/init.d/celeryd  && \
+         update-rc.d celeryd defaults && \
+         ./write_celeryd.sh && \
+         mkdir /var/log/celery /var/run/celery && \
+         chown -R celery:www-data /var/log/celery /var/run/celery/ && \
+         chmod -R 2775 /var/log/celery /var/run/celery/
+
 
-RUN cp docs/config.dist.xml docs/config.xml \
-    && (cd django/publicmapping && python setup.py ../../docs/config.xsd ../../docs/config.xml -v2 -d)
+RUN cat docs/config.dist.xml | sed -e 's|<Database name="YOUR-DATABASE-NAME" user="publicmapping" password="YOUR-DATABASE-PASSWORD" host="OPTIONAL"/>|<Database name="postgres" user="postgres" password="postgres" host="postgres"/>|' > docs/config.xml
 
-ENTRYPOINT ["python"]
-CMD ["django/publicmapping/manage.py"]
+ENTRYPOINT ["bash"]
+CMD ["start.sh"]

django/publicmapping/redistricting/tasks.py

@@ -25,7 +25,7 @@
 """
 
 from celery.task import task
-from celery.task.http import HttpDispatchTask
+from celery.task.http import HttpDispatch
 from codecs import open
 from django.core import management
 from django.contrib.comments.models import Comment
@@ -997,9 +997,9 @@ def createreport(planid, stamp, request, language=None):
 
         logger.debug('Firing web worker task.')
 
-        dispatcher = HttpDispatchTask()
+        dispatcher = HttpDispatch()
 
-        # Callbacks do not fire for HttpDispatchTask -- why not?
+        # Callbacks do not fire for HttpDispatch -- why not?
         #
         #def failure(self, exc, task_id, args, kwargs, einfo=None):
         #    self.log.get_default_logger().info('  CALLBACK: Failure!')

docker-compose.yml

@@ -1,19 +1,22 @@
-version: "3"
-services:
-  # database:
-  #   image: quay.io/azavea/postgis:2.2-postgres9.5-slim
-  #   environment:
-  #     - POSTGRES_DB=district_builder
-  #     - POSTGRES_USER=district_builder
-  #     - POSTGRES_PASSWORD=district_builder
+postgres:
+  image: mdillon/postgis:9.6-alpine
+  environment:
+   - POSTGRES_USER=postgres
+   - POSTGRES_PASSWORD=postgres
 
-  # redis:
-  #   image: redis:2-alpine
+geoserver:
+  image: thinkwhere/geoserver
 
-  django:
-    build: .
-    ports:
-      - "8080:8080"
-    command:
-      - "publicmapping/manage.py"
-      - "run_gunicorn"
+redis:
+  image: redis:3-alpine
+
+django:
+  build: .
+  links:
+    - postgres
+    - redis
+    - geoserver
+  ports:
+    - 8080:8080
+  command:
+    - /projects/PublicMapping/DistrictBuilder/start.sh

docs/config.dist.xml

@@ -1276,8 +1276,8 @@
             is to have no base layer, set it to the empty string.
 
             -->
-            <MapServer hostname="" ns="pmp" nshref="https://github.com/PublicMapping/"
-                adminuser="GEOSERVER-ADMIN-USER" adminpass="GEOSERVER-ADMIN-PASS" maxfeatures="100" 
+            <MapServer hostname="geoserver" ns="pmp" nshref="https://github.com/PublicMapping/"
+                adminuser="admin" adminpass="geoserver" maxfeatures="100" 
                 styles="/projects/PublicMapping/DistrictBuilder/sld" />
 
             <!-- 

docs/html/publicmapping.redistricting.tasks-module.html

@@ -382,7 +382,7 @@ <h1 class="epydoc">Module tasks</h1><p class="nomargin-top"><span class="codelin
 </table>
 <p class="indent-wrapped-lines"><b>Imports:</b>
   <span title="celery.task.task">task</span>,
-  <span title="celery.task.http.HttpDispatchTask">HttpDispatchTask</span>,
+  <span title="celery.task.http.HttpDispatch">HttpDispatch</span>,
   <span title="codecs.open">open</span>,
   <span title="django.core.management">management</span>,
   <span title="django.contrib.comments.models.Comment">Comment</span>,

docs/html/publicmapping.redistricting.tasks-pysrc.html

@@ -89,7 +89,7 @@ <h1 class="epydoc">Source Code for <a href="publicmapping.redistricting.tasks-mo
 <a name="L25"></a><tt class="py-lineno">  25</tt>  <tt class="py-line"><tt class="py-docstring">"""</tt> </tt>
 <a name="L26"></a><tt class="py-lineno">  26</tt>  <tt class="py-line"> </tt>
 <a name="L27"></a><tt class="py-lineno">  27</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">celery</tt><tt class="py-op">.</tt><tt class="py-name">task</tt> <tt class="py-keyword">import</tt> <tt class="py-name">task</tt> </tt>
-<a name="L28"></a><tt class="py-lineno">  28</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">celery</tt><tt class="py-op">.</tt><tt class="py-name">task</tt><tt class="py-op">.</tt><tt class="py-name">http</tt> <tt class="py-keyword">import</tt> <tt class="py-name">HttpDispatchTask</tt> </tt>
+<a name="L28"></a><tt class="py-lineno">  28</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">celery</tt><tt class="py-op">.</tt><tt class="py-name">task</tt><tt class="py-op">.</tt><tt class="py-name">http</tt> <tt class="py-keyword">import</tt> <tt class="py-name">HttpDispatch</tt> </tt>
 <a name="L29"></a><tt class="py-lineno">  29</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">codecs</tt> <tt class="py-keyword">import</tt> <tt class="py-name">open</tt> </tt>
 <a name="L30"></a><tt class="py-lineno">  30</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">django</tt><tt class="py-op">.</tt><tt class="py-name">core</tt> <tt class="py-keyword">import</tt> <tt id="link-0" class="py-name" targets="Package publicmapping.redistricting.management=publicmapping.redistricting.management-module.html"><a title="publicmapping.redistricting.management" class="py-name" href="#" onclick="return doclink('link-0', 'management', 'link-0');">management</a></tt> </tt>
 <a name="L31"></a><tt class="py-lineno">  31</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">django</tt><tt class="py-op">.</tt><tt class="py-name">contrib</tt><tt class="py-op">.</tt><tt class="py-name">comments</tt><tt class="py-op">.</tt><tt id="link-1" class="py-name" targets="Module publicmapping.redistricting.models=publicmapping.redistricting.models-module.html"><a title="publicmapping.redistricting.models" class="py-name" href="#" onclick="return doclink('link-1', 'models', 'link-1');">models</a></tt> <tt class="py-keyword">import</tt> <tt class="py-name">Comment</tt> </tt>
@@ -1820,9 +1820,9 @@ <h1 class="epydoc">Source Code for <a href="publicmapping.redistricting.tasks-mo
 publicmapping.setup.logger
 publicmapping.views.logger" class="py-name" href="#" onclick="return doclink('link-384', 'logger', 'link-11');">logger</a></tt><tt class="py-op">.</tt><tt class="py-name">debug</tt><tt class="py-op">(</tt><tt class="py-string">'Firing web worker task.'</tt><tt class="py-op">)</tt> </tt>
 <a name="L1063"></a><tt class="py-lineno">1063</tt>  <tt class="py-line"> </tt>
-<a name="L1064"></a><tt class="py-lineno">1064</tt>  <tt class="py-line">        <tt class="py-name">dispatcher</tt> <tt class="py-op">=</tt> <tt class="py-name">HttpDispatchTask</tt><tt class="py-op">(</tt><tt class="py-op">)</tt> </tt>
+<a name="L1064"></a><tt class="py-lineno">1064</tt>  <tt class="py-line">        <tt class="py-name">dispatcher</tt> <tt class="py-op">=</tt> <tt class="py-name">HttpDispatch</tt><tt class="py-op">(</tt><tt class="py-op">)</tt> </tt>
 <a name="L1065"></a><tt class="py-lineno">1065</tt>  <tt class="py-line"> </tt>
-<a name="L1066"></a><tt class="py-lineno">1066</tt>  <tt class="py-line">        <tt class="py-comment"># Callbacks do not fire for HttpDispatchTask -- why not?</tt> </tt>
+<a name="L1066"></a><tt class="py-lineno">1066</tt>  <tt class="py-line">        <tt class="py-comment"># Callbacks do not fire for HttpDispatch -- why not?</tt> </tt>
 <a name="L1067"></a><tt class="py-lineno">1067</tt>  <tt class="py-line">        <tt class="py-comment">#</tt> </tt>
 <a name="L1068"></a><tt class="py-lineno">1068</tt>  <tt class="py-line">        <tt class="py-comment">#def failure(self, exc, task_id, args, kwargs, einfo=None):</tt> </tt>
 <a name="L1069"></a><tt class="py-lineno">1069</tt>  <tt class="py-line">        <tt class="py-comment">#    self.log.get_default_logger().info('  CALLBACK: Failure!')</tt> </tt>

docs/html/publicmapping.redistricting.utils-module.html

@@ -189,7 +189,7 @@ <h1 class="epydoc">Module utils</h1><p class="nomargin-top"><span class="codelin
 </table>
 <p class="indent-wrapped-lines"><b>Imports:</b>
   <span title="celery.decorators.task">task</span>,
-  <span title="celery.task.http.HttpDispatchTask">HttpDispatchTask</span>,
+  <span title="celery.task.http.HttpDispatch">HttpDispatch</span>,
   <span title="django.core.management">management</span>,
   <span title="django.contrib.comments.models.Comment">Comment</span>,
   <span title="django.contrib.sessions.models.Session">Session</span>,

docs/html/publicmapping.redistricting.utils-pysrc.html

@@ -89,7 +89,7 @@ <h1 class="epydoc">Source Code for <a href="publicmapping.redistricting.utils-mo
 <a name="L25"></a><tt class="py-lineno">  25</tt>  <tt class="py-line"><tt class="py-docstring">"""</tt> </tt>
 <a name="L26"></a><tt class="py-lineno">  26</tt>  <tt class="py-line"> </tt>
 <a name="L27"></a><tt class="py-lineno">  27</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">celery</tt><tt class="py-op">.</tt><tt class="py-name">decorators</tt> <tt class="py-keyword">import</tt> <tt class="py-name">task</tt> </tt>
-<a name="L28"></a><tt class="py-lineno">  28</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">celery</tt><tt class="py-op">.</tt><tt class="py-name">task</tt><tt class="py-op">.</tt><tt class="py-name">http</tt> <tt class="py-keyword">import</tt> <tt class="py-name">HttpDispatchTask</tt> </tt>
+<a name="L28"></a><tt class="py-lineno">  28</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">celery</tt><tt class="py-op">.</tt><tt class="py-name">task</tt><tt class="py-op">.</tt><tt class="py-name">http</tt> <tt class="py-keyword">import</tt> <tt class="py-name">HttpDispatch</tt> </tt>
 <a name="L29"></a><tt class="py-lineno">  29</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">django</tt><tt class="py-op">.</tt><tt class="py-name">core</tt> <tt class="py-keyword">import</tt> <tt id="link-0" class="py-name" targets="Package publicmapping.redistricting.management=publicmapping.redistricting.management-module.html"><a title="publicmapping.redistricting.management" class="py-name" href="#" onclick="return doclink('link-0', 'management', 'link-0');">management</a></tt> </tt>
 <a name="L30"></a><tt class="py-lineno">  30</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">django</tt><tt class="py-op">.</tt><tt class="py-name">contrib</tt><tt class="py-op">.</tt><tt class="py-name">comments</tt><tt class="py-op">.</tt><tt id="link-1" class="py-name" targets="Module publicmapping.redistricting.models=publicmapping.redistricting.models-module.html"><a title="publicmapping.redistricting.models" class="py-name" href="#" onclick="return doclink('link-1', 'models', 'link-1');">models</a></tt> <tt class="py-keyword">import</tt> <tt class="py-name">Comment</tt> </tt>
 <a name="L31"></a><tt class="py-lineno">  31</tt>  <tt class="py-line"><tt class="py-keyword">from</tt> <tt class="py-name">django</tt><tt class="py-op">.</tt><tt class="py-name">contrib</tt><tt class="py-op">.</tt><tt class="py-name">sessions</tt><tt class="py-op">.</tt><tt id="link-2" class="py-name"><a title="publicmapping.redistricting.models" class="py-name" href="#" onclick="return doclink('link-2', 'models', 'link-1');">models</a></tt> <tt class="py-keyword">import</tt> <tt class="py-name">Session</tt> </tt>
@@ -1492,9 +1492,9 @@ <h1 class="epydoc">Source Code for <a href="publicmapping.redistricting.utils-mo
 <a name="L991"></a><tt class="py-lineno"> 991</tt>  <tt class="py-line">        <tt class="py-keyword">if</tt> <tt class="py-name">settings</tt><tt class="py-op">.</tt><tt class="py-name">DEBUG</tt><tt class="py-op">:</tt> </tt>
 <a name="L992"></a><tt class="py-lineno"> 992</tt>  <tt class="py-line">            <tt class="py-name">sys</tt><tt class="py-op">.</tt><tt class="py-name">stderr</tt><tt class="py-op">.</tt><tt class="py-name">write</tt><tt class="py-op">(</tt><tt class="py-string">'Firing web worker task.'</tt><tt class="py-op">)</tt> </tt>
 <a name="L993"></a><tt class="py-lineno"> 993</tt>  <tt class="py-line"> </tt>
-<a name="L994"></a><tt class="py-lineno"> 994</tt>  <tt class="py-line">        <tt class="py-name">dispatcher</tt> <tt class="py-op">=</tt> <tt class="py-name">HttpDispatchTask</tt><tt class="py-op">(</tt><tt class="py-op">)</tt> </tt>
+<a name="L994"></a><tt class="py-lineno"> 994</tt>  <tt class="py-line">        <tt class="py-name">dispatcher</tt> <tt class="py-op">=</tt> <tt class="py-name">HttpDispatch</tt><tt class="py-op">(</tt><tt class="py-op">)</tt> </tt>
 <a name="L995"></a><tt class="py-lineno"> 995</tt>  <tt class="py-line"> </tt>
-<a name="L996"></a><tt class="py-lineno"> 996</tt>  <tt class="py-line">        <tt class="py-comment"># Callbacks do not fire for HttpDispatchTask -- why not?</tt> </tt>
+<a name="L996"></a><tt class="py-lineno"> 996</tt>  <tt class="py-line">        <tt class="py-comment"># Callbacks do not fire for HttpDispatch -- why not?</tt> </tt>
 <a name="L997"></a><tt class="py-lineno"> 997</tt>  <tt class="py-line">        <tt class="py-comment">#</tt> </tt>
 <a name="L998"></a><tt class="py-lineno"> 998</tt>  <tt class="py-line">        <tt class="py-comment">#def failure(self, exc, task_id, args, kwargs, einfo=None):</tt> </tt>
 <a name="L999"></a><tt class="py-lineno"> 999</tt>  <tt class="py-line">        <tt class="py-comment">#    self.log.get_default_logger().info('  CALLBACK: Failure!')</tt> </tt>

requirements.txt

@@ -1,6 +1,6 @@
 Gunicorn==19.7.1
 psycopg2==2.7.3
-Django==1.4.22
+Django==1.5.12
 celery==3.1.23
 django-celery==3.2.1
 django-staticfiles==1.2.1
@@ -12,7 +12,7 @@ django-compressor==1.2
 python-sld==1.0.9
 numpy==1.6.2
 scipy==0.11.0
-Pillow==1.7.8
+Pillow==2.3.0
 PySAL==1.5.0
 git+git://github.com/PublicMapping/modestmaps-py.git@db-mod_1.4
 git+git://github.com/PublicMapping/django-rosetta.git@db-mod_2.0

start.sh

@@ -0,0 +1,10 @@
+pushd /projects/PublicMapping/DistrictBuilder
+# set password so that postgres doesn't prompt for it
+echo "*:*:*:postgres:postgres" > ~/.pgpass
+chmod 600 ~/.pgpass
+./wait-for-postgres.sh postgres psql -h postgres -U postgres -w -f sql/publicmapping_db.sql
+pushd /projects/PublicMapping/DistrictBuilder/django/publicmapping
+../../wait-for-geoserver.sh geoserver python setup.py ../../docs/config.xsd ../../docs/config.xml
+service celeryd start
+python manage.py -h
+python manage.py runserver

wait-for-geoserver.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# wait-for-postgres.sh
+#  https://docs.docker.com/compose/startup-order/
+
+set -e
+
+host="$1"
+shift
+cmd="$@"
+
+until curl "http://$host:8080/geoserver/wfs?request=GetFeature&version=1.1.0&typeName=gsml:GeologicUnit"; do
+    >&2 echo "geoserver is unavailable - sleeping"
+    sleep 1
+done
+
+>&2 echo "geoserver is up - executing command"
+exec $cmd
+

wait-for-postgres.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# wait-for-postgres.sh
+#  https://docs.docker.com/compose/startup-order/
+
+set -e
+
+host="$1"
+shift
+cmd="$@"
+
+until psql -h "$host" -U postgres -w -c '\l'; do
+    >&2 echo "Postgres is unavailable - sleeping"
+    sleep 1
+done
+
+>&2 echo "Postgres is up - executing command"
+exec $cmd
+

write_celeryd.sh

@@ -0,0 +1,22 @@
+APPDIR="/projects/PublicMapping/DistrictBuilder"
+
+echo <<EOF > /etc/default/celeryd
+# Where to chdir at start.
+CELERYD_CHDIR="$APPDIR/django/publicmapping/"
+# Path to celeryd
+CELERYD="$APPDIR/django/publicmapping/manage.py celeryd_detach"
+# Options for celeryd that start the scheduler
+CELERYD_OPTS='-B -s /tmp/celerybeat-schedule'
+# Name of the projects settings module.
+export DJANGO_SETTINGS_MODULE='settings'
+# Include the 'publicmapping' module
+export PYTHONPATH=$PYTHONPATH:"$APPDIR/django/"
+# User to run celeryd as. Default is current user.
+CELERYD_USER='celery'
+# Group to run celeryd as. Default is current user.
+CELERYD_GROUP='www-data'
+# Where to put the logfile
+CELERYD_LOG_FILE='/var/log/celery/celeryd.log'
+# Where to track the process id file
+CELERYD_PID_FILE='/var/run/celery/celeryd.pid'
+EOF