Merge pull request #10 from ikks/heroku

files are served from Heroku and collectstatic runs flawlessly

.circleci/config.yml

@@ -6,9 +6,14 @@ jobs:
       CIRCLE_ENV: true
       DATABASE_URL: postgresql://root@localhost/circle_test?sslmode=disable
       INFLUXDB_HOST: localhost
+      DJANGO_SETTINGS_MODULE: canairio.settings.development
+      INFLUXDB_DATABASE: canairio
+      DATABASE_NAME: circle_test
+      ENVIRONMENT: development
+      SECRET_KEY: so_secret
     docker:
-      - image: circleci/python:3.6.4
-      - image: circleci/postgres:11
+      - image: circleci/python:3.6.4-stretch
+      - image: circleci/postgres:11-postgis-ram
         environment:
           POSTGRES_USER: root
           POSTGRES_DB: circle_test
@@ -17,20 +22,26 @@ jobs:
       - checkout
       - run: sudo chown -R circleci:circleci /usr/local/bin
       - run: sudo chown -R circleci:circleci /usr/local/lib/python3.6/site-packages
+      - run: sudo rm /etc/apt/sources.list
+      - run: echo "deb http://deb.debian.org/debian/ stretch main" | sudo tee -a /etc/apt/sources.list
+      - run: echo "Acquire::Check-Valid-Until false;" | sudo tee -a /etc/apt/apt.conf.d/10-nocheckvalid
+      - run: echo 'Package:\ *\nPin:\ origin \"archive.debian.org\"\nPin-Priority:\ 500' | sudo tee -a /etc/apt/preferences.d/10-archive-pin
+      - run: sudo apt-get update
+      - run: sudo apt-get install -y libpq-dev libxml2-dev libxmlsec1-dev libxmlsec1-openssl influxdb-client binutils libproj-dev gdal-bin
       - run:
           command: |
             sudo pip install -r requirements.txt
       - run:
           name: Lint code after importing dependencies
           command: |
             scripts/pylint.sh
-#      - run:
-#          name: Create an influxdb
-#          command: |
-#            scripts/create_influx_db.sh
-#      - run:
-#          command: |
-#            python manage.py test
+      - run:
+          name: Create an influxdb
+          command: |
+            scripts/create_influx_db.sh
+      - run:
+          command: |
+            python manage.py test --settings=canairio.settings.testing
       - store_test_results:
           path: test-results
       - store_artifacts:

.gitignore

@@ -10,3 +10,5 @@ backup_process.md
 canairioSnapshot/
 node_modules/
 __pycache__
+test-results/
+canairio/staticfiles/

Dockerfile

@@ -1,13 +1,14 @@
 FROM python:3.6
 
 RUN apt-get update
-RUN apt-get install -y libpq-dev libxml2-dev libxmlsec1-dev libxmlsec1-openssl influxdb-client
+RUN apt-get install -y libpq-dev libxml2-dev libxmlsec1-dev libxmlsec1-openssl influxdb-client binutils libproj-dev gdal-bin
 
+RUN pip install --upgrade pip
 ENV app /canairio
 
 RUN mkdir $app
 WORKDIR $app
 
 ADD . $app
-RUN pip install -r requirements.txt
-RUN python manage.py migrate
+
+COPY start $app/start

Dockerfile_timescaledb

@@ -0,0 +1,4 @@
+FROM timescale/timescaledb-postgis:latest-pg11
+ARG db_name
+ENV POSTGRES_DB $db_name
+COPY create-dbs.sh /docker-entrypoint-initdb.d/

Procfile

@@ -1 +1 @@
-web: gunicorn canairio.wsgi
+web: python manage.py collectstatic --noinput --settings=canairio.settings.production; gunicorn canairio.wsgi --log-file -

README.md

@@ -1,5 +1,7 @@
 # Canairio Backend
 
+[![CircleCI](https://circleci.com/gh/kike-canaries/canairio_backend.svg?style=svg)](https://circleci.com/gh/kike-canaries/canairio_backend)
+
 > Recibimos los datos provenientes de los [canair.ios](https://canair.io)
 
 Si tiene interés en analizar información proveniente de la red ciudadana de calidad del aire,
@@ -87,15 +89,6 @@ o para obtener la medición más reciente.
 Cada vez que haga cambios y los guarde en su entorno de desarrollo local estos
 se verán reflejados en la instancia que está ejecutando en docker.
 
-Para que pueda ver las trazas de los posibles erores por favor copie el archivo de configuración, en el mismo podrá hacer cambios de configuraciones si así lo desea.
-
-```bash
-cp scripts/dev_settings_local.py canairio/settings_local.py
-```
-
-Esto solamente debe hacerlo la primera vez, porque es posible que después quiera
-hacer más cambios sobre el mismo para su entorno de desarrollo.
-
 ### Pruebas
 
 Para correr las pruebas, ubíquese en el directorio clonado y ejecute:
@@ -109,7 +102,7 @@ docker-compose exec web bash
 Y una vez en el shell, podrá ejercitar las pruebas
 
 ```bash
-python manage.py test
+python manage.py test --settings=canairio.settings.testing
 ```
 
 Tenga en cuenta que la información se almacena en la misma instancia de

canairio/settings.py → canairio/settings/base.py

@@ -9,30 +9,34 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/2.1/ref/settings/
 """
-import django_heroku
 import os
 from dj_database_url import config
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 from datetime import timedelta
 
-ENVIRONMENT = 'development'
+def getenvvar(name):
+    v = os.environ.get(name)
+    if not v:
+        raise Exception('Environment variable {} undefined'.format(name))
+    return v
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/
 
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = '(e7f8&@wbya^bm7km^&-bd9-!0oqb3u*^62wq^3*9gqsoki#7!'
+SECRET_KEY = getenvvar('SECRET_KEY')
+
+DATABASE_URL = getenvvar('DATABASE_URL')
 
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = True
 
-ALLOWED_HOSTS = ['gblabs.co']
+DATABASES = {
+    'default': config(default=DATABASE_URL)
+}
 
-CORS_ORIGIN_WHITELIST = ['canairiofront.herokuapp.com']
+STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
 
 # Application definition
 
@@ -130,47 +134,17 @@
 
 TRACK_COLLECTION_NAME = 'test_tracks_data'
 
-FB_API_KEY = os.getenv('FB_API_KEY', '')
-FB_AUTH_DOMAIN = os.getenv('FB_AUTH_DOMAIN', '')
-FB_DATABASE_URL = os.getenv('FB_DATABASE_URL', '')
-FB_STORAGE_BUCKET = os.getenv('FB_STORAGE_BUCKET', '')
+# Firebase
+FB_API_KEY = ''
+FB_AUTH_DOMAIN = ''
+FB_DATABASE_URL = ''
+FB_STORAGE_BUCKET = ''
 
-INFLUXDB_HOST = os.getenv('INFLUXDB_HOST', '')
+INFLUXDB_HOST = getenvvar('INFLUXDB_HOST')
 INFLUXDB_PORT = 8086
 INFLUXDB_USERNAME = None
 INFLUXDB_PASSWORD = None
-INFLUXDB_DATABASE = 'canairio'
+INFLUXDB_DATABASE = getenvvar('INFLUXDB_DATABASE')
 INFLUXDB_TIMEOUT = 10
 
-ENVIRONMENT = os.getenv('ENVIRONMENT', ENVIRONMENT)
-if 'production' in ENVIRONMENT:
-
-
-    FB_API_KEY = os.getenv('FB_API_KEY', '')
-    FB_AUTH_DOMAIN = os.getenv('FB_AUTH_DOMAIN', '')
-    FB_DATABASE_URL = os.getenv('FB_DATABASE_URL', '')
-    FB_STORAGE_BUCKET = os.getenv('FB_STORAGE_BUCKET', '')
-
-    INFLUXDB_HOST = os.getenv('INFLUXDB_HOST', '')
-    INFLUXDB_PORT = 8086
-    INFLUXDB_USERNAME = None
-    INFLUXDB_PASSWORD = None
-    INFLUXDB_DATABASE = os.getenv('INFLUXDB_DATABASE', '')
-    INFLUXDB_TIMEOUT = 10
-
-    DATABASES = {
-        'default': config(
-            default=config('DATABASE_URL')
-        )
-    }
-
-else:
-    try:
-        from canairio.settings_local import *
-    except ImportError:
-        print('You must create a settings_local.py file.')
-
-django_heroku.settings(locals())
-
-if 'CIRCLE_ENV' in os.environ:
-    del DATABASES['default']['OPTIONS']['sslmode']
+ENVIRONMENT = getenvvar('ENVIRONMENT')

canairio/settings/development.py

@@ -0,0 +1,3 @@
+from .base import *
+
+ENVIRONMENT = 'development'

canairio/settings/production.py

@@ -0,0 +1,67 @@
+from .base import *
+
+import django_heroku
+
+ENVIRONMENT = 'production'
+
+DEBUG = False
+
+ALLOWED_HOSTS = ['gblabs.co']
+
+CORS_ORIGIN_WHITELIST = ['canairiofront.herokuapp.com']
+
+FB_API_KEY = getenvvar('FB_API_KEY')
+FB_AUTH_DOMAIN = getenvvar('FB_AUTH_DOMAIN')
+FB_DATABASE_URL = getenvvar('FB_DATABASE_URL')
+FB_STORAGE_BUCKET = getenvvar('FB_STORAGE_BUCKET')
+
+SECRET_KEY = getenvvar('SECRET_KEY')
+
+STATICFILES_DIRS = (
+    os.path.join(BASE_DIR, 'static'),
+)
+
+MIDDLEWARE.insert(1, 'whitenoise.middleware.WhiteNoiseMiddleware')
+
+django_heroku.settings(locals())
+
+## This is used during development phase when testing production
+# with heroku, look at
+# https://github.com/kennethreitz/dj-database-url/issues/107
+# Uncomment when necessary
+# del DATABASES['default']['OPTIONS']['sslmode']
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'verbose': {
+            'format': ('%(asctime)s [%(process)d] [%(levelname)s] ' +
+                       'pathname=%(pathname)s lineno=%(lineno)s ' +
+                       'funcname=%(funcName)s %(message)s'),
+            'datefmt': '%Y-%m-%d %H:%M:%S'
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        }
+    },
+    'handlers': {
+        'null': {
+            'level': 'DEBUG',
+            'class': 'logging.NullHandler',
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose'
+        }
+    },
+    'loggers': {
+        'testlogger': {
+            'handlers': ['console'],
+            'level': 'INFO',
+        }
+    }
+}
+
+DEBUG_PROPAGATE_EXCEPTIONS = True

canairio/settings/testing.py

@@ -0,0 +1,11 @@
+from .base import *
+
+ENVIRONMENT = 'test'
+
+# Please make sure that you configure in the testing environment the
+# name without the _test suffix
+DATABASES['default']['NAME'] = '{}_test'.format(getenvvar('DATABASE_NAME'))
+
+TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner'
+TEST_OUTPUT_VERBOSE = 2
+TEST_OUTPUT_DIR = 'test-results'

canairio/wsgi.py

@@ -11,6 +11,6 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'canairio.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'canairio.settings.production')
 
 application = get_wsgi_application()

create-dbs.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+POSTGRES="psql --username ${POSTGRES_USER}"
+
+echo "Creating database: ${POSTGRES_DB}"
+echo "Creating database: ${POSTGRES_TEST_DB}"
+
+$POSTGRES <<EOSQL
+CREATE DATABASE "${POSTGRES_DB}" OWNER "${POSTGRES_USER}";
+CREATE DATABASE "${POSTGRES_TEST_DB}" OWNER "${POSTGRES_USER}";
+EOSQL

docker-compose.yml

@@ -7,3 +7,39 @@ services:
     ports:
       - "8083:8083"
       - "8086:8086"
+
+  timescale:
+    image: timescale/timescaledb-postgis:latest-pg11
+    ports:
+      - "15432:5432"
+    build:
+      context: .
+      args:
+        - db_name=canairio
+      dockerfile: Dockerfile_timescaledb
+    environment:
+      - POSTGRES_USER=canairio
+      - POSTGRES_PASSWORD=canairio
+      - POSTGRES_DB=canairio
+      - POSTGRES_TEST_DB=canairio_test
+  web:
+    build: .
+    command: ./start
+    volumes:
+      - .:/canairio
+    ports:
+      - "8000:8000"
+    depends_on:
+      - influxdb
+      - timescale
+    links:
+      - influxdb
+      - timescale
+    environment:
+      - DJANGO_SETTINGS_MODULE=canairio.settings.development
+      - DATABASE_URL=postgresql://canairio:canairio@timescale/canairio
+      - DATABASE_NAME=canairio
+      - INFLUXDB_HOST=influxdb
+      - INFLUXDB_DATABASE=canairio
+      - ENVIRONMENT=development
+      - SECRET_KEY=so_secret

requirements.txt

@@ -48,6 +48,7 @@ rsa==4.0
 ruamel.yaml==0.15.96
 simple-http-server==0.1.7
 six==1.12.0
+unittest-xml-reporting==2.5.1
 toml==0.10.0
 uritemplate==3.0.0
 urllib3==1.24.1

runtime.txt

@@ -0,0 +1 @@
+python-3.7.3

start

@@ -1,7 +1,7 @@
 #!/bin/sh
-sleep 2
-influx -host $INFLUXDB_HOST -precision rfc3339 -execute 'CREATE DATABASE canairio'
 pip install -r requirements.txt
+influx -host $INFLUXDB_HOST -precision rfc3339 -execute 'CREATE DATABASE '$INFLUXDB_DATABASE
+python manage.py migrate
 
 while [ 1 ]; do
     date