docker-compose.yml

version: "3.7"
services:
  # Internal database for Airflow.
  database:
    image: amsterdam/postgres11
    ports:
      - "5417:5432"
    environment:
      POSTGRES_DB: ds_airflow
      POSTGRES_USER: ds_airflow
      POSTGRES_PASSWORD: insecure

  # Database for actual datasets, shared with DSO-API.
  dso_database:
    image: amsterdam/postgres11
    ports:
      - "5416:5432"
    environment:
      POSTGRES_DB: dataservices
      POSTGRES_USER: dataservices
      POSTGRES_PASSWORD: insecure
    extra_hosts:
      admin.data.amsterdam.nl: 10.243.16.4

  airflow:
    build: src
    container_name: airflow
    depends_on:
      - database
      - dso_database
    environment:
      AIRFLOW_CONN_POSTGRES_DEFAULT:
        "postgresql://dataservices:insecure@dso_database:5432\
        /dataservices?cursor=dictcursor&"
      AIRFLOW_CONN_POSTGRES_AZURE_BENK:
        "postgresql://cvps_airflow_benk_dev:insecure@dso_database:5432\
        /mdbdataservices?cursor=dictcursor&sslmode=require"
      AIRFLOW_CONN_POSTGRES_AZURE_BOR:
        "postgresql://cvps_airflow_bor_dev:insecure@dso_database:5432\
        /mdbdataservices?cursor=dictcursor&sslmode=require"
      AIRFLOW_CONN_POSTGRES_AZURE_DGEN: "postgresql://cvps_airflow_dgen_dev:insecure@dso_database:5432\
        /mdbdataservices?cursor=dictcursor&sslmode=require"
      AIRFLOW_CONN_POSTGRES_AZURE_DSOC: "postgresql://cvps_airflow_dsoc_dev:insecure@dso_database:5432\
        /mdbdataservices?cursor=dictcursor&sslmode=require"
      AIRFLOW_CONN_POSTGRES_AZURE_MOBI: "postgresql://cvps_airflow_mobi_dev:insecure@dso_database:5432\
        /mdbdataservices?cursor=dictcursor&sslmode=require"
      AIRFLOW_CONN_POSTGRES_AZURE_OPDR: "postgresql://cvps_airflow_opdr_dev:insecure@dso_database:5432\
        /mdbdataservices?cursor=dictcursor&sslmode=require"
      AIRFLOW_CONN_POSTGRES_AZURE_SOEB: "postgresql://cvps_airflow_soeb_dev:insecure@dso_database:5432\
        /mdbdataservices?cursor=dictcursor&sslmode=require"
      AIRFLOW_CONN_POSTGRES_AZURE_STAT: "postgresql://cvps_airflow_stat_dev:insecure@dso_database:5432\
        /mdbdataservices?cursor=dictcursor&sslmode=require"
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: "postgresql://ds_airflow:insecure@database:5432/ds_airflow"
      AIRFLOW_CONN_ORACLE_DWH_STADSDELEN: "oracle://oracle_user:insecure@localhost:1521/SID001?encoding=UTF-8&nencoding=UTF-8"
      AIRFLOW_CONN_ORACLE_DWH_AMI: "oracle://oracle_user:insecure@localhost:port/database?encoding=UTF-8&nencoding=UTF-8"
      AIRFLOW_CONN_OBJECTSTORE_DATACATALOGUS: "s3://catalogus:${datacatalogus_objectstore_key_safe}@e85bcf2124fb4437b1bc6eb75dfc3abf"
      AIRFLOW_CONN_OBJECTSTORE_DATASERVICES:
        "s3://dataservices:${DATASERVICES_PASSWD}@\
        356c76835e424b968ed6d654c51204f0"
      AIRFLOW_CONN_OBJECTSTORE_PARKEERVAKKEN:
        "s3://parkeervakken:${PARKEERVAKKEN_PASSWD}@\
        091e3bedc85447ef936e82bcda88fcac"
      AIRFLOW_CONN_OBJECTSTORE_PROCESSENVERBAALVERKIEZINGEN:
        "s3://dataportaal_publicaties_verkiezingen:${PROCESSENVERBAALVERKIEZINGEN_PASSWD}@\
        0855010431b44f9caec7803bce29def8"
      AIRFLOW_CONN_POSTGRES_DBWAARNEMINGEN:
        "postgresql://mensen:${mensen_password}\
        @dbwaarnemingen-postgres-read.service.consul:5432/mensen"
      AIRFLOW_CONN_POSTGRES_BASISKAART:
        "postgresql://${db_basiskaart_user}:${db_basiskaart_password}@\
        ${db_basiskaart_host}:${db_basiskaart_port}/${db_basiskaart_name}"
      AIRFLOW_CONN_OBJECTSTORE_MILIEUTHEMAS:
        "s3://milieuthemas:${MILIEUTHEMAS_PASSWD}@\
        e063b706cffc4002883c28d531f0234f"
      AIRFLOW_CONN_OBJECTSTORE_VICTOR:
        "s3://Victor:${VICTOR_PASSWD}@\
        9c5404482acd43e488de31927f558201"
      AIRFLOW_CONN_OBJECTSTORE_DATARUIMTE:
        "s3://${DATARUIMTE_USER}:${DATARUIMTE_PASSWD}@\
        e2e0feab4cd54f7cb0c8517586ac7bb1"
      AIRFLOW_CONN_SWIFT_DEFAULT: "s3://vsd_user:${VSD_PASSWD}@\
        4028c44d91dc48b8990069433c203c1f"
      AIRFLOW_CONN_OBJECTSTORE_GOB: "s3://GOB_user:${GOB_PASSWD}@\
        2ede4a78773e453db73f52500ef748e5"
      AIRFLOW_CONN_OBJECTSTORE_OPENBAREVERLICHTING:
          "s3://${OPENBAREVERLICHTING_USER}:${OPENBAREVERLICHTING_PASSWD}@\
          5f0dae75f100401b9f49338747792dce"
      AIRFLOW_CONN_GOB_GRAPHQL: "https://acc.api.data.amsterdam.nl/https"
      AIRFLOW_CONN_OIDC_SERVER: "https://iam.amsterdam.nl/https"
      AIRFLOW_CONN_BOUWSTROOMPUNTEN_PASSWD: ${dataservices_bouwstroompunten_passwd}
      AIRFLOW_CONN_BOUWSTROOMPUNTEN_USER: ${dataservices_bouwstroompunten_user}
      AIRFLOW_CONN_BOUWSTROOMPUNTEN_BASE_URL: "https://maps.nrgaccounting.nl/https"
      AIRFLOW_CONN_RIVM_BASE_URL: "https://data.rivm.nl/https"
      AIRFLOW_CONN_WIOR_PASSWD: ${dataservices_wior_passwd}
      AIRFLOW_CONN_WIOR_USER: ${dataservices_wior_user}
      AIRFLOW_CONN_WIOR_BASE_URL: "https://data-wior-amsterdam.webgis.nl"
      AIRFLOW_CONN_FELYX_BASE_URL: "https://data.felyx.com"
      AIRFLOW_CONN_FELYX_API_KEY: ${dataservices_felyx_api_key}
      AIRFLOW_CONN_RIDECHECK_BASE_URL: "https://api.ridecheck.app"
      AIRFLOW_CONN_RIDECHECK_TOKEN_URL: "https://accounts.ridecheck.app/oauth2/token"
      AIRFLOW_CONN_RIDECHECK_CLIENT_ID: ${dataservices_ridecheck_client_id}
      AIRFLOW_CONN_RIDECHECK_CLIENT_SECRET: ${dataservices_ridecheck_client_secret}
      AIRFLOW_CONN_MYWHEELS_BASE_URL: "https://mywheels.nl"
      AIRFLOW_CONN_MYWHEELS_API_KEY: ${dataservices_mywheels_api_key}
      AIRFLOW_CONN_RDW_BASE_URL: "https://opendata.rdw.nl/resource"
      AIRFLOW_CONN_STROOMSTORING_BASE_URL: "https://services1.arcgis.com/https"
      AIRFLOW_CONN_OOV_BRIEVENBUS_GEBIEDEN: "sftp://${oov_brievenbus_user_overlastgebieden}:\
        ${oov_brievenbus_password_overlastgebieden}@${oov_brievenbus_host}:${oov_brievenbus_port}"
      AIRFLOW_CONN_OOV_BRIEVENBUS_CORONA: "sftp://${oov_brievenbus_user_corona}:\
        ${oov_brievenbus_password_corona}@${oov_brievenbus_host}:${oov_brievenbus_port}"
      AIRFLOW__API__AUTH_BACKENDS: ${AIRFLOW__API__AUTH_BACKENDS:-airflow.api.auth.backend.basic_auth}
      GOB_PUBLIC_ENDPOINT: "gob/public/graphql/streaming/"
      GOB_SECURE_ENDPOINT: "gob/graphql/streaming/"
      OAUTH_TOKEN_EXPIRES_MARGIN: 20
      FERNET_KEY: ${FERNET_KEY:-"YlCImzjge_TeZc7jPJ7Jz2pgOtb4yTssA1pVyqIADWg="}
      SECRET_KEY: ${SECRET_KEY:-"105398791e3fbc571de0f053f1e577ad92e257d321242405d31294594312"}
      AIRFLOW_LOGGING_LEVEL: ${AIRFLOW_LOGGING_LEVEL:-INFO}
      AIRFLOW__WEBSERVER__BASE_URL: http://localhost
      AIRFLOW__WEBSERVER__BASE_URL_PORT: 8080
      DATAPUNT_ENVIRONMENT: development
      SLACK_WEBHOOK_HOST: https://hooks.slack.com/services
      SLACK_WEBHOOK: ${SLACK_WEBHOOK}
      SLACK_CHANNEL: "#airflow"
      OS_USERNAME: vsd_user
      OS_PASSWORD: ${VSD_PASSWD}
      OS_TENANT_NAME: 4028c44d91dc48b8990069433c203c1f
      OS_AUTH_URL: https://identity.stack.cloudvps.com/v2.0
      SHARED_DIR: /tmp
      OPENBARE_VERLICHTING_DATA_SRC: https://asd2.techtek.eu/asd/services/rest_sia.php/SIAService/objecten
      OPENBARE_VERLICHTING_DATA_TYPES_SRC: https://asd2.techtek.eu/asd/services/rest_sia.php/SIAService/types
      SCHEMA_URL: ${SCHEMA_URL}
      AIRFLOW_USER_ADMIN_PASSWD: ${dataservices_airflow_user_admin_passwd}
      AIRFLOW_USER_DATASERVICES_PASSWD: ${dataservices_airflow_user_dataservices_passwd}
      AIRFLOW_USER_TEAM_RUIMTE_PASSWD: ${dataservices_airflow_user_team_ruimte_passwd}
      AIRFLOW_USER_TEAM_BENK_PASSWD: ${dataservices_airflow_user_team_benk_passwd}
      AIRFLOW_USER_TEAM_MOSS_PASSWD: ${dataservices_airflow_user_team_moss_passwd}
      OIDC_TOKEN_ENDPOINT: ${OIDC_TOKEN_ENDPOINT}
      OIDC_CLIENT_ID: ${DATASERVICES_GOB_EXPORT_CLIENT_ID}
      OIDC_CLIENT_SECRET: ${DATASERVICES_GOB_EXPORT_CLIENT_SECRET}
      AIRFLOW_LOG_CLEANUP__MAX_LOG_AGE_IN_DAYS: 90
      AIRFLOW_LOG_CLEANUP__ENABLE_DELETE_CHILD_LOG: "true"
      SENDGRID_MAIL_FROM: ${SENDGRID_MAIL_FROM}
      SENDGRID_API_KEY: ${SENDGRID_API_KEY}
      NO_AUTOSTART_AIRFLOW: ${NO_AUTOSTART_AIRFLOW-}
    volumes:
      - ./src/dags:/usr/local/airflow/dags
      - ./src/plugins:/usr/local/airflow/plugins
      - ./src/vars:/usr/local/airflow/vars
      - ./src/vsd:/usr/local/airflow/vsd
      - ./develop:/usr/local/airflow/develop
      - /var/run/docker.sock:/var/run/docker.sock
    # extra_hosts:
    #- "dockerhost:$DOCKERHOST"

    ports:
      - "8080:8080"

    # Required for ptrace-based debuggers like C++, Go, and Rust
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp:unconfined

    # Overrides default command so things don't shut down after the process ends.
    # command: /bin/sh -c "while sleep 1000; do :; done"