docker-compose.yml

---
version: '3'
volumes:
  atlasdata: null
services:

  #<------------ ZOOKEEPER ------------>
  zookeeper:
    
    image: wurstmeister/zookeeper:3.4.6
    ports:
      - "2181:2181"

  #<------------ KAFKA ------------>
  kafka:
    
    image: confluentinc/cp-kafka:5.4.0
    hostname: kafka
    ports:
      - 29092:29092
      - 9092:9092
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:29092,PLAINTEXT_HOST://${HOST_IP}:9092
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
    depends_on:
      - zookeeper

  kafka-schema-registry:
    
    image: confluentinc/cp-schema-registry:5.3.6
    hostname: kafka-schema-registry
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181"
    depends_on:
      - zookeeper
      - kafka

  #<------------ SPARK ------------>
  spark-master:
    build: images/pyspark
    hostname: "spark-master"
    expose: 
      - "7077"
    environment: 
      - SPARK_MODE=master
      - ATLAS_HOST=atlas:21000
      - ATLAS_USER=${ATLAS_USER}
      - ATLAS_PASSWORD=${ATLAS_PASSWORD}
    volumes:
      - ./app/pii_schema_inf:/opt/app/pii_schema_inf
      - ./app/custom_entities:/opt/app/custom_entities
      - ./keys:/opt/app/keys

  spark-slave:
    build: images/pyspark
    hostname: "spark-slave"
    expose: 
      - "7077"
    environment: 
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_WORKER_PORT=7075
      - SPARK_WORKER_WEBUI_PORT=8082
      - ATLAS_HOST=atlas:21000
      - ATLAS_USER=${ATLAS_USER}
      - ATLAS_PASSWORD=${ATLAS_PASSWORD}
    volumes:
      - ./app/pii_schema_inf:/opt/app/pii_schema_inf
      - ./app/custom_entities:/opt/app/custom_entities
      - ./keys:/opt/app/keys
    depends_on:
      - spark-master

  #<------------ ATLAS ---------------->
  atlas:
    
    image: sansarip/apache-atlas
    hostname: "atlas"
    expose:
      - "21000"
    ports:
      - "21000:21000"
    volumes:
      - atlasdata:/opt/apache-atlas-2.0.0


  #<------------ PIPELINE WORKER ---------->
  pipeline_worker:
    build: "images/pipeline_worker"
    hostname: "pipeline_worker"
    tty: true
    environment: 
      - ATLAS_HOST=atlas:21000
      - ATLAS_USER=${ATLAS_USER}
      - ATLAS_PASSWORD=${ATLAS_PASSWORD}
      - SCHEMA_SERVER=http://kafka-schema-registry:8081
      - KAFKA_SERVER=kafka:29092
    volumes:
      - ./app/deid:/opt/app/deid
      - ./app/reid:/opt/app/reid
      - ./app/custom_entities:/opt/app/custom_entities
      - ./keys:/opt/app/keys
    depends_on: 
      - kafka
      - atlas
      - kafka-schema-registry