airflow.all.yaml

apiVersion: v1
kind: Service
metadata:
  name: postgres
spec:
  type: ClusterIP
  selector:
    app: airflow
    tier: db
  ports:
    - name: postgres
      protocol: TCP
      port: 5432
      targetPort: postgres
---
apiVersion: v1
kind: Service
metadata:
  name: rabbitmq
spec:
  type: ClusterIP
  selector:
    app: airflow
    tier: rabbitmq
  ports:
    - name: node
      protocol: TCP
      port: 5672
      targetPort: node
    - name: management
      protocol: TCP
      port: 15672
      targetPort: management
---
apiVersion: v1
kind: Service
metadata:
  name: web
spec:
  type: LoadBalancer
  loadBalancerSourceRanges:
  - 0.0.0.0/0
  selector:
    app: airflow
    tier: web
  ports:
    - name: web
      protocol: TCP
      port: 8080
      targetPort: web
      nodePort: 32080
---
apiVersion: v1
kind: Service
metadata:
  name: flower
spec:
  type: NodePort
  selector:
    app: airflow
    tier: flower
  ports:
    - name: flower
      protocol: TCP
      port: 5555
      targetPort: flower
      nodePort: 32081
---
apiVersion: v1
kind: Service
metadata:
  name: worker-subdomain
spec:
  clusterIP: None
  selector:
    app: airflow
    tier: worker
  ports:
    - name: worker
      port: 8793
      targetPort: 8793
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: postgres
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: airflow
        tier: db
    spec:
      containers:
      - name: postgres
        image: postgres:10
        ports:
        - name: postgres
          containerPort: 5432
        volumeMounts:
          - mountPath: /var/lib/postgresql/data
            name: postgres-data
        env:
         - name: POSTGRES_USER
           value: "airflow"
         - name: POSTGRES_PASSWORD
           value: "airflow"
         - name: POSTGRES_DB
           value: "airflow"
         - name: PGDATA
           value: "/var/lib/postgresql/data/pgdata"
      volumes:
      - name: postgres-data
        gcePersistentDisk:
          pdName: postgres-data
          fsType: ext4
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: rabbitmq
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: airflow
        tier: rabbitmq
    spec:
      restartPolicy: Always
      containers:
      - name: rabbitmq
        image: rabbitmq:3-management
        ports:
        - name: management
          containerPort: 15672
        - name: node
          containerPort: 5672
        env:
        - name: RABBITMQ_DEFAULT_USER
          value: airflow
        - name: RABBITMQ_DEFAULT_PASS
          value: airflow
        - name: RABBITMQ_DEFAULT_VHOST
          value: airflow
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: web
spec:
  replicas: 1
#    replicas should be 2 for maxUnavailable=0
#  minReadySeconds: 120
#  strategy:
#    # indicate which strategy we want for rolling update
#    type: RollingUpdate
#    rollingUpdate:
#      maxSurge: 1
#      maxUnavailable: 0
  template:
    metadata:
      labels:
        app: airflow
        tier: web
    spec:
      restartPolicy: Always
      containers:
      - name: web
        image: %%REMOTE_IMAGE_PATH%%
        imagePullPolicy: Always
        env:
        - name: AIRFLOW_HOME
          value: "/usr/local/airflow"
        - name: GOOGLE_APPLICATION_CREDENTIALS
          value: "/usr/local/airflow/gcp-airflow.json"
        ports:
        - name: web
          containerPort: 8080
        args: ["webserver"]
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: flower
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: airflow
        tier: flower
    spec:
      restartPolicy: Always
      containers:
      - name: flower
        image: %%REMOTE_IMAGE_PATH%%
        imagePullPolicy: Always
        env:
        - name: AIRFLOW_HOME
          value: "/usr/local/airflow"
        - name: GOOGLE_APPLICATION_CREDENTIALS
          value: "/usr/local/airflow/gcp-airflow.json"
        # To prevent the error: ValueError: invalid literal for int() with base 10: 'tcp://10.0.0.83:5555'
        - name: FLOWER_PORT
          value: "5555"
        ports:
        - name: flower
          containerPort: 5555
        args: ["flower"]
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: scheduler
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: airflow
        tier: scheduler
    spec:
      restartPolicy: Always
      containers:
      - name: scheduler
        image: %%REMOTE_IMAGE_PATH%%
        imagePullPolicy: Always
        # volumes:
        #     - /localpath/to/dags:/usr/local/airflow/dags
        env:
        - name: AIRFLOW_HOME
          value: "/usr/local/airflow"
        - name: GOOGLE_APPLICATION_CREDENTIALS
          value: "/usr/local/airflow/gcp-airflow.json"
        args: ["scheduler"]
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: worker
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: airflow
        tier: worker
    spec:
      restartPolicy: Always
      hostname: worker
      subdomain: worker-subdomain
      containers:
      - name: worker
        image: %%REMOTE_IMAGE_PATH%%
        imagePullPolicy: Always
        # volumes:
        #     - /localpath/to/dags:/usr/local/airflow/dags
        env:
        - name: AIRFLOW_HOME
          value: "/usr/local/airflow"
        - name: GOOGLE_APPLICATION_CREDENTIALS
          value: "/usr/local/airflow/gcp-airflow.json"
        args: ["worker"]