Parallelise our CI/CD deployment of hubs

.github/actions/setup-deploy/action.yaml

@@ -0,0 +1,58 @@
+name: "Deploy prerequisites"
+description: "Prerequisites needed to deploy hubs in a 2i2c cluster"
+inputs:
+  provider:
+    description: "Cloud provider a cluster runs on"
+    required: true
+    default: "gcp"
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repo
+      uses: actions/checkout@v3
+
+    - name: Setup gcloud
+      uses: google-github-actions/setup-gcloud@v0
+      with:
+        version: "290.0.1"
+        # This is used for KMS only
+        project_id: two-eye-two-see
+        service_account_key: ${{ secrets.GCP_KMS_DECRYPTOR_KEY }}
+        export_default_credentials: true
+
+    - name: Get gcloud info
+      run: gcloud info
+      shell: bash
+
+    - name: Setup gcloud auth for docker
+      # FIXME: Add more auth providers & registries here as needed
+      run: |
+        gcloud auth configure-docker us-central1-docker.pkg.dev
+      shell: bash
+
+    - name: Setup helm
+      uses: azure/[email protected]
+
+    - name: Setup sops
+      uses: mdgreenwald/mozilla-sops-action@v1
+      with:
+        version: v3.7.1
+
+    - name: Setup kops
+      if: inputs.provider == 'aws'
+      run: |
+        curl -Lo /tmp/kops https://github.com/kubernetes/kops/releases/download/$KOPS_VERSION/kops-linux-amd64
+        chmod +x /tmp/kops
+        sudo mv /tmp/kops /usr/local/bin/kops
+      env:
+        KOPS_VERSION: "v1.21.1"
+
+    - name: Setup Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
+
+    - name: Setup dependencies
+      run: |
+        python -m pip install -r requirements.txt
+      shell: bash

.github/workflows/deploy-hubs.yaml

@@ -6,121 +6,129 @@ on:
       - master
     paths:
       - deployer/**
-      - helm-charts/**
       - requirements.txt
-      - dev-requirements.txt
-      - config/secrets.yaml
-      - config/clusters/**
       - .github/workflows/deploy-hubs.yaml
-      - .github/actions/deploy/**
+      - .github/actions/setup-deploy/**
+      - "**.yaml"
+    tags:
+      - "**"
+  pull_request:
+    branches:
+      - master
+    paths:
+      - deployer/**
+      - requirements.txt
+      - .github/workflows/deploy-hubs.yaml
+      - .github/actions/setup-deploy/**
+      - "**.yaml"
 
 # When multiple PRs triggering this workflow are merged, queue them instead
 # of running them in parallel
 # https://github.blog/changelog/2021-04-19-github-actions-limit-workflow-run-or-job-concurrency/
 concurrency: deploy
 
 jobs:
-  deploy:
+  generate-jobs:
     runs-on: ubuntu-latest
-    strategy:
-      # Don't stop other deployments if one fails
-      fail-fast: false
-      matrix:
-        include:
-          # To enable auto-deployments for other clusters,
-          # add its name and provider to the list
-          - cluster_name: 2i2c
-            provider: gcp
-          - cluster_name: cloudbank
-            provider: gcp
-          - cluster_name: carbonplan
-            provider: aws
-          - cluster_name: farallon
-            provider: aws
-          - cluster_name: openscapes
-            provider: aws
-          - cluster_name: uwhackweeks
-            provider: aws
-          - cluster_name: meom-ige
-            provider: gcp
-          - cluster_name: pangeo-hubs
-            provider: gcp
-          - cluster_name: leap
-            provider: gcp
-          - cluster_name: utoronto
-            provider: kubeconfig
-          - cluster_name: azure.carbonplan
-            provider: kubeconfig
+    outputs:
+      support-and-staging-matrix-jobs: ${{ steps.generated-jobs.outputs.support-and-staging-matrix-jobs }}
+      prob-hub-matrix-jobs: ${{ steps.generated-jobs.outputs.prod-hub-matrix-jobs }}
 
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
-
-      - name: Check if any of our base files have changed
-        uses: dorny/paths-filter@v2
-        id: base_files
         with:
-          filters: |
-            files:
-              - "deployer/**"
-              - "helm-charts/**"
-              - "requirements.txt"
-              - "config/secrets.yaml"
-              - ".github/workflows/deploy-hubs.yaml"
-              - ".github/actions/deploy/*"
-
-      - name: Check which cluster directory has changes (if any)
-        uses: dorny/paths-filter@v2
-        id: config_files
+          fetch-depth: 0
+
+      - name: Install Python 3.9
+        uses: actions/setup-python@v3
         with:
-          filters: |
-            hub_config:
-              - "config/clusters/${{ matrix.cluster_name }}/**"
-
-      - name: Setup gcloud
-        if: |
-          (steps.base_files.outputs.files == 'true') ||
-          (steps.config_files.outputs.hub_config == 'true')
-        uses: google-github-actions/setup-gcloud@v0
+          python-version: "3.9"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+
+      - name: Identify files that have been added or modified
+        uses: jitterbit/get-changed-files@v1
+        id: changed-files
         with:
-          version: "290.0.1"
-          # This is used for KMS only
-          project_id: two-eye-two-see
-          service_account_key: ${{ secrets.GCP_KMS_DECRYPTOR_KEY }}
-          export_default_credentials: true
-
-      - name: Setup helm
-        if: |
-          (steps.base_files.outputs.files == 'true') ||
-          (steps.config_files.outputs.hub_config == 'true')
-        uses: azure/[email protected]
-
-      - name: Setup sops
-        if: |
-          (steps.base_files.outputs.files == 'true') ||
-          (steps.config_files.outputs.hub_config == 'true')
-        uses: mdgreenwald/mozilla-sops-action@v1
+          format: space-delimited
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate matrix jobs
+        if: github.event_name != 'pull_request'
+        run: |
+          python deployer generate-helm-upgrade-jobs "${{ steps.changed-files.outputs.added_modified }}"
+
+      # TODO: Post the output of this step as a comment on the Pull Request. Do this in
+      # a Netlify-style way where if a comment already exists, it is updated instead
+      # of generating a new comment.
+      - name: Pretty print matrix jobs
+        if: github.event_name != 'pull_request'
+        run: |
+          python deployer generate-helm-upgrade-jobs --pretty-print "${{ steps.changed-files.outputs.added_modified }}"
+
+  upgrade-support-and-staging:
+    runs-on: ubuntu-latest
+    needs: generate-jobs
+    # Only run this job on pushes to the default branch and when the job output is not
+    # an empty list
+    if: |
+      (github.event_name == 'push' && contains(github.ref, 'master')) &&
+      ${{ fromJson(needs.generate-jobs.outputs.support-and-staging-matrix-jobs) }} != '[]'
+    strategy:
+      # Don't stop other deployments if one fails
+      fail-fast: false
+      matrix:
+        jobs: ${{ fromJson(needs.generate-jobs.outputs.support-and-staging-matrix-jobs) }}
+
+    steps:
+      - name: Setup deploy for ${{ matrix.jobs.cluster_name }}
+        uses: ./.github/actions/setup-deploy
         with:
-          version: v3.7.1
+          provider: ${{ matrix.provider }}
 
-      - name: Setup kops
-        if: |
-          ((steps.base_files.outputs.files == 'true') ||
-          (steps.config_files.outputs.hub_config == 'true')) &&
-          (matrix.provider == 'aws')
+      - name: Upgrade support chart on cluster ${{ matrix.jobs.cluster_name }}
+        if: matrix.jobs.upgrade_support == 'true'
         run: |
-          curl -Lo /tmp/kops https://github.com/kubernetes/kops/releases/download/$KOPS_VERSION/kops-linux-amd64
-          chmod +x /tmp/kops
-          sudo mv /tmp/kops /usr/local/bin/kops
-        env:
-          KOPS_VERSION: "v1.21.1"
-
-      - name: Deploy ${{ matrix.cluster_name }}
-        if: |
-          (steps.base_files.outputs.files == 'true') ||
-          (steps.config_files.outputs.hub_config == 'true')
-        uses: ./.github/actions/deploy
+          python deployer deploy-support ${{ matrix.jobs.cluster_name }}
+
+      - name: Upgrade staging hub on cluster ${{ matrix.jobs.cluster_name }}
+        if: matrix.jobs.upgrade_staging == 'true'
+        run: |
+          python deployer deploy ${{ matrix.jobs.cluster_name }} staging
+
+      - name: Upgrade dask-staging hub on cluster ${{ matrix.jobs.cluster_name }}
+        if: matrix.jobs.upgrade_staging == 'true' && matrix.jobs.cluster_name == '2i2c'
+        run: |
+          python deployer deploy ${{ matrix.jobs.cluster_name }} dask-staging
+
+  # TODO: Add in "job 3a" where the matrix for prod jobs is edited to exclude any clusters
+  # that failed the preceding upgrade-support-and-staging job. See discussion in:
+  # https://github.com/2i2c-org/infrastructure/issues/1131
+
+  upgrade-prod-hubs:
+    runs-on: ubuntu-latest
+    needs: [generate-jobs, upgrade-support-and-staging]
+    # Only run this job on pushes to the default branch and when the job output is not
+    # an empty list
+    if: |
+      (github.event_name == 'push' && contains(github.ref, 'master')) &&
+      ${{ fromJson(needs.generate-jobs.outputs.prob-hub-matrix-jobs) }} != '[]'
+    strategy:
+      # Don't stop other deployments if one fails
+      fail-fast: false
+      matrix:
+        jobs: ${{ fromJson(needs.generate-jobs.outputs.prob-hub-matrix-jobs) }}
+
+    steps:
+      - name: Setup deploy for ${{ matrix.jobs.cluster_name }}
+        uses: ./.github/actions/setup-deploy
         with:
-          cluster: ${{ matrix.cluster_name }}
-        env:
-          TERM: xterm
+          provider: ${{ matrix.provider }}
+
+      - name: Upgrade and test ${{ matrix.jobs.hub_name }} on cluster ${{ matrix.jobs.cluster_name }}
+        run: |
+          python deployer deploy ${{ matrix.jobs.cluster_name }} ${{ matrix.jobs.hub_name }}

deployer/helm_upgrade_decision.py

@@ -47,7 +47,7 @@ def discover_modified_common_files(modified_paths):
         "requirements.txt",
         # Filepaths related to GitHub Actions infrastructure
         ".github/workflows/deploy-hubs.yaml",
-        ".github/actions/deploy/*",
+        ".github/actions/setup-deploy/*",
         # Filepaths related to helm chart infrastructure
         "helm-charts/basehub/*",
         "helm-charts/daskhub/*",