Merge pull request #1045 from consideRatio/pr/add-helm-chart-schemas-…

…and-validate-etc

.github/actions/deploy/action.yml

@@ -19,7 +19,6 @@ runs:
     - name: Setup dependencies
       run: |
         python3 -m pip install -r requirements.txt
-        python3 -m pip install -r dev-requirements.txt
       shell: bash
     - name: Deploy support components
       run: |

.github/workflows/deploy-grafana-dashboards.yaml

@@ -22,7 +22,6 @@ jobs:
       - name: Setup dependencies
         run: |
           python3 -m pip install -r requirements.txt
-          python3 -m pip install -r dev-requirements.txt
           sudo apt install jsonnet
 
       - name: Setup gcloud

.github/workflows/deploy-hubs.yaml

@@ -63,7 +63,6 @@ jobs:
               - "deployer/**"
               - "helm-charts/**"
               - "requirements.txt"
-              - "dev-requirements.txt"
               - "config/secrets.yaml"
               - ".github/workflows/deploy-hubs.yaml"
               - ".github/actions/deploy/*"

.github/workflows/doc-links.yml

@@ -13,11 +13,6 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
-        with:
-          # chartpress is used by doc/conf.py,
-          # and requires information about the latest tagged commit, which
-          # requires the git history.
-          fetch-depth: 0
 
       - name: Install environment
         uses: conda-incubator/setup-miniconda@v2

.github/workflows/validate-clusters.yaml

@@ -0,0 +1,96 @@
+# This is a GitHub workflow defining a set of jobs with a set of steps. ref:
+# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
+#
+# Runs the deployer script to validate clusters. This will both validate
+# cluster.yaml files as well as each hubs passed non-encrypted values files
+# against the Helm charts' values schema.
+#
+name: Validate clusters
+
+on:
+  pull_request:
+    paths:
+      - config/clusters/**
+      - deployer/**
+      - helm-charts/basehub/**
+      - helm-charts/daskhub/**
+      - requirements.txt
+      - .github/workflows/validate-hubs.yaml
+  push:
+    paths:
+      - config/clusters/**
+      - deployer/**
+      - helm-charts/basehub/**
+      - helm-charts/daskhub/**
+      - requirements.txt
+      - .github/workflows/validate-hubs.yaml
+    branches-ignore:
+      - "dependabot/**"
+      - "pre-commit-ci-update-config"
+    tags:
+      - "**"
+  workflow_dispatch:
+
+jobs:
+  validate-hubs-values-files:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - cluster_name: 2i2c
+          - cluster_name: azure.carbonplan
+          - cluster_name: carbonplan
+          - cluster_name: cloudbank
+          - cluster_name: farallon
+          - cluster_name: meom-ige
+          - cluster_name: openscapes
+          - cluster_name: pangeo-hubs
+          - cluster_name: utoronto
+          - cluster_name: uwhackweeks
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Check if any cluster common files has changed
+        uses: dorny/paths-filter@v2
+        id: cluster_common_files
+        with:
+          filters: |
+            files:
+              - deployer/**
+              - helm-charts/basehub/**
+              - helm-charts/daskhub/**
+              - requirements.txt
+              - .github/workflows/validate-hubs.yaml
+
+      - name: Check if cluster specific files has changes
+        uses: dorny/paths-filter@v2
+        id: cluster_specific_files
+        with:
+          filters: |
+            changes:
+              - config/clusters/${{ matrix.cluster_name }}/**
+
+      # To continue this cluster specific job we must either have manually
+      # invoked this workflow to run for all clusters, or there should have been
+      # changes to the cluster common files or cluster specific files.
+      - name: Decide if the job should continue
+        id: decision
+        run: |
+          echo ::set-output name=continue-job::${{ github.event_name == 'workflow_dispatch' || (steps.cluster_common_files.outputs.files == 'true' || steps.cluster_specific_files.outputs.changes == 'true') }}
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+
+      - name: Install deployer script dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: "Validate cluster: ${{ matrix.cluster_name }}"
+        if: steps.decision.outputs.continue-job == 'true'
+        env:
+          TERM: xterm
+        run: |
+          python deployer validate ${{ matrix.cluster_name }}

.gitignore

@@ -15,6 +15,10 @@
 # have secret in their name.
 !**/templates/**/*secret*.yaml
 
+# Ignore the .json version of our Helm chart's schema files so we only version
+# control the .yaml file that we use to generate the .json file.
+values.schema.json
+
 # Ignore helm chart generated outputs
 Chart.lock
 **/charts/*.tgz

config/clusters/farallon/staging.values.yaml

@@ -35,7 +35,7 @@ basehub:
             url: https://2i2c.org
           funded_by:
             name: Farallon Institute
-            urL: http://www.faralloninstitute.org/
+            url: http://www.faralloninstitute.org/
     singleuser:
       initContainers:
         # Need to explicitly fix ownership here, since EFS doesn't do anonuid

config/clusters/openscapes/staging.values.yaml

@@ -1,5 +1,3 @@
-scratchBucket:
-  enabled: false
 basehub:
   nfs:
     pv:

config/clusters/utoronto/staging.values.yaml

@@ -47,6 +47,10 @@ jupyterhub:
     extraFiles:
       github-app-private-key.pem:
         mountPath: /etc/github/github-app-private-key.pem
+        # stringData field will be set via encrypted values files but added here
+        # to meet the chart schema validation requirements without the need to
+        # use secret values during the validation.
+        stringData: "dummy"
       gitconfig:
         mountPath: /etc/gitconfig
         # app-id comes from https://github.com/organizations/utoronto-2i2c/settings/apps/utoronto-jupyterhub-private-cloner

config/clusters/uwhackweeks/staging.values.yaml

@@ -1,5 +1,3 @@
-scratchBucket:
-  enabled: false
 basehub:
   nfs:
     pv:
@@ -31,7 +29,7 @@ basehub:
             name: 2i2c
             url: https://2i2c.org
           funded_by:
-            name:
+            name: ICESat Hackweek
             url: https://icesat-2.hackweek.io
     singleuser:
       serviceAccountName: cloud-user-sa

deployer/__main__.py

@@ -3,23 +3,26 @@
 """
 import argparse
 import os
+import shutil
 import subprocess
+import sys
 from pathlib import Path
 
 import jsonschema
 from ruamel.yaml import YAML
-import shutil
 
 from auth import KeyProvider
 from hub import Cluster
 from utils import (
     get_decrypted_file,
+    prepare_helm_charts_dependencies_and_schemas,
     print_colour,
     find_absolute_path_to_cluster_file,
 )
 
 # Without `pure=True`, I get an exception about str / byte issues
 yaml = YAML(typ="safe", pure=True)
+helm_charts_dir = Path(__file__).parent.parent.joinpath("helm-charts")
 
 
 def use_cluster_credentials(cluster_name):
@@ -31,8 +34,7 @@ def use_cluster_credentials(cluster_name):
     This function is to be used with the `use-cluster-credentials` CLI
     command only - it is not used by the rest of the deployer codebase.
     """
-    # Validate our config with JSON Schema first before continuing
-    validate(cluster_name)
+    validate_cluster_config(cluster_name)
 
     config_file_path = find_absolute_path_to_cluster_file(cluster_name)
     with open(config_file_path) as f:
@@ -55,8 +57,7 @@ def deploy_support(cluster_name):
     """
     Deploy support components to a cluster
     """
-    # Validate our config with JSON Schema first before continuing
-    validate(cluster_name)
+    validate_cluster_config(cluster_name)
 
     config_file_path = find_absolute_path_to_cluster_file(cluster_name)
     with open(config_file_path) as f:
@@ -75,8 +76,7 @@ def deploy_grafana_dashboards(cluster_name):
     Grafana dashboards and deployment mechanism in question are maintained in
     this repo: https://github.com/jupyterhub/grafana-dashboards
     """
-    # Validate our config with JSON Schema first before continuing
-    validate(cluster_name)
+    validate_cluster_config(cluster_name)
 
     config_file_path = find_absolute_path_to_cluster_file(cluster_name)
     with open(config_file_path) as f:
@@ -165,9 +165,8 @@ def deploy(cluster_name, hub_name, skip_hub_health_test, config_path):
     """
     Deploy one or more hubs in a given cluster
     """
-
-    # Validate our config with JSON Schema first before continuing
-    validate(cluster_name)
+    validate_cluster_config(cluster_name)
+    validate_hub_config(cluster_name, hub_name)
 
     with get_decrypted_file(config_path) as decrypted_file_path:
         with open(decrypted_file_path) as f:
@@ -209,19 +208,64 @@ def deploy(cluster_name, hub_name, skip_hub_health_test, config_path):
                 hub.deploy(k, SECRET_KEY, skip_hub_health_test)
 
 
-def validate(cluster_name):
-    schema_file = Path(os.getcwd()).joinpath(
+def validate_cluster_config(cluster_name):
+    """
+    Validates cluster.yaml configuration against a JSONSchema.
+    """
+    cluster_schema_file = Path(os.getcwd()).joinpath(
         "shared", "deployer", "cluster.schema.yaml"
     )
-    config_file = find_absolute_path_to_cluster_file(cluster_name)
+    cluster_file = find_absolute_path_to_cluster_file(cluster_name)
 
-    with open(config_file) as cf, open(schema_file) as sf:
+    with open(cluster_file) as cf, open(cluster_schema_file) as sf:
         cluster_config = yaml.load(cf)
         schema = yaml.load(sf)
         # Raises useful exception if validation fails
         jsonschema.validate(cluster_config, schema)
 
 
+def validate_hub_config(cluster_name, hub_name):
+    """
+    Validates the provided non-encrypted helm chart values files for each hub of
+    a specific cluster.
+    """
+    prepare_helm_charts_dependencies_and_schemas()
+
+    config_file_path = find_absolute_path_to_cluster_file(cluster_name)
+    with open(config_file_path) as f:
+        cluster = Cluster(yaml.load(f), config_file_path.parent)
+
+    hubs = []
+    if hub_name:
+        hubs = [h for h in cluster.hubs if h.spec["name"] == hub_name]
+    else:
+        hubs = cluster.hubs
+
+    for i, hub in enumerate(hubs):
+        print_colour(
+            f"{i+1} / {len(hubs)}: Validating non-encrypted hub values files for {hub.spec['name']}..."
+        )
+
+        cmd = [
+            "helm",
+            "template",
+            str(helm_charts_dir.joinpath(hub.spec["helm_chart"])),
+        ]
+        for values_file in hub.spec["helm_chart_values_files"]:
+            if "secret" not in os.path.basename(values_file):
+                cmd.append(f"--values={config_file_path.parent.joinpath(values_file)}")
+        # Workaround the current requirement for dask-gateway 0.9.0 to have a
+        # JupyterHub api-token specified, for updates if this workaround can be
+        # removed, see https://github.com/dask/dask-gateway/issues/473.
+        if hub.spec["helm_chart"] == "daskhub":
+            cmd.append("--set=dask-gateway.gateway.auth.jupyterhub.apiToken=dummy")
+        try:
+            subprocess.check_output(cmd, text=True)
+        except subprocess.CalledProcessError as e:
+            print(e.stdout)
+            sys.exit(1)
+
+
 def main():
     argparser = argparse.ArgumentParser(
         description="""A command line tool to perform various functions related
@@ -270,7 +314,12 @@ def main():
     validate_parser = subparsers.add_parser(
         "validate",
         parents=[base_parser],
-        help="Validate the cluster configuration against a JSON schema",
+        help="Validate the cluster.yaml configuration itself, as well as the provided non-encrypted helm chart values files for each hub or the specified hub.",
+    )
+    validate_parser.add_argument(
+        "hub_name",
+        nargs="?",
+        help="The hub, or list of hubs, to validate provided non-encrypted helm chart values for.",
     )
 
     # deploy-support subcommand
@@ -305,7 +354,8 @@ def main():
             args.config_path,
         )
     elif args.action == "validate":
-        validate(args.cluster_name)
+        validate_cluster_config(args.cluster_name)
+        validate_hub_config(args.cluster_name, args.hub_name)
     elif args.action == "deploy-support":
         deploy_support(args.cluster_name)
     elif args.action == "deploy-grafana-dashboards":

deployer/auth.py

@@ -1,8 +1,8 @@
+import re
+
 from auth0.v3.authentication import GetToken
 from auth0.v3.management import Auth0
-
 from yarl import URL
-import re
 
 # What key in the authenticated user's profile to use as hub username
 # This shouldn't be changeable by the user!