From b93ebc09521e529c92f7c9b1adf3de6f534a5eee Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 1 Mar 2022 23:23:05 +0100 Subject: [PATCH 1/8] helm chart refactoring: white space chomping consistency Rule of thumbs: 1. Always chomp left 2. Chomp right if no content has been rendered so far in a file or named template. --- helm-charts/basehub/templates/azure-file.yaml | 4 ++-- .../basehub/templates/cloud-resources/gcp/_helpers.tpl | 6 +++--- .../templates/cloud-resources/gcp/service-account.yaml | 2 +- .../templates/cloud-resources/gcp/storage-bucket.yaml | 6 +++--- helm-charts/basehub/templates/nfs-share-creator.yaml | 8 ++++---- helm-charts/basehub/templates/nfs.yaml | 6 +++--- helm-charts/basehub/templates/user-sa.yaml | 6 +++--- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/helm-charts/basehub/templates/azure-file.yaml b/helm-charts/basehub/templates/azure-file.yaml index bee3dc7b8f..7e0d076a05 100644 --- a/helm-charts/basehub/templates/azure-file.yaml +++ b/helm-charts/basehub/templates/azure-file.yaml @@ -1,4 +1,4 @@ -{{ if .Values.azureFile.enabled }} +{{- if .Values.azureFile.enabled -}} apiVersion: v1 kind: PersistentVolume metadata: @@ -26,4 +26,4 @@ spec: resources: requests: storage: 1Mi -{{ end }} +{{- end }} diff --git a/helm-charts/basehub/templates/cloud-resources/gcp/_helpers.tpl b/helm-charts/basehub/templates/cloud-resources/gcp/_helpers.tpl index 3407140dd2..4e317cc625 100644 --- a/helm-charts/basehub/templates/cloud-resources/gcp/_helpers.tpl +++ b/helm-charts/basehub/templates/cloud-resources/gcp/_helpers.tpl @@ -1,9 +1,9 @@ {{- define "cloudResources.gcp.serviceAccountName" -}} -{{.Release.Name}}-user-sa +{{ .Release.Name }}-user-sa {{- end }} {{- define "cloudResources.scratchBucket.name" -}} {{- if eq .Values.jupyterhub.custom.cloudResources.provider "gcp" -}} {{ .Values.jupyterhub.custom.cloudResources.gcp.projectId }}-{{ .Release.Name }}-scratch-bucket -{{- end -}} -{{- end }} \ No newline at end of file +{{- end }} +{{- end }} diff --git a/helm-charts/basehub/templates/cloud-resources/gcp/service-account.yaml b/helm-charts/basehub/templates/cloud-resources/gcp/service-account.yaml index bca7a1ab7e..57c60b42bf 100644 --- a/helm-charts/basehub/templates/cloud-resources/gcp/service-account.yaml +++ b/helm-charts/basehub/templates/cloud-resources/gcp/service-account.yaml @@ -1,4 +1,4 @@ -{{ if .Values.jupyterhub.custom.cloudResources.scratchBucket.enabled}} +{{- if .Values.jupyterhub.custom.cloudResources.scratchBucket.enabled -}} apiVersion: iam.cnrm.cloud.google.com/v1beta1 kind: IAMServiceAccount metadata: diff --git a/helm-charts/basehub/templates/cloud-resources/gcp/storage-bucket.yaml b/helm-charts/basehub/templates/cloud-resources/gcp/storage-bucket.yaml index 4d695815f7..f4760c13d1 100644 --- a/helm-charts/basehub/templates/cloud-resources/gcp/storage-bucket.yaml +++ b/helm-charts/basehub/templates/cloud-resources/gcp/storage-bucket.yaml @@ -1,5 +1,5 @@ -{{ if .Values.jupyterhub.custom.cloudResources.scratchBucket.enabled }} -{{ if eq .Values.jupyterhub.custom.cloudResources.provider "gcp" }} +{{- if .Values.jupyterhub.custom.cloudResources.scratchBucket.enabled -}} +{{- if eq .Values.jupyterhub.custom.cloudResources.provider "gcp" -}} apiVersion: storage.cnrm.cloud.google.com/v1beta1 kind: StorageBucket metadata: @@ -31,4 +31,4 @@ spec: kind: StorageBucket name: {{ include "cloudResources.scratchBucket.name" . }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm-charts/basehub/templates/nfs-share-creator.yaml b/helm-charts/basehub/templates/nfs-share-creator.yaml index 680ffcf91f..65edf729cb 100644 --- a/helm-charts/basehub/templates/nfs-share-creator.yaml +++ b/helm-charts/basehub/templates/nfs-share-creator.yaml @@ -1,5 +1,5 @@ -{{ if .Values.nfs.enabled }} -{{ if .Values.nfs.shareCreator.enabled }} +{{- if .Values.nfs.enabled -}} +{{- if .Values.nfs.shareCreator.enabled -}} apiVersion: batch/v1 kind: Job metadata: @@ -49,5 +49,5 @@ spec: nfs: server: {{ .Values.nfs.pv.serverIP | quote }} path: {{ .Values.nfs.pv.baseShareName | quote }} -{{ end }} -{{ end }} +{{- end }} +{{- end }} diff --git a/helm-charts/basehub/templates/nfs.yaml b/helm-charts/basehub/templates/nfs.yaml index ed7bf04ea2..d14980035f 100644 --- a/helm-charts/basehub/templates/nfs.yaml +++ b/helm-charts/basehub/templates/nfs.yaml @@ -1,4 +1,4 @@ -{{ if .Values.nfs.enabled }} +{{- if .Values.nfs.enabled -}} apiVersion: v1 kind: PersistentVolume metadata: @@ -9,7 +9,7 @@ spec: accessModes: - ReadWriteMany nfs: - server: {{ .Values.nfs.pv.serverIP | quote}} + server: {{ .Values.nfs.pv.serverIP | quote }} path: "{{ .Values.nfs.pv.baseShareName }}{{ .Release.Name }}" mountOptions: {{ .Values.nfs.pv.mountOptions | toJson }} --- @@ -25,4 +25,4 @@ spec: resources: requests: storage: 1Mi -{{ end }} +{{- end }} diff --git a/helm-charts/basehub/templates/user-sa.yaml b/helm-charts/basehub/templates/user-sa.yaml index c6c5701d35..7b3bd83f9b 100644 --- a/helm-charts/basehub/templates/user-sa.yaml +++ b/helm-charts/basehub/templates/user-sa.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: ServiceAccount metadata: annotations: - {{ if .Values.jupyterhub.custom.cloudResources.scratchBucket.enabled}} - {{ if eq .Values.jupyterhub.custom.cloudResources.provider "gcp" }} + {{- if .Values.jupyterhub.custom.cloudResources.scratchBucket.enabled }} + {{- if eq .Values.jupyterhub.custom.cloudResources.provider "gcp" }} iam.gke.io/gcp-service-account: {{ include "cloudResources.gcp.serviceAccountName" .}}@{{ .Values.jupyterhub.custom.cloudResources.gcp.projectId }}.iam.gserviceaccount.com {{- end }} {{- end }} - name: user-sa \ No newline at end of file + name: user-sa From 90af804577c1d4856c6bb670a2f6602c6d0bd3f6 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 1 Mar 2022 23:36:28 +0100 Subject: [PATCH 2/8] helm chart schemas: prepare .gitignore and .helmignore --- .gitignore | 4 ++++ helm-charts/basehub/.helmignore | 9 +++++++++ helm-charts/daskhub/.helmignore | 9 +++++++++ 3 files changed, 22 insertions(+) diff --git a/.gitignore b/.gitignore index 841150f081..a9dd7e3bb6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,10 @@ # have secret in their name. !**/templates/**/*secret*.yaml +# Ignore the .json version of our Helm chart's schema files so we only version +# control the .yaml file that we use to generate the .json file. +values.schema.json + # Ignore helm chart generated outputs Chart.lock **/charts/*.tgz diff --git a/helm-charts/basehub/.helmignore b/helm-charts/basehub/.helmignore index f0c1319444..8bc7cf9216 100644 --- a/helm-charts/basehub/.helmignore +++ b/helm-charts/basehub/.helmignore @@ -1,3 +1,12 @@ +# Non default entries manually added by basehub developers + +# Ignore the .yaml that generates the .json, only the .json is relevant to +# bundle with the Helm chart when it is packaged or "helm dep up" is used to +# copy it over to another location where it is referenced. +values.schema.yaml + +# ----------------------------------------------------------------------------- + # Patterns to ignore when building packages. # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. diff --git a/helm-charts/daskhub/.helmignore b/helm-charts/daskhub/.helmignore index f0c1319444..8bc7cf9216 100644 --- a/helm-charts/daskhub/.helmignore +++ b/helm-charts/daskhub/.helmignore @@ -1,3 +1,12 @@ +# Non default entries manually added by basehub developers + +# Ignore the .yaml that generates the .json, only the .json is relevant to +# bundle with the Helm chart when it is packaged or "helm dep up" is used to +# copy it over to another location where it is referenced. +values.schema.yaml + +# ----------------------------------------------------------------------------- + # Patterns to ignore when building packages. # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. From 9ef0ca92e587855b40c11474ed219b294db2d343 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 1 Mar 2022 23:39:53 +0100 Subject: [PATCH 3/8] helm chart schemas: add chart jsonschemas as yaml files --- helm-charts/basehub/values.schema.yaml | 254 +++++++++++++++++++++++++ helm-charts/daskhub/values.schema.yaml | 26 +++ 2 files changed, 280 insertions(+) create mode 100644 helm-charts/basehub/values.schema.yaml create mode 100644 helm-charts/daskhub/values.schema.yaml diff --git a/helm-charts/basehub/values.schema.yaml b/helm-charts/basehub/values.schema.yaml new file mode 100644 index 0000000000..b8c0087468 --- /dev/null +++ b/helm-charts/basehub/values.schema.yaml @@ -0,0 +1,254 @@ +# This schema (a jsonschema in YAML format) is used to generate +# values.schema.json which is, when available, used by the helm CLI for client +# side validation by Helm of the chart's values before template rendering. +# +# We look to document everything we have default values for in values.yaml, but +# we don't look to enforce the perfect validation logic within this file. +# +# ref: https://json-schema.org/learn/getting-started-step-by-step.html +# +$schema: http://json-schema.org/draft-07/schema# +type: object +additionalProperties: false +required: + - azureFile + - nfs + - inClusterNFS + - global + - jupyterhub +properties: + azureFile: + type: object + additionalProperties: false + required: + - enabled + - pv + properties: + enabled: + type: boolean + pv: + type: object + additionalProperties: false + required: + - secretNamespace + - secretName + - shareName + - mountOptions + properties: + secretNamespace: + type: string + secretName: + type: string + shareName: + type: string + mountOptions: + type: array + items: + type: string + nfs: + type: object + additionalProperties: false + required: + - enabled + - shareCreator + - pv + properties: + enabled: + type: boolean + shareCreator: + type: object + additionalProperties: false + required: + - enabled + - tolerations + properties: + enabled: + type: boolean + tolerations: + type: array + items: + type: object + additionalProperties: true + pv: + type: object + additionalProperties: false + required: + - mountOptions + - serverIP + - baseShareName + properties: + mountOptions: + type: array + items: + type: string + serverIP: + type: string + baseShareName: + type: string + inClusterNFS: + type: object + additionalProperties: false + required: + - enabled + - size + properties: + enabled: + type: boolean + size: + type: string + global: + type: object + additionalProperties: true + jupyterhub: + type: object + additionalProperties: true + required: + - custom + properties: + custom: + type: object + additionalProperties: true + required: + - singleuserAdmin + - cloudResources + - docs_service + - 2i2c + properties: + homepage: + type: object + additionalProperties: false + required: + - templateVars + properties: + templateVars: + type: object + additionalProperties: false + required: + - org + - designed_by + - operated_by + - funded_by + properties: + announcements: + type: array + items: + type: string + org: + type: object + additionalProperties: false + required: + - name + - logo_url + - url + properties: + name: + type: string + logo_url: + type: string + url: + type: string + designed_by: + type: object + additionalProperties: false + required: + - name + - url + properties: + name: + type: string + url: + type: string + operated_by: + type: object + additionalProperties: false + required: + - name + - url + properties: + name: + type: string + url: + type: string + funded_by: + type: object + additionalProperties: false + required: + - name + - url + properties: + name: + type: string + url: + type: string + singleuserAdmin: + type: object + additionalProperties: false + required: + - extraVolumeMounts + properties: + extraVolumeMounts: + type: array + items: + type: object + additionalProperties: true + cloudResources: + type: object + additionalProperties: false + required: + - provider + - gcp + - scratchBucket + properties: + provider: + enum: ["", gcp] + gcp: + type: object + additionalProperties: false + required: + - projectId + properties: + projectId: + type: string + scratchBucket: + type: object + additionalProperties: false + required: + - enabled + properties: + enabled: + type: boolean + docs_service: + type: object + additionalProperties: false + required: + - enabled + - repo + - branch + properties: + enabled: + type: boolean + repo: + type: string + branch: + type: string + 2i2c: + type: object + additionalProperties: false + required: + - add_staff_user_ids_to_admin_users + - add_staff_user_ids_of_type + - staff_github_ids + - staff_google_ids + properties: + add_staff_user_ids_to_admin_users: + type: boolean + add_staff_user_ids_of_type: + type: string + staff_github_ids: + type: array + items: + type: string + staff_google_ids: + type: array + items: + type: string diff --git a/helm-charts/daskhub/values.schema.yaml b/helm-charts/daskhub/values.schema.yaml new file mode 100644 index 0000000000..e2aa57a57f --- /dev/null +++ b/helm-charts/daskhub/values.schema.yaml @@ -0,0 +1,26 @@ +# This schema (a jsonschema in YAML format) is used to generate +# values.schema.json which is, when available, used by the helm CLI for client +# side validation by Helm of the chart's values before template rendering. +# +# We look to document everything we have default values for in values.yaml, but +# we don't look to enforce the perfect validation logic within this file. +# +# ref: https://json-schema.org/learn/getting-started-step-by-step.html +# +$schema: http://json-schema.org/draft-07/schema# +type: object +additionalProperties: false +required: + - basehub + - dask-gateway + - global +properties: + basehub: + type: object + additionalProperties: true + dask-gateway: + type: object + additionalProperties: true + global: + type: object + additionalProperties: true From 9e16beff3d5c6a81f405649ceb463bfb927dfad4 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 1 Mar 2022 23:25:37 +0100 Subject: [PATCH 4/8] Cleanup broken chart config without changing current outcomes --- config/clusters/openscapes/staging.values.yaml | 2 -- config/clusters/uwhackweeks/staging.values.yaml | 2 -- helm-charts/daskhub/values.yaml | 7 ------- 3 files changed, 11 deletions(-) diff --git a/config/clusters/openscapes/staging.values.yaml b/config/clusters/openscapes/staging.values.yaml index 05912e6c6b..e6228290da 100644 --- a/config/clusters/openscapes/staging.values.yaml +++ b/config/clusters/openscapes/staging.values.yaml @@ -1,5 +1,3 @@ -scratchBucket: - enabled: false basehub: nfs: pv: diff --git a/config/clusters/uwhackweeks/staging.values.yaml b/config/clusters/uwhackweeks/staging.values.yaml index c2cdb12c5a..a911b3e24d 100644 --- a/config/clusters/uwhackweeks/staging.values.yaml +++ b/config/clusters/uwhackweeks/staging.values.yaml @@ -1,5 +1,3 @@ -scratchBucket: - enabled: false basehub: nfs: pv: diff --git a/helm-charts/daskhub/values.yaml b/helm-charts/daskhub/values.yaml index c8195ca6da..9e970828b9 100644 --- a/helm-charts/daskhub/values.yaml +++ b/helm-charts/daskhub/values.yaml @@ -1,10 +1,3 @@ -scratchBucket: - # Enable a 'scratch' bucket per-hub, with read-write permissions for all - # users. This will set a `SCRATCH_BUCKET` env variable (and a PANGEO_SCRATCH variable - # too, for backwards compatibility). Users can share data with each other using - # this bucket. - enabled: true - basehub: # Copied from https://github.com/dask/helm-chart/blob/master/daskhub/values.yaml # FIXME: Properly use the upstream chart. From ba08c9ecb9780a9e1238f11740f885a779a2ed2e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 2 Mar 2022 01:11:09 +0100 Subject: [PATCH 5/8] Fix schema validation errors --- config/clusters/farallon/staging.values.yaml | 2 +- config/clusters/utoronto/staging.values.yaml | 4 ++++ config/clusters/uwhackweeks/staging.values.yaml | 2 +- helm-charts/basehub/values.yaml | 9 +++++++-- helm-charts/daskhub/values.yaml | 5 +++++ 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/config/clusters/farallon/staging.values.yaml b/config/clusters/farallon/staging.values.yaml index d1f07a81d1..9ea9cbbc2c 100644 --- a/config/clusters/farallon/staging.values.yaml +++ b/config/clusters/farallon/staging.values.yaml @@ -35,7 +35,7 @@ basehub: url: https://2i2c.org funded_by: name: Farallon Institute - urL: http://www.faralloninstitute.org/ + url: http://www.faralloninstitute.org/ singleuser: initContainers: # Need to explicitly fix ownership here, since EFS doesn't do anonuid diff --git a/config/clusters/utoronto/staging.values.yaml b/config/clusters/utoronto/staging.values.yaml index 5546e56fb1..17f600fdcb 100644 --- a/config/clusters/utoronto/staging.values.yaml +++ b/config/clusters/utoronto/staging.values.yaml @@ -47,6 +47,10 @@ jupyterhub: extraFiles: github-app-private-key.pem: mountPath: /etc/github/github-app-private-key.pem + # stringData field will be set via encrypted values files but added here + # to meet the chart schema validation requirements without the need to + # use secret values during the validation. + stringData: "dummy" gitconfig: mountPath: /etc/gitconfig # app-id comes from https://github.com/organizations/utoronto-2i2c/settings/apps/utoronto-jupyterhub-private-cloner diff --git a/config/clusters/uwhackweeks/staging.values.yaml b/config/clusters/uwhackweeks/staging.values.yaml index a911b3e24d..0371d67e5e 100644 --- a/config/clusters/uwhackweeks/staging.values.yaml +++ b/config/clusters/uwhackweeks/staging.values.yaml @@ -29,7 +29,7 @@ basehub: name: 2i2c url: https://2i2c.org funded_by: - name: + name: ICESat Hackweek url: https://icesat-2.hackweek.io singleuser: serviceAccountName: cloud-user-sa diff --git a/helm-charts/basehub/values.yaml b/helm-charts/basehub/values.yaml index 01d87dbd96..668231afb6 100644 --- a/helm-charts/basehub/values.yaml +++ b/helm-charts/basehub/values.yaml @@ -30,6 +30,11 @@ inClusterNFS: enabled: false size: 100Gi +# A placeholder as global values that can be referenced from the same location +# of any chart should be possible to provide, but aren't necessarily provided or +# used. +global: {} + jupyterhub: custom: singleuserAdmin: @@ -38,9 +43,9 @@ jupyterhub: mountPath: /home/jovyan/shared-readwrite subPath: _shared cloudResources: - provider: + provider: "" gcp: - projectId: + projectId: "" scratchBucket: enabled: false docs_service: diff --git a/helm-charts/daskhub/values.yaml b/helm-charts/daskhub/values.yaml index 9e970828b9..b21da4a926 100644 --- a/helm-charts/daskhub/values.yaml +++ b/helm-charts/daskhub/values.yaml @@ -201,3 +201,8 @@ dask-gateway: k8s.dask.org/node-purpose: core service: type: ClusterIP # Access Dask Gateway through JupyterHub. To access the Gateway from outside JupyterHub, this must be changed to a `LoadBalancer`. + +# A placeholder as global values that can be referenced from the same location +# of any chart should be possible to provide, but aren't necessarily provided or +# used. +global: {} From 5268ec76b898c2556ff2e01597a95126eb05e1b8 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 2 Mar 2022 01:13:26 +0100 Subject: [PATCH 6/8] deployer: support helm chart schema validation --- deployer/__main__.py | 82 ++++++++++++++++++++++++++------- deployer/auth.py | 4 +- deployer/hub.py | 22 ++++----- deployer/utils.py | 50 ++++++++++++++++++-- helm-charts/basehub/values.yaml | 2 +- 5 files changed, 122 insertions(+), 38 deletions(-) diff --git a/deployer/__main__.py b/deployer/__main__.py index 9ed1c80fe7..29f685240d 100644 --- a/deployer/__main__.py +++ b/deployer/__main__.py @@ -3,23 +3,26 @@ """ import argparse import os +import shutil import subprocess +import sys from pathlib import Path import jsonschema from ruamel.yaml import YAML -import shutil from auth import KeyProvider from hub import Cluster from utils import ( get_decrypted_file, + prepare_helm_charts_dependencies_and_schemas, print_colour, find_absolute_path_to_cluster_file, ) # Without `pure=True`, I get an exception about str / byte issues yaml = YAML(typ="safe", pure=True) +helm_charts_dir = Path(__file__).parent.parent.joinpath("helm-charts") def use_cluster_credentials(cluster_name): @@ -31,8 +34,7 @@ def use_cluster_credentials(cluster_name): This function is to be used with the `use-cluster-credentials` CLI command only - it is not used by the rest of the deployer codebase. """ - # Validate our config with JSON Schema first before continuing - validate(cluster_name) + validate_cluster_config(cluster_name) config_file_path = find_absolute_path_to_cluster_file(cluster_name) with open(config_file_path) as f: @@ -55,8 +57,7 @@ def deploy_support(cluster_name): """ Deploy support components to a cluster """ - # Validate our config with JSON Schema first before continuing - validate(cluster_name) + validate_cluster_config(cluster_name) config_file_path = find_absolute_path_to_cluster_file(cluster_name) with open(config_file_path) as f: @@ -75,8 +76,7 @@ def deploy_grafana_dashboards(cluster_name): Grafana dashboards and deployment mechanism in question are maintained in this repo: https://github.com/jupyterhub/grafana-dashboards """ - # Validate our config with JSON Schema first before continuing - validate(cluster_name) + validate_cluster_config(cluster_name) config_file_path = find_absolute_path_to_cluster_file(cluster_name) with open(config_file_path) as f: @@ -165,9 +165,8 @@ def deploy(cluster_name, hub_name, skip_hub_health_test, config_path): """ Deploy one or more hubs in a given cluster """ - - # Validate our config with JSON Schema first before continuing - validate(cluster_name) + validate_cluster_config(cluster_name) + validate_hub_config(cluster_name, hub_name) with get_decrypted_file(config_path) as decrypted_file_path: with open(decrypted_file_path) as f: @@ -209,19 +208,64 @@ def deploy(cluster_name, hub_name, skip_hub_health_test, config_path): hub.deploy(k, SECRET_KEY, skip_hub_health_test) -def validate(cluster_name): - schema_file = Path(os.getcwd()).joinpath( +def validate_cluster_config(cluster_name): + """ + Validates cluster.yaml configuration against a JSONSchema. + """ + cluster_schema_file = Path(os.getcwd()).joinpath( "shared", "deployer", "cluster.schema.yaml" ) - config_file = find_absolute_path_to_cluster_file(cluster_name) + cluster_file = find_absolute_path_to_cluster_file(cluster_name) - with open(config_file) as cf, open(schema_file) as sf: + with open(cluster_file) as cf, open(cluster_schema_file) as sf: cluster_config = yaml.load(cf) schema = yaml.load(sf) # Raises useful exception if validation fails jsonschema.validate(cluster_config, schema) +def validate_hub_config(cluster_name, hub_name): + """ + Validates the provided non-encrypted helm chart values files for each hub of + a specific cluster. + """ + prepare_helm_charts_dependencies_and_schemas() + + config_file_path = find_absolute_path_to_cluster_file(cluster_name) + with open(config_file_path) as f: + cluster = Cluster(yaml.load(f), config_file_path.parent) + + hubs = [] + if hub_name: + hubs = [h for h in cluster.hubs if h.spec["name"] == hub_name] + else: + hubs = cluster.hubs + + for i, hub in enumerate(hubs): + print_colour( + f"{i+1} / {len(hubs)}: Validating non-encrypted hub values files for {hub.spec['name']}..." + ) + + cmd = [ + "helm", + "template", + str(helm_charts_dir.joinpath(hub.spec["helm_chart"])), + ] + for values_file in hub.spec["helm_chart_values_files"]: + if "secret" not in os.path.basename(values_file): + cmd.append(f"--values={config_file_path.parent.joinpath(values_file)}") + # Workaround the current requirement for dask-gateway 0.9.0 to have a + # JupyterHub api-token specified, for updates if this workaround can be + # removed, see https://github.com/dask/dask-gateway/issues/473. + if hub.spec["helm_chart"] == "daskhub": + cmd.append("--set=dask-gateway.gateway.auth.jupyterhub.apiToken=dummy") + try: + subprocess.check_output(cmd, text=True) + except subprocess.CalledProcessError as e: + print(e.stdout) + sys.exit(1) + + def main(): argparser = argparse.ArgumentParser( description="""A command line tool to perform various functions related @@ -270,7 +314,12 @@ def main(): validate_parser = subparsers.add_parser( "validate", parents=[base_parser], - help="Validate the cluster configuration against a JSON schema", + help="Validate the cluster.yaml configuration itself, as well as the provided non-encrypted helm chart values files for each hub or the specified hub.", + ) + validate_parser.add_argument( + "hub_name", + nargs="?", + help="The hub, or list of hubs, to validate provided non-encrypted helm chart values for.", ) # deploy-support subcommand @@ -305,7 +354,8 @@ def main(): args.config_path, ) elif args.action == "validate": - validate(args.cluster_name) + validate_cluster_config(args.cluster_name) + validate_hub_config(args.cluster_name, args.hub_name) elif args.action == "deploy-support": deploy_support(args.cluster_name) elif args.action == "deploy-grafana-dashboards": diff --git a/deployer/auth.py b/deployer/auth.py index 97719502c1..117d6b6860 100644 --- a/deployer/auth.py +++ b/deployer/auth.py @@ -1,8 +1,8 @@ +import re + from auth0.v3.authentication import GetToken from auth0.v3.management import Auth0 - from yarl import URL -import re # What key in the authenticated user's profile to use as hub username # This shouldn't be changeable by the user! diff --git a/deployer/hub.py b/deployer/hub.py index 6d2c1671ec..505e8b33d0 100644 --- a/deployer/hub.py +++ b/deployer/hub.py @@ -1,26 +1,28 @@ -from auth import KeyProvider import hashlib import hmac import json import os -import sys import subprocess +import sys import tempfile from contextlib import contextmanager, redirect_stderr, redirect_stdout -from textwrap import dedent from pathlib import Path +from textwrap import dedent import pytest from ruamel.yaml import YAML +from auth import KeyProvider from utils import ( get_decrypted_file, - print_colour, get_decrypted_files, + prepare_helm_charts_dependencies_and_schemas, + print_colour, ) # Without `pure=True`, I get an exception about str / byte issues yaml = YAML(typ="safe", pure=True) +helm_charts_dir = Path(__file__).parent.parent.joinpath("helm-charts") class Cluster: @@ -526,6 +528,8 @@ def deploy(self, auth_provider, secret_key, skip_hub_health_test=False): """ Deploy this hub """ + prepare_helm_charts_dependencies_and_schemas() + # Support overriding domain configuration in the loaded cluster.yaml via # a cluster.yaml specified enc-.secret.yaml file that only # includes the domain configuration of a typical cluster.yaml file. @@ -550,16 +554,6 @@ def deploy(self, auth_provider, secret_key, skip_hub_health_test=False): generated_values = self.get_generated_config(auth_provider, secret_key) - # Ensure helm charts are up to date - helm_charts_dir = (Path(__file__).parent.parent).joinpath("helm-charts") - subprocess.check_call( - ["helm", "dep", "up", helm_charts_dir.joinpath("basehub")] - ) - if self.spec["helm_chart"] == "daskhub": - subprocess.check_call( - ["helm", "dep", "up", helm_charts_dir.joinpath("daskhub")] - ) - with tempfile.NamedTemporaryFile( mode="w" ) as generated_values_file, get_decrypted_files( diff --git a/deployer/utils.py b/deployer/utils.py index b4bd48d5c6..90164eed8c 100644 --- a/deployer/utils.py +++ b/deployer/utils.py @@ -1,14 +1,17 @@ -import os +import functools import json -import tempfile +import os import subprocess -from ruamel.yaml import YAML -from ruamel.yaml.scanner import ScannerError +import tempfile +import warnings from contextlib import contextmanager, ExitStack from pathlib import Path -import warnings + +from ruamel.yaml import YAML +from ruamel.yaml.scanner import ScannerError yaml = YAML(typ="safe", pure=True) +helm_charts_dir = Path(__file__).parent.parent.joinpath("helm-charts") def assert_file_exists(filepath): @@ -148,6 +151,43 @@ def get_decrypted_files(files, abspath): ] +@functools.lru_cache +def _generate_values_schema_json(helm_chart_dir): + """ + This script reads the values.schema.yaml files part of our Helm charts and + generates a values.schema.json that can allowing helm the CLI to perform + validation of passed values before rendering templates or making changes in k8s. + + FIXME: Currently we have a hard coupling between the deployer script and the + Helm charts part of this repo. Managing the this logic here is a + compromise but it should really be managed as part of packaging it + and uploading it to a helm chart registry instead. + """ + values_schema_yaml = os.path.join(helm_chart_dir, "values.schema.yaml") + values_schema_json = os.path.join(helm_chart_dir, "values.schema.json") + + with open(values_schema_yaml) as f: + schema = yaml.load(f) + with open(values_schema_json, "w") as f: + json.dump(schema, f) + + +@functools.lru_cache +def prepare_helm_charts_dependencies_and_schemas(): + """ + Ensures that the helm charts we deploy, basehub and daskhub, have got their + dependencies updated and .json schema files generated so that they can be + rendered during validation or deployment. + """ + basehub_dir = helm_charts_dir.joinpath("basehub") + _generate_values_schema_json(basehub_dir) + subprocess.check_call(["helm", "dep", "up", basehub_dir]) + + daskhub_dir = helm_charts_dir.joinpath("daskhub") + _generate_values_schema_json(daskhub_dir) + subprocess.check_call(["helm", "dep", "up", daskhub_dir]) + + def print_colour(msg: str): """Print messages in colour to be distinguishable in CI logs diff --git a/helm-charts/basehub/values.yaml b/helm-charts/basehub/values.yaml index 668231afb6..1de6329180 100644 --- a/helm-charts/basehub/values.yaml +++ b/helm-charts/basehub/values.yaml @@ -353,7 +353,7 @@ jupyterhub: c.JupyterHub.template_paths = ['/usr/local/share/jupyterhub/custom_templates/'] c.JupyterHub.template_vars = { - 'custom':get_config('custom.homepage.templateVars') + 'custom': get_config('custom.homepage.templateVars') } 05-custom-admin: | from z2jh import get_config From 138693a53905f756904a45ae121f5b0d7869cf99 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 2 Mar 2022 01:48:35 +0100 Subject: [PATCH 7/8] ci: add workflow to run deployer validate on changed clusters' hubs --- .github/workflows/validate-clusters.yaml | 96 ++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 .github/workflows/validate-clusters.yaml diff --git a/.github/workflows/validate-clusters.yaml b/.github/workflows/validate-clusters.yaml new file mode 100644 index 0000000000..5ff06db7eb --- /dev/null +++ b/.github/workflows/validate-clusters.yaml @@ -0,0 +1,96 @@ +# This is a GitHub workflow defining a set of jobs with a set of steps. ref: +# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions +# +# Runs the deployer script to validate clusters. This will both validate +# cluster.yaml files as well as each hubs passed non-encrypted values files +# against the Helm charts' values schema. +# +name: Validate clusters + +on: + pull_request: + paths: + - config/clusters/** + - deployer/** + - helm-charts/basehub/** + - helm-charts/daskhub/** + - requirements.txt + - .github/workflows/validate-hubs.yaml + push: + paths: + - config/clusters/** + - deployer/** + - helm-charts/basehub/** + - helm-charts/daskhub/** + - requirements.txt + - .github/workflows/validate-hubs.yaml + branches-ignore: + - "dependabot/**" + - "pre-commit-ci-update-config" + tags: + - "**" + workflow_dispatch: + +jobs: + validate-hubs-values-files: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - cluster_name: 2i2c + - cluster_name: azure.carbonplan + - cluster_name: carbonplan + - cluster_name: cloudbank + - cluster_name: farallon + - cluster_name: meom-ige + - cluster_name: openscapes + - cluster_name: pangeo-hubs + - cluster_name: utoronto + - cluster_name: uwhackweeks + + steps: + - uses: actions/checkout@v2 + + - name: Check if any cluster common files has changed + uses: dorny/paths-filter@v2 + id: cluster_common_files + with: + filters: | + files: + - deployer/** + - helm-charts/basehub/** + - helm-charts/daskhub/** + - requirements.txt + - .github/workflows/validate-hubs.yaml + + - name: Check if cluster specific files has changes + uses: dorny/paths-filter@v2 + id: cluster_specific_files + with: + filters: | + changes: + - config/clusters/${{ matrix.cluster_name }}/** + + # To continue this cluster specific job we must either have manually + # invoked this workflow to run for all clusters, or there should have been + # changes to the cluster common files or cluster specific files. + - name: Decide if the job should continue + id: decision + run: | + echo ::set-output name=continue-job::${{ github.event_name == 'workflow_dispatch' || (steps.cluster_common_files.outputs.files == 'true' || steps.cluster_specific_files.outputs.changes == 'true') }} + + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + + - name: Install deployer script dependencies + run: | + pip install -r requirements.txt + + - name: "Validate cluster: ${{ matrix.cluster_name }}" + if: steps.decision.outputs.continue-job == 'true' + env: + TERM: xterm + run: | + python deployer validate ${{ matrix.cluster_name }} From c976eb9df4a90713262efe3b48e6073903ba09da Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 2 Mar 2022 01:14:35 +0100 Subject: [PATCH 8/8] Cleanup requirements.txt and dev-requirements.txt --- .github/actions/deploy/action.yml | 1 - .../workflows/deploy-grafana-dashboards.yaml | 1 - .github/workflows/deploy-hubs.yaml | 1 - .github/workflows/doc-links.yml | 5 ----- dev-requirements.txt | 14 +++++++++++--- docs/reference/ci-cd.md | 1 - requirements.txt | 19 +++++++++++++++---- 7 files changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index e57c8f5023..56f4f92991 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -19,7 +19,6 @@ runs: - name: Setup dependencies run: | python3 -m pip install -r requirements.txt - python3 -m pip install -r dev-requirements.txt shell: bash - name: Deploy support components run: | diff --git a/.github/workflows/deploy-grafana-dashboards.yaml b/.github/workflows/deploy-grafana-dashboards.yaml index 2c9832b5a7..52a3e48cff 100644 --- a/.github/workflows/deploy-grafana-dashboards.yaml +++ b/.github/workflows/deploy-grafana-dashboards.yaml @@ -22,7 +22,6 @@ jobs: - name: Setup dependencies run: | python3 -m pip install -r requirements.txt - python3 -m pip install -r dev-requirements.txt sudo apt install jsonnet - name: Setup gcloud diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index feeb54c64c..5f34af6df8 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -63,7 +63,6 @@ jobs: - "deployer/**" - "helm-charts/**" - "requirements.txt" - - "dev-requirements.txt" - "config/secrets.yaml" - ".github/workflows/deploy-hubs.yaml" - ".github/actions/deploy/*" diff --git a/.github/workflows/doc-links.yml b/.github/workflows/doc-links.yml index 9f782bc0b1..120a9624af 100644 --- a/.github/workflows/doc-links.yml +++ b/.github/workflows/doc-links.yml @@ -13,11 +13,6 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - with: - # chartpress is used by doc/conf.py, - # and requires information about the latest tagged commit, which - # requires the git history. - fetch-depth: 0 - name: Install environment uses: conda-incubator/setup-miniconda@v2 diff --git a/dev-requirements.txt b/dev-requirements.txt index 4f81875cca..5d92dc7bd2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,13 @@ -pytest -pytest-asyncio +# These requirements represents the needs for doing various tasks in this git +# repo besides using the deployer script. +# + +# chartpress is relevant to build and push helm-charts/images/hub/Dockerfile and +# update basehub's default values to reference the new image. +chartpress + +# requests is used by extra_scripts/rsync-active-users.py requests -beautifulsoup4 + +# rich is used by extra_scripts/count-auth0-apps.py rich diff --git a/docs/reference/ci-cd.md b/docs/reference/ci-cd.md index 7646574c16..d3c2deb681 100644 --- a/docs/reference/ci-cd.md +++ b/docs/reference/ci-cd.md @@ -11,7 +11,6 @@ following paths are modified: - deployer/** - helm-charts/** - requirements.txt -- dev-requirements.txt - config/secrets.yaml - config/clusters/** - .github/workflows/deploy-hubs.yaml diff --git a/requirements.txt b/requirements.txt index 8724b5e154..9e172679bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,18 @@ -backoff -chartpress -six # temp workaround for docker==5.0.0, used by chartpress, that failed to depend on six +# This file represents the needs for the deployer script to function, while the +# dev-requirements.txt file represents the needs in this repo in general. +# + +# ruamel.yaml is used to read and write .yaml files. ruamel.yaml + +# auth0 is used to communicate with Auth0's REST API that we integrate with in +# various ways. auth0-python -jhub-client==0.1.4 + +# jsonschema is used for validating cluster.yaml configurations jsonschema + +# jhub_client, pytest, and pytest_asyncio are used for our health checks +jhub-client==0.1.4 +pytest +pytest-asyncio