E2E Workflow scripts test against Enterprise follower in K8s (#349)

* Maintain shell on failure

* Test E2E workflow against Conjur Enterprise in KinD

Conjur Leader and Follower in K8s cluster.

* Enterprise E2E workflow updated for GKE

* E2E workflow: CONJUR_NAMESPACE to CONJUR_NAMESPACE_NAME

Maintains style with TEST_APP_NAMESPACE_NAME, and agrees with
kubernetes-conjur-deploy required env var

* Add Jenkinsfile stage for Enterprise/GKE E2E workflow

* Update CI GHA workflow

* E2E Workflow README updates

* PR Review updates

Jenkinsfile

@@ -82,6 +82,16 @@ pipeline {
       }
     }
 
+    stage('E2E Workflow Tests') {
+      parallel {
+        stage('Enterprise and test app deployed to GKE') {
+          steps {
+            sh 'cd bin/test-workflow && summon --environment gke ./start --enterprise --platform gke'
+          }
+        }
+      }
+    }
+
     stage('Publish client Docker images') {
       parallel {
         stage('On a master build') {

bin/test-workflow/0_prep_env.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -o pipefail
+
+### PLATFORM DETAILS
+export CONJUR_OSS_HELM_INSTALLED="${CONJUR_OSS_HELM_INSTALLED:-true}"
+
+# PLATFORM is used to differentiate between general Kubernetes platforms (K8s vs. oc), while
+# CLUSTER_TYPE is used to differentiate between sub-platforms (for vanilla K8s, KinD vs. GKE)
+if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
+  CLUSTER_TYPE="${CLUSTER_TYPE:-kind}"
+else
+  CLUSTER_TYPE="${CLUSTER_TYPE:-gke}"
+  export UNIQUE_TEST_ID="$(uuidgen | tr "[:upper:]" "[:lower:]" | head -c 10)"
+fi
+export CLUSTER_TYPE
+
+if [[ "$CLUSTER_TYPE" == "oc" ]]; then
+  PLATFORM="openshift"
+else
+  PLATFORM="kubernetes"
+fi
+export PLATFORM
+
+### DOCKER CONFIG
+export USE_DOCKER_LOCAL_REGISTRY="${USE_DOCKER_LOCAL_REGISTRY:-true}"
+export DOCKER_REGISTRY_URL="${DOCKER_REGISTRY_URL:-localhost:5000}"
+export DOCKER_REGISTRY_PATH="${DOCKER_REGISTRY_PATH:-localhost:5000}"
+export PULL_DOCKER_REGISTRY_URL="${PULL_DOCKER_REGISTRY_URL:-${DOCKER_REGISTRY_URL}}"
+export PULL_DOCKER_REGISTRY_PATH="${PULL_DOCKER_REGISTRY_PATH:-${DOCKER_REGISTRY_PATH}}"
+
+### CONJUR AND TEST APP CONFIG
+export CONJUR_ACCOUNT="${CONJUR_ACCOUNT:-myConjurAccount}"
+export AUTHENTICATOR_ID="${AUTHENTICATOR_ID:-my-authenticator-id}"
+export CONJUR_AUTHN_LOGIN_RESOURCE="${CONJUR_AUTHN_LOGIN_RESOURCE:-service_account}"
+export CONJUR_AUTHN_LOGIN_PREFIX="${CONJUR_AUTHN_LOGIN_PREFIX:-host/conjur/authn-k8s/$AUTHENTICATOR_ID/apps}"
+export CONJUR_VERSION="${CONJUR_VERSION:-5}"
+export TEST_APP_NAMESPACE_NAME="${TEST_APP_NAMESPACE_NAME:-app-test}"
+export TEST_APP_DATABASE="${TEST_APP_DATABASE:-postgres}"
+
+if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
+    conjur_service="conjur-oss"
+    export CONJUR_NAMESPACE_NAME="${CONJUR_NAMESPACE_NAME:-$conjur_service}"
+else
+    conjur_service="conjur-master"
+    export CONJUR_NAMESPACE_NAME="${CONJUR_NAMESPACE_NAME:-$conjur_service-${UNIQUE_TEST_ID}}"
+    export TEST_APP_NAMESPACE_NAME="$TEST_APP_NAMESPACE_NAME-$UNIQUE_TEST_ID"
+fi
+
+export CONJUR_APPLIANCE_URL=${CONJUR_APPLIANCE_URL:-https://$conjur_service.$CONJUR_NAMESPACE_NAME.svc.cluster.local}
+export SAMPLE_APP_BACKEND_DB_PASSWORD="$(openssl rand -hex 12)"
+
+### PLATFORM SPECIFIC CONFIG
+if [[ "$CLUSTER_TYPE" == "gke" ]]; then
+    export CONJUR_FOLLOWER_URL="https://conjur-follower.$CONJUR_NAMESPACE_NAME.svc.cluster.local"
+    export CONJUR_ADMIN_PASSWORD="MySecretP@ss1"
+    export CONJUR_APPLIANCE_IMAGE="registry2.itci.conjur.net/conjur-appliance:5.0-stable"
+    export CONJUR_FOLLOWER_COUNT=1
+    export CONJUR_AUTHN_LOGIN="host/conjur/authn-k8s/${AUTHENTICATOR_ID}/apps/$CONJUR_NAMESPACE_NAME/service_account/conjur-cluster"
+    export STOP_RUNNING_ENV=true
+    export DEPLOY_MASTER_CLUSTER=true
+    export CONFIGURE_CONJUR_MASTER=true
+    export PLATFORM_CONTAINER="platform-container"
+
+    docker build --tag "$PLATFORM_CONTAINER:$CONJUR_NAMESPACE_NAME" \
+        --file Dockerfile \
+        --build-arg KUBECTL_VERSION="$KUBECTL_VERSION" \
+        .
+fi

bin/test-workflow/1_deploy_conjur.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -eo pipefail
+cd "$(dirname "$0")" || ( echo "cannot cd into dir" && exit 1 )
+
+source utils.sh
+
+function setup_conjur_enterprise {
+  check_env_var GCLOUD_PROJECT_NAME
+  check_env_var GCLOUD_ZONE
+  check_env_var GCLOUD_CLUSTER_NAME
+  check_env_var GCLOUD_SERVICE_KEY
+
+  docker pull "$CONJUR_APPLIANCE_IMAGE"
+
+  pushd temp > /dev/null
+    git clone --single-branch --branch master [email protected]:cyberark/kubernetes-conjur-deploy "kubernetes-conjur-deploy-$UNIQUE_TEST_ID"
+  popd > /dev/null
+
+  announce "Deploying Conjur Enterprise"
+
+  if [[ "${CLUSTER_TYPE}" == "gke" ]]; then
+    run_command_with_platform "cd temp/kubernetes-conjur-deploy-$UNIQUE_TEST_ID && ./start"
+  fi
+}
+
+function setup_conjur_open_source {
+  pushd temp > /dev/null
+    git clone https://github.com/cyberark/conjur-oss-helm-chart.git
+
+    pushd conjur-oss-helm-chart/examples/common > /dev/null
+      source ./utils.sh
+
+      announce "Setting demo environment variable defaults"
+      source ../kubernetes-in-docker/0_export_env_vars.sh
+
+      announce "Creating a Kubernetes-in-Docker cluster if necessary"
+      ./1_create_kind_cluster.sh
+
+      announce "Helm installing/upgrading Conjur OSS cluster"
+      ./2_helm_install_or_upgrade_conjur.sh
+
+      # Wait for Conjur pods to become ready (just in case there are old
+      # Conjur pods getting terminated as part of Helm upgrade)
+      announce "Waiting for Conjur to become ready"
+      wait_for_conjur_ready
+
+      announce "Enabling the Conjur Kubernetes authenticator if necessary"
+      ./4_ensure_authn_k8s_enabled.sh
+    popd > /dev/null
+
+    rm -rf conjur-oss-helm-chart
+  popd > /dev/null
+}
+
+mkdir -p temp
+if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
+  setup_conjur_open_source
+else
+  setup_conjur_enterprise
+fi

bin/test-workflow/2_admin_load_conjur_policies.sh

@@ -7,13 +7,17 @@ PLATFORM="${PLATFORM:-kubernetes}"
 
 source utils.sh
 
+if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
+  export CONJUR_ADMIN_PASSWORD="$(get_admin_password)"
+fi
+
 check_env_var TEST_APP_NAMESPACE_NAME
 check_env_var CONJUR_VERSION
 check_env_var CONJUR_ACCOUNT
 check_env_var CONJUR_APPLIANCE_URL
 check_env_var CONJUR_ADMIN_PASSWORD
 check_env_var AUTHENTICATOR_ID
-check_env_var CONJUR_NAMESPACE
+check_env_var CONJUR_NAMESPACE_NAME
 check_env_var TEST_APP_DATABASE
 check_env_var SAMPLE_APP_BACKEND_DB_PASSWORD
 
@@ -22,12 +26,12 @@ announce "Generating Conjur policy."
 prepare_conjur_cli_image() {
   announce "Pulling and pushing Conjur CLI image."
 
-  docker pull cyberark/conjur-cli:$CONJUR_VERSION-latest
+  docker pull cyberark/conjur-cli:"$CONJUR_VERSION"-latest
 
-  cli_app_image=$(platform_image_for_push conjur-cli)
-  docker tag cyberark/conjur-cli:$CONJUR_VERSION-latest $cli_app_image
+  cli_app_image="$(platform_image_for_push conjur-cli)"
+  docker tag cyberark/conjur-cli:"$CONJUR_VERSION"-latest "$cli_app_image"
 
-  docker push $cli_app_image
+  docker push "$cli_app_image"
 }
 
 deploy_conjur_cli() {
@@ -69,36 +73,39 @@ pushd policy > /dev/null
   fi
 
   sed "s#{{ AUTHENTICATOR_ID }}#$AUTHENTICATOR_ID#g" ./templates/cluster-authn-svc-def.template.yml |
-    sed "s#{{ CONJUR_NAMESPACE }}#$CONJUR_NAMESPACE#g" > ./generated/$TEST_APP_NAMESPACE_NAME.cluster-authn-svc.yml
+    sed "s#{{ CONJUR_NAMESPACE }}#$CONJUR_NAMESPACE_NAME#g" > ./generated/"$TEST_APP_NAMESPACE_NAME".cluster-authn-svc.yml
 
   sed "s#{{ AUTHENTICATOR_ID }}#$AUTHENTICATOR_ID#g" ./templates/project-authn-def.template.yml |
     sed "s#{{ IS_OPENSHIFT }}#$is_openshift#g" |
     sed "s#{{ IS_KUBERNETES }}#$is_kubernetes#g" |
-    sed "s#{{ TEST_APP_NAMESPACE_NAME }}#$TEST_APP_NAMESPACE_NAME#g" > ./generated/$TEST_APP_NAMESPACE_NAME.project-authn.yml
+    sed "s#{{ TEST_APP_NAMESPACE_NAME }}#$TEST_APP_NAMESPACE_NAME#g" > ./generated/"$TEST_APP_NAMESPACE_NAME".project-authn.yml
 
   sed "s#{{ AUTHENTICATOR_ID }}#$AUTHENTICATOR_ID#g" ./templates/app-identity-def.template.yml |
-    sed "s#{{ TEST_APP_NAMESPACE_NAME }}#$TEST_APP_NAMESPACE_NAME#g" > ./generated/$TEST_APP_NAMESPACE_NAME.app-identity.yml
+    sed "s#{{ TEST_APP_NAMESPACE_NAME }}#$TEST_APP_NAMESPACE_NAME#g" > ./generated/"$TEST_APP_NAMESPACE_NAME".app-identity.yml
 
   sed "s#{{ AUTHENTICATOR_ID }}#$AUTHENTICATOR_ID#g" ./templates/authn-any-policy-branch.template.yml |
     sed "s#{{ IS_OPENSHIFT }}#$is_openshift#g" |
-    sed "s#{{ TEST_APP_NAMESPACE_NAME }}#$TEST_APP_NAMESPACE_NAME#g" > ./generated/$TEST_APP_NAMESPACE_NAME.authn-any-policy-branch.yml
+    sed "s#{{ TEST_APP_NAMESPACE_NAME }}#$TEST_APP_NAMESPACE_NAME#g" > ./generated/"$TEST_APP_NAMESPACE_NAME".authn-any-policy-branch.yml
 popd > /dev/null
 
-set_namespace "$CONJUR_NAMESPACE"
+set_namespace "$CONJUR_NAMESPACE_NAME"
 
 announce "Finding or creating a Conjur CLI pod"
-conjur_cli_pod=$(get_conjur_cli_pod_name)
+conjur_cli_pod="$(get_conjur_cli_pod_name)"
 if [ -z "$conjur_cli_pod" ]; then
   prepare_conjur_cli_image
   deploy_conjur_cli
-  conjur_cli_pod=$(get_conjur_cli_pod_name)
+  conjur_cli_pod="$(get_conjur_cli_pod_name)"
+fi
+
+if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
+  ensure_conjur_cli_initialized "$conjur_cli_pod"
 fi
-ensure_conjur_cli_initialized $conjur_cli_pod
 
 announce "Loading Conjur policy."
 
-$cli exec $conjur_cli_pod -- rm -rf /policy
-$cli cp ./policy $conjur_cli_pod:/policy
+$cli exec "$conjur_cli_pod" -- rm -rf /policy
+$cli cp ./policy "$conjur_cli_pod:/policy"
 
 wait_for_it 300 "$cli exec $conjur_cli_pod -- \
   bash -c \"
@@ -112,6 +119,6 @@ wait_for_it 300 "$cli exec $conjur_cli_pod -- \
   \"
 "
 
-$cli exec $conjur_cli_pod -- rm -rf ./policy
+$cli exec "$conjur_cli_pod" -- rm -rf ./policy
 
 echo "Conjur policy loaded."

bin/test-workflow/3_admin_init_conjur_cert_authority.sh

@@ -3,25 +3,23 @@
 set -euo pipefail
 cd "$(dirname "$0")" || ( echo "cannot cd into dir" && exit 1 )
 
-PLATFORM="${PLATFORM:-kubernetes}"
-
 source utils.sh
 
-check_env_var CONJUR_NAMESPACE
+check_env_var CONJUR_NAMESPACE_NAME
 check_env_var CONJUR_OSS_HELM_INSTALLED
 check_env_var CONJUR_ACCOUNT
 check_env_var AUTHENTICATOR_ID
 
 announce "Initializing Conjur certificate authority."
 
-set_namespace $CONJUR_NAMESPACE
+set_namespace $CONJUR_NAMESPACE_NAME
 
-conjur_master=$(get_master_pod_name)
+conjur_master="$(get_master_pod_name)"
 
 if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
-    $cli exec $conjur_master -c conjur-oss -- bash -c "CONJUR_ACCOUNT=$CONJUR_ACCOUNT rake authn_k8s:ca_init['conjur/authn-k8s/$AUTHENTICATOR_ID']"
+    $cli exec "$conjur_master" -c conjur-oss -- bash -c "CONJUR_ACCOUNT=$CONJUR_ACCOUNT rake authn_k8s:ca_init['conjur/authn-k8s/$AUTHENTICATOR_ID']"
 else
-    $cli exec $conjur_master -- chpst -u conjur conjur-plugin-service possum rake authn_k8s:ca_init["conjur/authn-k8s/$AUTHENTICATOR_ID"]
+    $cli exec "$conjur_master" -- chpst -u conjur conjur-plugin-service possum rake authn_k8s:ca_init["conjur/authn-k8s/$AUTHENTICATOR_ID"]
 fi
 
 echo "Certificate authority initialized."

bin/test-workflow/4_admin_cluster_prep.sh

@@ -3,27 +3,40 @@
 set -euo pipefail
 cd "$(dirname "$0")" || ( echo "cannot cd into dir" && exit 1 )
 
-PLATFORM="${PLATFORM:-kubernetes}"
 TIMEOUT="${TIMEOUT:-5m0s}"
 
 source utils.sh
 
 check_env_var CONJUR_APPLIANCE_URL
-check_env_var CONJUR_NAMESPACE
+check_env_var CONJUR_NAMESPACE_NAME
 check_env_var CONJUR_ACCOUNT
 check_env_var AUTHENTICATOR_ID
+if [[ "$CONJUR_OSS_HELM_INSTALLED" == "false" ]]; then
+  check_env_var CONJUR_FOLLOWER_URL
+fi
 
 set_namespace default
 
 # Prepare our cluster with conjur and authnK8s credentials in a golden configmap
 announce "Installing cluster prep chart"
 pushd ../../helm/conjur-config-cluster-prep > /dev/null
+  if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
     ./bin/get-conjur-cert.sh -v -i -s -u "$CONJUR_APPLIANCE_URL"
 
-    helm upgrade --install cluster-prep . -n "$CONJUR_NAMESPACE" --debug --wait --timeout $TIMEOUT \
+    helm upgrade --install cluster-prep . -n "$CONJUR_NAMESPACE_NAME" --debug --wait --timeout "$TIMEOUT" \
         --set conjur.account="$CONJUR_ACCOUNT" \
         --set conjur.applianceUrl="$CONJUR_APPLIANCE_URL" \
         --set conjur.certificateFilePath="files/conjur-cert.pem" \
         --set authnK8s.authenticatorID="$AUTHENTICATOR_ID"
-
+  else
+    ./bin/get-conjur-cert.sh -v -i -s -u "$CONJUR_FOLLOWER_URL"
+
+    helm upgrade --install cluster-prep . -n "$CONJUR_NAMESPACE_NAME" --debug --wait --timeout "$TIMEOUT" \
+        --set conjur.account="$CONJUR_ACCOUNT" \
+        --set conjur.applianceUrl="$CONJUR_FOLLOWER_URL" \
+        --set conjur.certificateFilePath="files/conjur-cert.pem" \
+        --set authnK8s.authenticatorID="$AUTHENTICATOR_ID" \
+        --set authnK8s.serviceAccount.create=false \
+        --set authnK8s.serviceAccount.name="conjur-cluster"
+  fi
 popd > /dev/null