fix the flakiness

.tekton/pipeline/test-pipeline-service-deployment.yaml

@@ -51,6 +51,7 @@ spec:
         name: setup-ci-runner-container
       runAfter:
         - "deploy-cluster"
+      retries: 2
       params:
         - name: target_branch
           value: $(params.target_branch)

.tekton/pipeline/test-pipeline-service-upgrade.yaml

@@ -48,6 +48,7 @@ spec:
         name: setup-ci-runner-container
       runAfter:
         - "deploy-cluster"
+      retries: 2
       params:
         - name: target_branch
           value: $(params.target_branch)

ci/images/ci-runner/hack/bin/copy-plnsvc-code.sh

@@ -5,7 +5,15 @@ set -o pipefail
 set -x
 
 echo "Copy source code of pipeline service to the ci-runner container"
-kubectl cp "./" "default/ci-runner:/workspace/source"
+# kubectl cp "./" "default/ci-runner:/workspace/source"
+
+for _ in {1..10}; do
+    if kubectl cp "./" "default/ci-runner:/workspace/source"; then
+        break
+    fi
+    echo "Failed to Copy source code of pipeline service to the ci-runner container, retrying ..."
+    sleep 5
+done
 
 echo "Copy sidecar sources to the ci-runner container"
 kubectl cp "./ci/images/ci-runner/hack/sidecar" "default/ci-runner:/workspace/sidecar"

ci/images/ci-runner/hack/bin/create-ci-runner-container.sh

@@ -10,8 +10,11 @@ MANIFEST_DIR=$(
 )
 kubectl -n default apply -k "$MANIFEST_DIR/sidecar"
 
-kubectl -n default wait pod/ci-runner --for=condition=Ready --timeout=90s
-kubectl -n default describe pod/ci-runner
+if ! kubectl -n default wait pod/ci-runner --for=condition=Ready --timeout=180s; then
+    echo "ci-runner is not ready" >&2
+    kubectl -n default describe pod/ci-runner >&2
+    exit 1
+fi
 
 echo
 echo "KUBECONFIG:"

ci/images/ci-runner/hack/bin/run-plnsvc-setup.sh

@@ -8,5 +8,13 @@ set -x
 touch "$PWD/destroy-cluster.txt"
 
 echo "Execute dev_setup.sh script to set up pipeline-service ..."
-kubectl -n default exec pod/ci-runner -- \
-    sh -c "/workspace/sidecar/bin/plnsvc_setup.sh $REPO_URL $REPO_REVISION"
+# if the following comand fail, it retries 3 times with 5 seconds sleep
+for _ in {1..3}; do
+    if kubectl -n default exec pod/ci-runner -- \
+        sh -c "/workspace/sidecar/bin/plnsvc_setup.sh $REPO_URL $REPO_REVISION"; then
+        break
+    fi
+    echo "Failed to execute dev_setup.sh script, retrying ..."
+    sleep 5
+done
+

operator/test/test.sh

@@ -4,6 +4,7 @@
 set -o errexit
 set -o nounset
 set -o pipefail
+set -x
 
 usage() {
   echo "
@@ -87,7 +88,11 @@ setup_test() {
 }
 
 wait_for_pipeline() {
-  kubectl wait --for=condition=succeeded "$1" -n "$2" --timeout 300s >"$DEBUG_OUTPUT"
+  if ! kubectl wait --for=condition=succeeded "$1" -n "$2" --timeout 300s >"$DEBUG_OUTPUT"; then
+    echo "[ERROR] Pipeline failed to complete successful" >&2
+    kubectl get pipelineruns "$1" -n "$2" >"$DEBUG_OUTPUT"
+    exit 1
+  fi
 }
 
 check_pod_security() {