From 9891b49dd54d0ae18cba86e75e8ae3da5a0e6e1b Mon Sep 17 00:00:00 2001
From: Nicolas Pepin-Perreault <nicolas.pepin-perreault@camunda.com>
Date: Mon, 11 Dec 2023 14:41:26 +0100
Subject: [PATCH] feat: add job push cluster terminate experiment

---
 .../job-push/cluster-terminate.json           | 76 +++++++++++++++++++
 .../camunda-cloud/manifest.yml                |  4 +
 2 files changed, 80 insertions(+)
 create mode 100644 go-chaos/internal/chaos-experiments/camunda-cloud/job-push/cluster-terminate.json

diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/job-push/cluster-terminate.json b/go-chaos/internal/chaos-experiments/camunda-cloud/job-push/cluster-terminate.json
new file mode 100644
index 000000000..ed7cf0937
--- /dev/null
+++ b/go-chaos/internal/chaos-experiments/camunda-cloud/job-push/cluster-terminate.json
@@ -0,0 +1,76 @@
+{
+  "version": "0.1.0",
+  "title": "Job push cluster restart experiment",
+  "description": "Job workers with streaming enabled should be fault-tolerant. The worker should receive and complete jobs even if all brokers crash.",
+  "contributions": {
+      "performance": "high",
+      "reliability": "high",
+      "availability": "high"
+  },
+  "steady-state-hypothesis": {
+      "title": "Zeebe is alive",
+      "probes": [
+          {
+              "name": "All pods should be ready",
+              "type": "probe",
+              "tolerance": 0,
+              "provider": {
+                  "type": "process",
+                  "path": "zbchaos",
+                  "arguments": ["verify", "readiness"],
+                  "timeout": 900
+              }
+          },
+          {
+              "name": "Can deploy process model",
+              "type": "probe",
+              "tolerance": 0,
+              "provider": {
+                  "type": "process",
+                  "path": "zbchaos",
+                  "arguments": ["deploy", "process"],
+                  "timeout": 900
+              }
+          },
+          {
+            "name": "Can deploy workers",
+            "type": "probe",
+            "tolerance": 0,
+            "provider": {
+                "type": "process",
+                "path": "zbchaos",
+                "arguments": ["deploy", "worker", "--pollingDelay", "86400000"],
+                "timeout": 900
+            },
+            "pauses": {
+                "after": 5
+            }
+          },
+          {
+              "name": "Should be able to create process instances",
+              "type": "probe",
+              "tolerance": 0,
+              "provider": {
+                  "type": "process",
+                  "path": "zbchaos",
+                  "arguments": ["verify", "instance-creation", "--awaitResult"],
+                  "timeout": 900
+              }
+          }
+      ]
+  },
+  "method": [
+    {
+        "name": "Terminate all brokers",
+        "type": "probe",
+        "tolerance": 0,
+        "provider": {
+            "type": "process",
+            "path": "zbchaos",
+            "arguments": ["terminate", "broker", "--all"],
+            "timeout": 900
+        }
+    }
+  ],
+  "rollbacks": []
+}
diff --git a/go-chaos/internal/chaos-experiments/camunda-cloud/manifest.yml b/go-chaos/internal/chaos-experiments/camunda-cloud/manifest.yml
index 4048c8c1f..5374a1a99 100644
--- a/go-chaos/internal/chaos-experiments/camunda-cloud/manifest.yml
+++ b/go-chaos/internal/chaos-experiments/camunda-cloud/manifest.yml
@@ -51,6 +51,10 @@ experiments:
     clusterPlans:
       - production-s
     minVersion: 8.4
+  - path: job-push/cluster-terminate.json
+    clusterPlans:
+      - production-s
+    minVersion: 8.4
   - path: worker-resilience/gateway-restart.json
     clusterPlans:
       - production-s