Merge pull request #55 from mojaloop/k8s

feat: support k8s
mojaloop · Apr 30, 2024 · d4ab3b9 · d4ab3b9
2 parents 19bd9f4 + 0d17f1f
commit d4ab3b9
Show file tree

Hide file tree

Showing 74 changed files with 1,896 additions and 143 deletions.
diff --git a/.env b/.env
@@ -41,7 +41,7 @@ KAFKA_EXPORTER_VERSION=latest
 CADVISER_VERSION=latest
 
 ## Load Docker Image Versions
-K6_VERSION=0.47.0
+K6_VERSION=0.50.0
 
 ## Performance Testing Replicas
 QS_REPLICAS=1

diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,6 @@
 **/reports/**
 
 automate_perf.env
+perf.override.env
+perf-on-prem.env
+k8s.yaml
diff --git a/README.md b/README.md
@@ -419,3 +419,37 @@ To capture results without running tests, use the following command
 ```bash
 ./automate_perf.sh -c -f <From Time in Milliseconds> -t <To time in Milliseconds>
 ```
+
+### Testing Performance of Remote Mojaloop Deployment
+
+For executing performance test scenarios against a Mojaloop deployment, follow the steps below:
+
+1. **Set Environment Variables:**
+   - Set `perf.override.env` with the proper endpoints of the Mojaloop services.
+
+2. **Customize Configurations:**
+   - Edit the file `docker/ml-testing-toolkit/test-cases/environments/remote-k8s-env.json` to customize currencies and MSISDNs according to your requirements.
+
+3. **Run Simulators and TTK Provisioning:**
+   ```bash
+   docker compose --project-name simulators -f docker-compose-perf.yml --profile 8dfsp --profile testing-toolkit --profile ttk-provisioning-remote-k8s --profile oracle up -d
+   ```
+
+4. **Run Monitoring Services:**
+   ```bash
+   docker compose --project-name monitoring --profile transfers-test -f docker-compose-monitoring.yml up -d
+   ```
+
+5. **Execute Single Transfer Test Case:**
+   ```bash
+   env K6_SCRIPT_CONFIG_FILE_NAME=fspiopSingleTransfer.json docker compose --project-name load -f docker-compose-load.yml up
+   ```
+
+6. **Stop Services:**
+   ```bash
+   docker compose --project-name simulators -f docker-compose-perf.yml --profile 8dfsp --profile testing-toolkit --profile ttk-provisioning-remote-k8s down -v
+   docker compose --project-name monitoring --profile transfers-test -f docker-compose-monitoring.yml down -v
+   ```
+
+> **Note:** The `-v` argument is optional and will delete any volume data created by the monitoring Docker Compose.
+
diff --git a/docker-compose-load.yml b/docker-compose-load.yml
@@ -16,22 +16,28 @@ networks:
 services:
   k6:
     image: grafana/k6:${K6_VERSION}
+    sysctls:
+      # websockets do not have keep-alive, resulting in many TIME_WAIT sockets
+      net.ipv4.tcp_tw_reuse: 1
     env_file:
       - .env
       - perf.env
+      - perf.override.env
     command:
       - run
       # - '--help' ## Useful to list run options
+      # - '--http-debug=full'
       - '--include-system-env-vars'
       - '/scripts/index.js'
     networks:
       - load-net
       - monitoring-net
       - mojaloop-net
     ports:
-      - "6565:6565"
+      - "5665:5665"
     environment:
       - K6_SCRIPT_CONFIG_FILE_NAME=${K6_SCRIPT_CONFIG_FILE_NAME:-test.json}
+      - K6_WEB_DASHBOARD=true
       # - K6_PROMETHEUS_RW_SERVER_URL=http://prometheus:9090/api/v1/write # Ref: https://k6.io/docs/results-output/real-time/prometheus-remote-write/#options
       # - K6_PROMETHEUS_RW_TREND_AS_NATIVE_HISTOGRAM=true # Ref: https://k6.io/docs/results-output/real-time/prometheus-remote-write/#options
       # - K6_OUT=experimental-prometheus-rw # Ref: https://k6.io/docs/results-output/real-time/prometheus-remote-write/

diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml
@@ -826,6 +826,29 @@ services:
     profiles:
       - ttk-provisioning-transfers
 
+  ttk-provisioning-remote-k8s:
+    image: mojaloop/ml-testing-toolkit-client-lib:${TEST_TTK_CLI_VERSION}
+    env_file:
+      - .env
+      - perf.env
+      - perf.override.env
+    volumes:
+      - "./docker/ml-testing-toolkit/test-cases/collections:/opt/app/collections"
+      - "./docker/ml-testing-toolkit/test-cases/environments:/opt/app/environments"
+      - "./reports:/opt/app/reports"
+    depends_on:
+      mojaloop-testing-toolkit:
+        condition: service_healthy
+    networks:
+      - mojaloop-net
+    command:
+      - sh
+      - -c
+      - 'sed -e "s~K6_CALLBACK_FQDN~$$K6_CALLBACK_FQDN~g" -e "s~K6_SCRIPT_ADMIN_ENDPOINT_URL~$$K6_SCRIPT_ADMIN_ENDPOINT_URL~g" -e "s~K6_SCRIPT_ALS_ADMIN_ENDPOINT_URL~$$K6_SCRIPT_ALS_ADMIN_ENDPOINT_URL~g" -e "s~K6_SCRIPT_FSPIOP_ALS_ENDPOINT_URL~$$K6_SCRIPT_FSPIOP_ALS_ENDPOINT_URL~g" -e "s~K6_SCRIPT_FSPIOP_QUOTES_ENDPOINT_URL~$$K6_SCRIPT_FSPIOP_QUOTES_ENDPOINT_URL~g" -e "s~K6_SCRIPT_ORACLE_ENDPOINT_URL~$$K6_SCRIPT_ORACLE_ENDPOINT_URL~g" environments/remote-k8s-env.json > /tmp/env.json && npm run cli -- -u http://mojaloop-testing-toolkit:5050 -l 2 -i collections/provisioning -e /tmp/env.json --report-format html --report-target file://reports/ttk-provisioning-report.html'
+    user: root
+    profiles:
+      - ttk-provisioning-remote-k8s
+
   ttk-provisioning-quotes:
     image: mojaloop/ml-testing-toolkit-client-lib:${TEST_TTK_CLI_VERSION}
     env_file:
@@ -886,8 +909,8 @@ services:
       context: ./packages/callback-handler-svc
       cache_from:
         - mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
-    # ports:
-    #   - "3001:3001"
+    ports:
+      - "30000:3001"
     networks:
       - mojaloop-net
     volumes:
@@ -906,6 +929,7 @@ services:
       - callback-handler
       - all-services
       - als-test
+      - oracle
 
   sdk-scheme-adapter-api-svc:
     image: mojaloop/sdk-scheme-adapter:${SDK_SCHEME_ADAPTER_VERSION}
@@ -991,19 +1015,23 @@ services:
       interval: 30s
 
   sim-perffsp1:
+    sysctls:
+      # websockets do not have keep-alive, resulting in many TIME_WAIT sockets
+      net.ipv4.tcp_tw_reuse: 1
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
     deploy:
       mode: replicated
       replicas: 1
     env_file:
       - .env
       - ./envs/perffsp1.env
+      - perf.override.env
     build:
       context: ./packages/callback-handler-svc
       cache_from:
         - mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
-    # ports:
-    #   - "3001:3001"
+    ports:
+      - "25001:3001"
     networks:
       - mojaloop-net
     volumes:
@@ -1026,19 +1054,23 @@ services:
       - 8dfsp
 
   sim-perffsp2:
+    sysctls:
+      # websockets do not have keep-alive, resulting in many TIME_WAIT sockets
+      net.ipv4.tcp_tw_reuse: 1
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
     # deploy:
     #   mode: replicated
     #   replicas: 1
     env_file:
       - .env
       - ./envs/perffsp2.env
+      - perf.override.env
     build:
       context: ./packages/callback-handler-svc
       cache_from:
         - mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
-    # ports:
-    #   - "3001:3001"
+    ports:
+      - "25002:3001"
     networks:
       - mojaloop-net
     volumes:
@@ -1061,19 +1093,23 @@ services:
       - 8dfsp
 
   sim-perffsp3:
+    sysctls:
+      # websockets do not have keep-alive, resulting in many TIME_WAIT sockets
+      net.ipv4.tcp_tw_reuse: 1
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
     deploy:
       mode: replicated
       replicas: 1
     env_file:
       - .env
       - ./envs/perffsp3.env
+      - perf.override.env
     build:
       context: ./packages/callback-handler-svc
       cache_from:
         - mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
-    # ports:
-    #   - "3001:3001"
+    ports:
+      - "25003:3001"
     networks:
       - mojaloop-net
     volumes:
@@ -1092,19 +1128,23 @@ services:
       - 8dfsp
 
   sim-perffsp4:
+    sysctls:
+      # websockets do not have keep-alive, resulting in many TIME_WAIT sockets
+      net.ipv4.tcp_tw_reuse: 1
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
     deploy:
       mode: replicated
       replicas: 1
     env_file:
       - .env
       - ./envs/perffsp4.env
+      - perf.override.env
     build:
       context: ./packages/callback-handler-svc
       cache_from:
         - mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
-    # ports:
-    #   - "3001:3001"
+    ports:
+      - "25004:3001"
     networks:
       - mojaloop-net
     volumes:
@@ -1123,19 +1163,23 @@ services:
       - 8dfsp
 
   sim-perffsp5:
+    sysctls:
+      # websockets do not have keep-alive, resulting in many TIME_WAIT sockets
+      net.ipv4.tcp_tw_reuse: 1
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
     deploy:
       mode: replicated
       replicas: 1
     env_file:
       - .env
       - ./envs/perffsp5.env
+      - perf.override.env
     build:
       context: ./packages/callback-handler-svc
       cache_from:
         - mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
-    # ports:
-    #   - "3001:3001"
+    ports:
+      - "25005:3001"
     networks:
       - mojaloop-net
     volumes:
@@ -1153,19 +1197,23 @@ services:
       - 8dfsp
 
   sim-perffsp6:
+    sysctls:
+      # websockets do not have keep-alive, resulting in many TIME_WAIT sockets
+      net.ipv4.tcp_tw_reuse: 1
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
     deploy:
       mode: replicated
       replicas: 1
     env_file:
       - .env
       - ./envs/perffsp6.env
+      - perf.override.env
     build:
       context: ./packages/callback-handler-svc
       cache_from:
         - mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
-    # ports:
-    #   - "3001:3001"
+    ports:
+      - "25006:3001"
     networks:
       - mojaloop-net
     volumes:
@@ -1183,19 +1231,23 @@ services:
       - 8dfsp
 
   sim-perffsp7:
+    sysctls:
+      # websockets do not have keep-alive, resulting in many TIME_WAIT sockets
+      net.ipv4.tcp_tw_reuse: 1
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
     deploy:
       mode: replicated
       replicas: 1
     env_file:
       - .env
       - ./envs/perffsp7.env
+      - perf.override.env
     build:
       context: ./packages/callback-handler-svc
       cache_from:
         - mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
-    # ports:
-    #   - "3001:3001"
+    ports:
+      - "25007:3001"
     networks:
       - mojaloop-net
     volumes:
@@ -1213,19 +1265,23 @@ services:
       - 8dfsp
 
   sim-perffsp8:
+    sysctls:
+      # websockets do not have keep-alive, resulting in many TIME_WAIT sockets
+      net.ipv4.tcp_tw_reuse: 1
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
     deploy:
       mode: replicated
       replicas: 1
     env_file:
       - .env
       - ./envs/perffsp8.env
+      - perf.override.env
     build:
       context: ./packages/callback-handler-svc
       cache_from:
         - mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
-    # ports:
-    #   - "3001:3001"
+    ports:
+      - "25008:3001"
     networks:
       - mojaloop-net
     volumes:

diff --git a/docker/ml-testing-toolkit/test-cases/collections/de-provisioning-perf-als/oracle.json b/docker/ml-testing-toolkit/test-cases/collections/de-provisioning-perf-als/oracle.json
@@ -0,0 +1,54 @@
+{
+  "options": {},
+  "name": "collections_tests_oracle",
+  "test_cases": [
+    {
+      "id": 1,
+      "name": "Add MSISDN Oracle",
+      "meta": {
+        "info": "Add MSISDN Oracle."
+      },
+      "fileInfo": {
+        "path": "collections/de-provisioning-perf-als/oracle.json"
+      },
+      "requests": [
+        {
+          "id": 6,
+          "description": "Add Oracle for MSISDN",
+          "params": {
+            "ID": "",
+            "Type": ""
+          },
+          "apiVersion": {
+            "minorVersion": 1,
+            "majorVersion": 1,
+            "type": "als_admin",
+            "specFile": "spec_files/api_definitions/als_admin_1.1/api_spec.yaml",
+            "callbackMapFile": "spec_files/api_definitions/als_admin_1.1/callback_map.json",
+            "responseMapFile": "spec_files/api_definitions/als_admin_1.1/response_map.json",
+            "jsfRefFile": "spec_files/api_definitions/als_admin_1.1/mockRef.json",
+            "triggerTemplatesFolder": "spec_files/api_definitions/als_admin_1.1/trigger_templates"
+          },
+          "operationPath": "/oracles",
+          "path": "/oracles",
+          "method": "post",
+          "url": "{$inputs.HOST_ACCOUNT_LOOKUP_ADMIN}",
+          "headers": {
+            "content-type": "application/vnd.interoperability.oracles+json;version=1.1",
+            "date": "",
+            "accept": "application/vnd.interoperability.oracles+json;version=1.1"
+          },
+          "body": {
+            "oracleIdType": "MSISDN",
+            "endpoint": {
+              "value": "{$inputs.PERF_ORACLE_ENDPOINT_BASE_URL}",
+              "endpointType": "URL"
+            },
+            "currency": "{$inputs.currency}"
+          },
+          "disabled": false
+        }
+      ]
+    }
+  ]
+}