From 12d4a157ac5be12bc94c1978c860f25c383ada29 Mon Sep 17 00:00:00 2001 From: Kevin Leyow Date: Wed, 23 Aug 2023 11:51:21 -0500 Subject: [PATCH] chore: add variables for replicas and move red panda definition (#24) * chore: add total transfers metric * chore: move kafka console to monitoring * chore: test * chore: some improvements * fix * comment * chore: comment --- .env | 7 ++ docker-compose-monitoring.yml | 15 +++ docker-compose-perf.yml | 118 ++++++++++-------- .../configs/ml-handler-notification.js | 3 +- ...Callback Hander Service-1689261777383.json | 19 ++- perf.env | 7 +- 6 files changed, 113 insertions(+), 56 deletions(-) diff --git a/.env b/.env index 7871f551..13cb708f 100644 --- a/.env +++ b/.env @@ -21,3 +21,10 @@ CADVISER_VERSION=latest ## Load Docker Image Versions K6_VERSION=0.45.0 + +## Performance Testing Replicas +ALS_REPLICAS=1 +CENTRAL_LEDGER_GENERAL_REPLICAS=4 +CENTRAL_LEDGER_POSITION_REPLICAS=8 +ML_API_ADAPTER_REPLICAS=4 +ML_NOTIFICATION_REPLICAS=4 diff --git a/docker-compose-monitoring.yml b/docker-compose-monitoring.yml index 5fb6d811..6a4a8f74 100644 --- a/docker-compose-monitoring.yml +++ b/docker-compose-monitoring.yml @@ -179,3 +179,18 @@ services: - monitoring-net command: - --kafka.server=kafka:9092 + + ## Debug utilities + kafka-debug-console: + # image: quay.io/cloudhut/kowl:v1.4.0 + image: docker.redpanda.com/redpandadata/console:latest + deploy: + replicas: 1 + restart: on-failure + hostname: kafka-debug-console + ports: + - "9080:8080" + networks: + - mojaloop-net + environment: + - KAFKA_BROKERS=kafka:29092 diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml index 6134fd8e..9438044b 100644 --- a/docker-compose-perf.yml +++ b/docker-compose-perf.yml @@ -11,7 +11,9 @@ services: env_file: - .env - perf.env - scale: 1 + deploy: + mode: replicated + replicas: ${ALS_REPLICAS} command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/account-lookup-service.js /opt/app/config/default.json && /opt/app/wait4/wait4.js account-lookup-service && npm run migrate && node src/index.js server" # ports: # - "4001:4001" @@ -96,7 +98,9 @@ services: env_file: - .env - perf.env - scale: 1 + deploy: + mode: replicated + replicas: 1 command: sh -c "npm run migrate" # ports: # - "3001:3001" @@ -132,7 +136,9 @@ services: env_file: - .env - perf.env - scale: 4 + deploy: + mode: replicated + replicas: ${CENTRAL_LEDGER_GENERAL_REPLICAS} command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-ledger-disabled-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/api/index.js" # ports: # - "3001:3001" @@ -177,7 +183,9 @@ services: env_file: - .env - perf.env - scale: 4 + deploy: + mode: replicated + replicas: ${CENTRAL_LEDGER_GENERAL_REPLICAS} command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/handlers/index.js handler --prepare" # ports: # - "3001:3001" @@ -219,7 +227,9 @@ services: env_file: - .env - perf.env - scale: 8 + deploy: + mode: replicated + replicas: ${CENTRAL_LEDGER_POSITION_REPLICAS} command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/handlers/index.js handler --position" # ports: # - "3001:3001" @@ -261,7 +271,9 @@ services: env_file: - .env - perf.env - scale: 4 + deploy: + mode: replicated + replicas: ${CENTRAL_LEDGER_GENERAL_REPLICAS} command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/handlers/index.js handler --fulfil" # ports: # - "3001:3001" @@ -380,7 +392,9 @@ services: env_file: - .env - perf.env - scale: 1 # We dont need to scale this unless we are testing funds-in/out + deploy: + mode: replicated + replicas: 1 # We dont need to scale this unless we are testing funds-in/out command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/handlers/index.js handler --admin" # ports: # - "3001:3001" @@ -453,16 +467,16 @@ services: # blocks until kafka is reachable kafka-topics.sh --bootstrap-server kafka:29092 --list echo -e 'Creating kafka topics' - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-prepare --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-position --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-fulfil --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-notification-event --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-get --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-admin-transfer --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM - # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-prepare --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM - # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-fulfil --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM - # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-processing --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM - # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-get --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-prepare --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-position --replication-factor 1 --partitions $$KAFKA_POSITION_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-fulfil --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-notification-event --replication-factor 1 --partitions $$KAFKA_NOTIFICATION_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-get --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-admin-transfer --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM + # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-prepare --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM + # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-fulfil --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM + # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-processing --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM + # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-get --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM echo -e 'Successfully created the following topics:' kafka-topics.sh --bootstrap-server kafka:29092 --list " @@ -486,7 +500,7 @@ services: ALLOW_PLAINTEXT_LISTENER: "yes" KAFKA_ADVERTISED_HOST_NAME: kafka KAFKA_CFG_LISTENERS: CONTROLLER://:9093,LISTENER_DOCKER://kafka:29092,LISTENER_EXTERN://:9092 - KAFKA_CFG_ADVERTISED_LISTENERS: LISTENER_DOCKER://kafka:29092,LISTENER_EXTERN://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: LISTENER_DOCKER://kafka:29092,LISTENER_EXTERN://localhost:9092 KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,LISTENER_DOCKER:PLAINTEXT,LISTENER_EXTERN:PLAINTEXT KAFKA_CFG_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 @@ -549,7 +563,9 @@ services: env_file: - .env - perf.env - scale: 4 + deploy: + mode: replicated + replicas: ${ML_API_ADAPTER_REPLICAS} # container_name: ml-api-adapter command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/ml-api-adapter-disabled-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js ml-api-adapter && node src/api/index.js" # ports: @@ -586,7 +602,9 @@ services: env_file: - .env - perf.env - scale: 4 + deploy: + mode: replicated + replicas: ${ML_NOTIFICATION_REPLICAS} command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/ml-handler-notification.js /opt/app/config/default.json && /opt/app/wait4/wait4.js ml-api-adapter && node src/handlers/index.js handler --notification" # ports: # - "3000:3000" @@ -731,8 +749,10 @@ services: - ttk-provisioning-transfers callback-handler-svc-oracle-sim: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - perf.env @@ -762,8 +782,10 @@ services: - als-test callback-handler-svc-cl-sim: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - ./envs/cl-sim.env @@ -794,8 +816,10 @@ services: - transfers-test sim-perffsp1: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - ./envs/perffsp1.env @@ -828,8 +852,10 @@ services: - 8dfsp sim-perffsp2: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - ./envs/perffsp2.env @@ -862,8 +888,10 @@ services: - 8dfsp sim-perffsp3: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - ./envs/perffsp3.env @@ -891,8 +919,10 @@ services: - 8dfsp sim-perffsp4: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - ./envs/perffsp4.env @@ -920,8 +950,10 @@ services: - 8dfsp sim-perffsp5: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - ./envs/perffsp5.env @@ -948,8 +980,10 @@ services: - 8dfsp sim-perffsp6: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - ./envs/perffsp6.env @@ -976,8 +1010,10 @@ services: - 8dfsp sim-perffsp7: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - ./envs/perffsp7.env @@ -1004,8 +1040,10 @@ services: - 8dfsp sim-perffsp8: - scale: 1 image: mojaloop/callback-handler-svc:local + deploy: + mode: replicated + replicas: 1 env_file: - .env - ./envs/perffsp8.env @@ -1030,23 +1068,3 @@ services: user: root profiles: - 8dfsp - - - ## Debug utilities - kafka-debug-console: - # image: quay.io/cloudhut/kowl:v1.4.0 - image: docker.redpanda.com/redpandadata/console:latest - deploy: - replicas: 1 - restart: on-failure - hostname: kafka-debug-console - ports: - - "9080:8080" - networks: - - mojaloop-net - environment: - - KAFKA_BROKERS=kafka:29092 - depends_on: - - kafka - profiles: - - debug \ No newline at end of file diff --git a/docker/config-modifier/configs/ml-handler-notification.js b/docker/config-modifier/configs/ml-handler-notification.js index 76afbd1c..1c241a6a 100644 --- a/docker/config-modifier/configs/ml-handler-notification.js +++ b/docker/config-modifier/configs/ml-handler-notification.js @@ -11,7 +11,8 @@ module.exports = { "EVENT": { "config": { "rdkafkaConf": { - "metadata.broker.list": "kafka:29092" + "metadata.broker.list": "kafka:29092", + "enable.auto.commit": false } } } diff --git a/docker/grafana/provisioning/dashboards/Supporting Services - Callback Hander Service-1689261777383.json b/docker/grafana/provisioning/dashboards/Supporting Services - Callback Hander Service-1689261777383.json index d404aaaf..69610680 100644 --- a/docker/grafana/provisioning/dashboards/Supporting Services - Callback Hander Service-1689261777383.json +++ b/docker/grafana/provisioning/dashboards/Supporting Services - Callback Hander Service-1689261777383.json @@ -18,6 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, + "id": 2, "links": [ { "asDropdown": false, @@ -231,6 +232,19 @@ "legendFormat": "{{instance}} op:{{operation}} success: {{success}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(cbs_ing_callbackHandler_count[$__rate_interval])) by (operation, success)", + "hide": false, + "instant": false, + "legendFormat": "Total op:{{operation}} success: {{success}}", + "range": true, + "refId": "B" } ], "title": "Handler Ingress - Processed Per Second", @@ -631,8 +645,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -735,6 +748,6 @@ "timezone": "browser", "title": "Supporting Services - Callback Hander Service", "uid": "afe9dda2-878b-40c8-90df-072e7ecaaaee", - "version": 2, + "version": 4, "weekStart": "" } diff --git a/perf.env b/perf.env index 503d37e6..32129a0d 100644 --- a/perf.env +++ b/perf.env @@ -51,8 +51,6 @@ K6_SCRIPT_FSPIOP_TRANSFERS_CURRENCY=USD K6_SCRIPT_CONFIG_FILE_NAME=test.json -KAFKA_PARTITIONS_NUM=19 - ## General Config UV_THREADPOOL_SIZE=24 LOG_LEVEL=info @@ -60,6 +58,10 @@ LOG_LEVEL=info ## Disable Audit & Trace events from being dumped to logfile # EVENT_SDK_LOG_FILTER="" +KAFKA_GENERAL_PARTITIONS_NUM=19 +KAFKA_POSITION_PARTITIONS_NUM=19 +KAFKA_NOTIFICATION_PARTITIONS_NUM=19 + ALS_DATABASE__HOST=mysql-als ALS_DATABASE__USER=account_lookup ALS_DATABASE__PASSWORD=password @@ -80,3 +82,4 @@ CLEDG_DATABASE__SCHEMA=central_ledger ## MLAPI - ML API Adapter API/Handler Config MLAPI_TRANSFERS__SEND_TRANSFER_CONFIRMATION_TO_PAYEE=false +# MLAPI_ENDPOINT_SOURCE_URL=http://callback-handler-svc-cl-sim:3001/admin