diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml index 2d3b4830..fcbc179c 100644 --- a/docker-compose-perf.yml +++ b/docker-compose-perf.yml @@ -93,6 +93,7 @@ services: ## Services for Transfer tests central-ledger: image: mojaloop/central-ledger:${CENTRAL_LEDGER_VERSION} + scale: 1 command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-ledger-disabled-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/api/index.js" # ports: # - "3001:3001" @@ -131,6 +132,7 @@ services: central-handler-prepare: image: mojaloop/central-ledger:${CENTRAL_LEDGER_VERSION} + scale: 2 command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/handlers/index.js handler --prepare" # ports: # - "3001:3001" @@ -169,6 +171,7 @@ services: central-handler-position: image: mojaloop/central-ledger:${CENTRAL_LEDGER_VERSION} + scale: 8 command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/handlers/index.js handler --position" # ports: # - "3001:3001" @@ -207,6 +210,7 @@ services: central-handler-fulfil: image: mojaloop/central-ledger:${CENTRAL_LEDGER_VERSION} + scale: 2 command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/handlers/index.js handler --fulfil" # ports: # - "3001:3001" @@ -322,6 +326,7 @@ services: central-handler-admin: image: mojaloop/central-ledger:${CENTRAL_LEDGER_VERSION} + scale: 2 command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/handlers/index.js handler --admin" # ports: # - "3001:3001" @@ -384,6 +389,8 @@ services: networks: - mojaloop-net image: docker.io/bitnami/kafka:3.4.0 + env_file: + - perf.env depends_on: - kafka entrypoint: [ '/bin/sh', '-c' ] @@ -392,16 +399,16 @@ services: # blocks until kafka is reachable kafka-topics.sh --bootstrap-server kafka:29092 --list echo -e 'Creating kafka topics' - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-prepare --replication-factor 1 --partitions 1 - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-position --replication-factor 1 --partitions 1 - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-fulfil --replication-factor 1 --partitions 1 - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-notification-event --replication-factor 1 --partitions 1 - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-get --replication-factor 1 --partitions 1 - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-admin-transfer --replication-factor 1 --partitions 1 - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-prepare --replication-factor 1 --partitions 1 - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-fulfil --replication-factor 1 --partitions 1 - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-processing --replication-factor 1 --partitions 1 - kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-get --replication-factor 1 --partitions 1 + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-prepare --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-position --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-fulfil --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-notification-event --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-get --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-admin-transfer --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-prepare --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-fulfil --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-processing --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM + # kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-bulk-get --replication-factor 1 --partitions $$KAFKA_PARTITIONS_NUM echo -e 'Successfully created the following topics:' kafka-topics.sh --bootstrap-server kafka:29092 --list " @@ -483,7 +490,8 @@ services: ml-api-adapter: image: mojaloop/ml-api-adapter:${ML_API_ADAPTER_VERSION} - container_name: ml-api-adapter + scale: 2 + # container_name: ml-api-adapter command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/ml-api-adapter-disabled-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js ml-api-adapter && node src/api/index.js" # ports: # - "3000:3000" @@ -516,6 +524,7 @@ services: ml-handler-notification: image: mojaloop/ml-api-adapter:${ML_API_ADAPTER_VERSION} + scale: 4 command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/ml-handler-notification.js /opt/app/config/default.json && /opt/app/wait4/wait4.js ml-api-adapter && node src/handlers/index.js handler --notification" # ports: # - "3000:3000" diff --git a/docker/grafana/provisioning/dashboards/dashboard-k6-monitoring.json b/docker/grafana/provisioning/dashboards/dashboard-k6-monitoring.json index 784047b7..e11615c1 100644 --- a/docker/grafana/provisioning/dashboards/dashboard-k6-monitoring.json +++ b/docker/grafana/provisioning/dashboards/dashboard-k6-monitoring.json @@ -37,7 +37,7 @@ "title": "Image render", "tooltip": "Image Render", "type": "link", - "url": "/render/d/${__dashboard.uid}/${__dashboard}?height=4500&width=4000" + "url": "/render/d/${__dashboard.uid}/${__dashboard}?height=5000&width=4000" } ], "liveNow": true, @@ -918,18 +918,11 @@ "value": "ops" } ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "" - }, - "properties": [] } ] }, "gridPos": { - "h": 17, + "h": 28, "w": 24, "x": 0, "y": 6 @@ -962,9 +955,9 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", + "editorMode": "code", "exemplar": false, - "expr": "k6_vus{testid=~\"$testid\"}", + "expr": "avg by(testid) (k6_vus{testid=~\"$testid\"})", "hide": false, "instant": false, "interval": "", @@ -1009,7 +1002,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(histogram_sum(rate(k6_http_req_duration_seconds{testid=~\"$testid\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_duration_seconds{testid=~\"$testid\"}[$__rate_interval]))) ", + "expr": "avg by(testid) ((histogram_sum(rate(k6_http_req_duration_seconds{testid=~\"$testid\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_duration_seconds{testid=~\"$testid\"}[$__rate_interval]))))", "hide": false, "legendFormat": "Response Time (avg) - {{testid}} - {{scenario}} - {{name}}", "range": true, @@ -1021,7 +1014,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "(histogram_sum(rate(k6_iteration_duration_seconds{testid=~\"$testid\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_iteration_duration_seconds{testid=~\"$testid\"}[$__rate_interval]))) ", + "expr": "avg by(testid) ((histogram_sum(rate(k6_iteration_duration_seconds{testid=~\"$testid\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_iteration_duration_seconds{testid=~\"$testid\"}[$__rate_interval]))) )", "hide": false, "instant": false, "legendFormat": "Iteration Duration (avg) - {{testid}} - {{scenario}} - {{name}}", @@ -1048,14 +1041,30 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum by(testid, payerFspId, payeeFspId) (rate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", + "expr": "sum by(testid, payeeFspId) (rate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", "format": "time_series", "hide": false, "instant": false, "interval": "", - "legendFormat": "Requests Rate - {{testid}} - {{payerFspId}} -> {{payeeFspId}}", + "legendFormat": "Requests Rate - {{testid}} -> {{payeeFspId}}", "range": true, "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(testid, payerFspId, payeeFspId) (rate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "Requests Rate - {{testid}} - {{payerFspId}} -> {{payeeFspId}}", + "range": true, + "refId": "H" } ], "type": "timeseries" @@ -1117,7 +1126,7 @@ "h": 5, "w": 4, "x": 0, - "y": 23 + "y": 34 }, "id": 85, "interval": "1s", @@ -1245,7 +1254,7 @@ "h": 10, "w": 9, "x": 4, - "y": 23 + "y": 34 }, "id": 18, "interval": "5s", @@ -1398,7 +1407,7 @@ "h": 10, "w": 11, "x": 13, - "y": 23 + "y": 34 }, "id": 78, "interval": "5", @@ -1514,7 +1523,7 @@ "h": 5, "w": 4, "x": 0, - "y": 28 + "y": 39 }, "id": 92, "interval": "1s", @@ -1578,8 +1587,7 @@ "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "red" }, { "color": "#EAB839", @@ -1648,7 +1656,7 @@ "h": 7, "w": 13, "x": 0, - "y": 33 + "y": 44 }, "id": 104, "interval": "1", @@ -1805,8 +1813,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1858,7 +1865,7 @@ "h": 7, "w": 11, "x": 13, - "y": 33 + "y": 44 }, "id": 107, "interval": "1", @@ -1986,7 +1993,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 51 }, "id": 32, "panels": [], @@ -2025,8 +2032,7 @@ "mode": "absolute", "steps": [ { - "color": "blue", - "value": null + "color": "blue" } ] }, @@ -2534,7 +2540,7 @@ "h": 9, "w": 24, "x": 0, - "y": 41 + "y": 52 }, "id": 43, "interval": "1", @@ -2792,8 +2798,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2809,7 +2814,7 @@ "h": 11, "w": 24, "x": 0, - "y": 50 + "y": 61 }, "id": 41, "interval": "1s", @@ -2988,8 +2993,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3005,7 +3009,7 @@ "h": 11, "w": 24, "x": 0, - "y": 61 + "y": 72 }, "id": 121, "interval": "1s", @@ -3158,7 +3162,7 @@ "h": 7, "w": 24, "x": 0, - "y": 72 + "y": 83 }, "id": 54, "interval": "5s", @@ -3364,7 +3368,7 @@ "h": 1, "w": 24, "x": 0, - "y": 79 + "y": 90 }, "id": 35, "panels": [], @@ -3476,7 +3480,7 @@ "h": 7, "w": 24, "x": 0, - "y": 80 + "y": 91 }, "id": 116, "interval": "5s", @@ -4204,7 +4208,7 @@ "h": 7, "w": 24, "x": 0, - "y": 87 + "y": 98 }, "id": 122, "interval": "1", @@ -4670,7 +4674,7 @@ "h": 13, "w": 24, "x": 0, - "y": 94 + "y": 105 }, "id": 117, "interval": "5", @@ -4789,10 +4793,10 @@ "current": { "selected": true, "text": [ - "1691162724495" + "All" ], "value": [ - "1691162724495" + "$__all" ] }, "datasource": { @@ -4916,6 +4920,6 @@ "timezone": "", "title": "Official k6 Test Result", "uid": "01npcT44k", - "version": 1, + "version": 4, "weekStart": "" } \ No newline at end of file diff --git a/docker/grafana/provisioning/dashboards/dashboard-kafka.json b/docker/grafana/provisioning/dashboards/dashboard-kafka.json index 7be964e8..e3ace017 100644 --- a/docker/grafana/provisioning/dashboards/dashboard-kafka.json +++ b/docker/grafana/provisioning/dashboards/dashboard-kafka.json @@ -20,7 +20,6 @@ "fiscalYearStartMonth": 0, "gnetId": 7589, "graphTooltip": 0, - "id": 17, "links": [ { "asDropdown": false, @@ -37,6 +36,84 @@ ], "liveNow": false, "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "links": [], + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum by(topic) (kafka_topic_partitions{instance=\"$instance\",topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Partitions per Topic", + "type": "bargauge" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 21, + "title": "Per Second", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -100,7 +177,7 @@ "h": 10, "w": 24, "x": 0, - "y": 0 + "y": 8 }, "id": 14, "links": [], @@ -182,6 +259,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -200,12 +278,12 @@ "overrides": [] }, "gridPos": { - "h": 9, + "h": 10, "w": 24, "x": 0, - "y": 10 + "y": 18 }, - "id": 16, + "id": 23, "links": [], "options": { "legend": { @@ -234,14 +312,16 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(delta(kafka_topic_partition_current_offset{instance=~'$instance', topic=~\"$topic\"}[5m])/5) by (topic)", + "editorMode": "code", + "expr": "sum(rate(kafka_topic_partition_current_offset{instance=\"$instance\", topic=~\"$topic\"}[1m])) by (topic, partition)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "A" + "legendFormat": "{{topic}} [{{partition}}]", + "range": true, + "refId": "B" } ], - "title": "Message in per minute", + "title": "Message in per second by partition", "type": "timeseries" }, { @@ -307,7 +387,7 @@ "h": 9, "w": 24, "x": 0, - "y": 19 + "y": 28 }, "id": 12, "links": [], @@ -337,6 +417,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "expr": "sum(kafka_consumergroup_lag{instance=\"$instance\",topic=~\"$topic\"}) by (consumergroup, topic) ", "format": "time_series", "instant": false, @@ -390,6 +471,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -411,7 +493,112 @@ "h": 9, "w": 24, "x": 0, - "y": 28 + "y": 37 + }, + "id": 19, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(kafka_consumergroup_lag{instance=\"$instance\",topic=~\"$topic\"}) by (consumergroup, topic, partition) ", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}} (topic: {{topic}}) [{{partition}}]", + "refId": "A" + } + ], + "title": "Lag by Consumer Group by Partition", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 46 }, "id": 18, "links": [], @@ -441,14 +628,16 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(delta(kafka_consumergroup_current_offset{instance=~'$instance',topic=~\"$topic\"}[5m])/5) by (consumergroup, topic)", + "editorMode": "code", + "expr": "sum(delta(kafka_consumergroup_current_offset{instance=~'$instance',topic=~\"$topic\"}[1m])) by (consumergroup, topic)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{consumergroup}} (topic: {{topic}})", + "range": true, "refId": "A" } ], - "title": "Message consume per minute", + "title": "Message consume per second", "type": "timeseries" }, { @@ -459,7 +648,37 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -474,32 +693,37 @@ "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 9, "w": 24, "x": 0, - "y": 37 + "y": 55 }, - "id": 8, + "id": 24, "links": [], "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "auto", - "reduceOptions": { + "legend": { "calcs": [ + "min", + "mean", + "max", "lastNotNull" ], - "fields": "", - "values": false + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, - "showUnfilled": true, - "valueMode": "color" + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "pluginVersion": "10.0.2", "targets": [ @@ -508,18 +732,237 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum by(topic) (kafka_topic_partitions{instance=\"$instance\",topic=~\"$topic\"})", + "editorMode": "code", + "expr": "sum(delta(kafka_consumergroup_current_offset{instance=~'$instance',topic=~\"$topic\"}[1m])) by (consumergroup, topic, partition)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{topic}}", + "legendFormat": "{{consumergroup}} (topic: {{topic}}) [{{partition}}]", + "range": true, "refId": "A" } ], - "title": "Partitions per Topic", - "type": "bargauge" + "title": "Message consume per second by partition", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 20, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 16, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true, + "width": 480 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(delta(kafka_topic_partition_current_offset{instance=~'$instance', topic=~\"$topic\"}[5m])/5) by (topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Message in per minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 22, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(delta(kafka_consumergroup_current_offset{instance=~'$instance',topic=~\"$topic\"}[5m])/5) by (consumergroup, topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}} (topic: {{topic}})", + "refId": "A" + } + ], + "title": "Message consume per minute", + "type": "timeseries" + } + ], + "title": "Per Minute", + "type": "row" } ], - "refresh": "", + "refresh": "5s", "schemaVersion": 38, "style": "dark", "tags": [ @@ -529,7 +972,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "mojaloop-services", "value": "mojaloop-services" }, @@ -556,7 +999,7 @@ }, { "current": { - "selected": true, + "selected": false, "text": "kafka-exporter:9308", "value": "kafka-exporter:9308" }, @@ -614,7 +1057,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-5m", "to": "now" }, "timepicker": { @@ -645,6 +1088,6 @@ "timezone": "browser", "title": "Kafka Exporter Overview", "uid": "jwPKIsniz", - "version": 1, + "version": 2, "weekStart": "" -} +} \ No newline at end of file diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 9e43882d..1559448a 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -84,6 +84,10 @@ scrape_configs: 'ml-core-central-handler-position-2:3001', 'ml-core-central-handler-position-3:3001', 'ml-core-central-handler-position-4:3001', + 'ml-core-central-handler-position-5:3001', + 'ml-core-central-handler-position-6:3001', + 'ml-core-central-handler-position-7:3001', + 'ml-core-central-handler-position-8:3001', 'ml-core-central-handler-fulfil-1:3001', 'ml-core-central-handler-fulfil-2:3001', 'ml-core-central-handler-fulfil-3:3001', @@ -95,7 +99,7 @@ scrape_configs: 'ml-core-central-handler-timeout-1:3001', 'ml-core-central-handler-timeout-2:3001', 'ml-core-central-handler-timeout-3:3001', - 'ml-core-central-handler-timeout-3:3001', + 'ml-core-central-handler-timeout-4:3001', # 'central-handler-get:3001', 'kafka-exporter:9308', ] diff --git a/envs/perffsp1.env b/envs/perffsp1.env index a2bff7b7..1518cee4 100644 --- a/envs/perffsp1.env +++ b/envs/perffsp1.env @@ -8,3 +8,4 @@ CBH_FSPIOP_ALS_ENDPOINT_URL=http://account-lookup-service:4002 # CBH_FSPIOP_ALS_ENDPOINT_URL=http://sim-perffsp1:3001/fspiop CBH_FSPIOP_TRANSFERS_ENDPOINT_URL=http://ml-api-adapter:3000 +# CBH_FSPIOP_TRANSFERS_ENDPOINT_URL=http://sim-perffsp2:3001/fspiop \ No newline at end of file diff --git a/envs/perffsp2.env b/envs/perffsp2.env index 446d7c19..9b20f1a6 100644 --- a/envs/perffsp2.env +++ b/envs/perffsp2.env @@ -8,3 +8,4 @@ CBH_FSPIOP_ALS_ENDPOINT_URL=http://account-lookup-service:4002 # CBH_FSPIOP_ALS_ENDPOINT_URL=http://sim-perffsp4:3001/fspiop CBH_FSPIOP_TRANSFERS_ENDPOINT_URL=http://ml-api-adapter:3000 +# CBH_FSPIOP_TRANSFERS_ENDPOINT_URL=http://sim-perffsp1:3001/fspiop diff --git a/packages/k6-tests/common/utils.js b/packages/k6-tests/common/utils.js index 2baf5aab..ea6f2c2c 100644 --- a/packages/k6-tests/common/utils.js +++ b/packages/k6-tests/common/utils.js @@ -1,4 +1,5 @@ import { check, fail } from "k6"; +import { randomItem } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js'; export function checkStatus({ response, expectedStatus, expectedContent, failOnError, printOnError, dynamicIds }) { if (isEmpty(expectedStatus) && isEmpty(expectedContent)) { @@ -54,3 +55,18 @@ function isEmpty(str) { return (!str || str.length === 0); } +export function getTwoItemsFromArray(inputArray) { + + // Option 1 + // const outputArray = inputArray.concat().sort(() => randomItem([-1,1])).slice(0, 2); + + // Option 2 + const tempArray = [ ...inputArray ] + const outputArray = [] + const selectedItem1 = randomItem(tempArray) + outputArray.push(selectedItem1) + tempArray.splice(tempArray.indexOf(selectedItem1), 1); + outputArray.push(randomItem(tempArray)) + + return outputArray +} \ No newline at end of file diff --git a/packages/k6-tests/config/fspiopTransfers.json b/packages/k6-tests/config/fspiopTransfers.json index c149f175..84e5dc57 100644 --- a/packages/k6-tests/config/fspiopTransfers.json +++ b/packages/k6-tests/config/fspiopTransfers.json @@ -8,8 +8,8 @@ }, "startVUs": 1, "stages": [ - { "duration": "2m", "target": 1 }, - { "duration": "5m", "target": 1 } + { "duration": "30s", "target": 12 }, + { "duration": "5m", "target": 12 } ] } }, diff --git a/packages/k6-tests/config/fspiopTransfersUnidirectional.json b/packages/k6-tests/config/fspiopTransfersUnidirectional.json index fda00227..effb8b85 100644 --- a/packages/k6-tests/config/fspiopTransfersUnidirectional.json +++ b/packages/k6-tests/config/fspiopTransfersUnidirectional.json @@ -8,8 +8,8 @@ }, "startVUs": 1, "stages": [ - { "duration": "2m", "target": 1 }, - { "duration": "5m", "target": 1 } + { "duration": "30s", "target": 10 }, + { "duration": "5m", "target": 10 } ] } }, diff --git a/packages/k6-tests/scripts/postTransfers.js b/packages/k6-tests/scripts/postTransfers.js index be97d3c1..f4cb8e64 100644 --- a/packages/k6-tests/scripts/postTransfers.js +++ b/packages/k6-tests/scripts/postTransfers.js @@ -4,7 +4,7 @@ import { check, fail, sleep, group } from 'k6'; import { WebSocket } from 'k6/experimental/websockets'; import { setTimeout, clearTimeout, setInterval, clearInterval } from 'k6/experimental/timers'; import { Trace } from "../common/trace.js"; -import { randomItem } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js'; +import { getTwoItemsFromArray } from "../common/utils.js"; console.log(`Env Vars --> K6_SCRIPT_WS_TIMEOUT_MS=${__ENV.K6_SCRIPT_WS_TIMEOUT_MS}, @@ -28,9 +28,9 @@ export function postTransfers() { payerFsp = fspList[0] payeeFsp = fspList[1] } else { - const randomSortedFsp = fspList.concat().sort(() => randomItem([-1,1])).slice(0, 2); - payerFsp = randomSortedFsp[0] - payeeFsp = randomSortedFsp[1] + const selectedFsps = getTwoItemsFromArray(fspList) + payerFsp = selectedFsps[0] + payeeFsp = selectedFsps[1] } const startTs = Date.now(); @@ -81,6 +81,7 @@ export function postTransfers() { 'Accept': 'application/vnd.interoperability.transfers+json;version=1.1', 'Content-Type': 'application/vnd.interoperability.transfers+json;version=1.1', 'FSPIOP-Source': payerFspId, + 'FSPIOP-Destination': payeeFspId, 'Date': (new Date()).toUTCString(), 'traceparent': traceParent.toString(), 'tracestate': `tx_end2end_start_ts=${startTs}` diff --git a/perf.env b/perf.env index 8a9a5959..39eec489 100644 --- a/perf.env +++ b/perf.env @@ -30,13 +30,19 @@ K6_SCRIPT_FSPIOP_ALS_ENDPOINT_URL=http://account-lookup-service:4002 # ] ## IMPORTANT: Update pool depending 2dfsp, 4dfsp or 8dfsp docker profile ## NOTE: Unidirectional will take the first two DFSP's in the pool -K6_SCRIPT_FSPIOP_FSP_POOL='[{"partyId":19012345001,"fspId":"perffsp1","wsUrl":"ws://sim-perffsp1:3002"},{"partyId":19012345002,"fspId":"perffsp2","wsUrl":"ws://sim-perffsp2:3002"},{"partyId":19012345003,"fspId":"perffsp3","wsUrl":"ws://sim-perffsp3:3002"},{"partyId":19012345004,"fspId":"perffsp4","wsUrl":"ws://sim-perffsp4:3002"},{"partyId":19012345005,"fspId":"perffsp5","wsUrl":"ws://sim-perffsp5:3002"},{"partyId":19012345006,"fspId":"perffsp6","wsUrl":"ws://sim-perffsp6:3002"},{"partyId":19012345007,"fspId":"perffsp7","wsUrl":"ws://sim-perffsp7:3002"},{"partyId":19012345008,"fspId":"perffsp8","wsUrl":"ws://sim-perffsp8:3002"}]' +# 8 DFSPs +#K6_SCRIPT_FSPIOP_FSP_POOL='[{"partyId":19012345001,"fspId":"perffsp1","wsUrl":"ws://sim-perffsp1:3002"},{"partyId":19012345002,"fspId":"perffsp2","wsUrl":"ws://sim-perffsp2:3002"},{"partyId":19012345003,"fspId":"perffsp3","wsUrl":"ws://sim-perffsp3:3002"},{"partyId":19012345004,"fspId":"perffsp4","wsUrl":"ws://sim-perffsp4:3002"},{"partyId":19012345005,"fspId":"perffsp5","wsUrl":"ws://sim-perffsp5:3002"},{"partyId":19012345006,"fspId":"perffsp6","wsUrl":"ws://sim-perffsp6:3002"},{"partyId":19012345007,"fspId":"perffsp7","wsUrl":"ws://sim-perffsp7:3002"},{"partyId":19012345008,"fspId":"perffsp8","wsUrl":"ws://sim-perffsp8:3002"}]' +# 2 DFSPs +#K6_SCRIPT_FSPIOP_FSP_POOL='[{"partyId":19012345001,"fspId":"perffsp1","wsUrl":"ws://sim-perffsp1:3002"},{"partyId":19012345002,"fspId":"perffsp2","wsUrl":"ws://sim-perffsp2:3002"}]' +# 4 DFSPs +K6_SCRIPT_FSPIOP_FSP_POOL='[{"partyId":19012345001,"fspId":"perffsp1","wsUrl":"ws://sim-perffsp1:3002"},{"partyId":19012345002,"fspId":"perffsp2","wsUrl":"ws://sim-perffsp2:3002"},{"partyId":19012345003,"fspId":"perffsp3","wsUrl":"ws://sim-perffsp3:3002"},{"partyId":19012345004,"fspId":"perffsp4","wsUrl":"ws://sim-perffsp4:3002"}]' K6_SCRIPT_ADMIN_ENDPOINT_URL=http://callback-handler-svc-cl-sim:3001/admin K6_SCRIPT_ORACLE_ENDPOINT_URL=http://callback-handler-svc-oracle-sim:3001/oracle K6_SCRIPT_FSPIOP_TRANSFERS_ENDPOINT_URL=http://ml-api-adapter:3000 +# K6_SCRIPT_FSPIOP_TRANSFERS_ENDPOINT_URL=http://sim-perffsp2:3001/fspiop K6_SCRIPT_FSPIOP_TRANSFERS_ILPPACKET='AYIDGQAAAAAAACcQIWcuZ3JlZW5iYW5rZnNwLm1zaXNkbi4yNzcxMzgwMzkxMoIC62V5SjBjbUZ1YzJGamRHbHZia2xrSWpvaU1ERXhaR1EyTldZdE5UQXpNeTAwTVdNMkxUazFaR1l0T1RFeFl6WTRPVFExWWpobUlpd2ljWFZ2ZEdWSlpDSTZJbVF3TXpJMU1EVTJMVE0xTldFdE5EUmxNUzFpT1RnMExXWXdZVFExTmpFMFkyRXpPQ0lzSW5CaGVXVmxJanA3SW5CaGNuUjVTV1JKYm1adklqcDdJbkJoY25SNVNXUlVlWEJsSWpvaVRWTkpVMFJPSWl3aWNHRnlkSGxKWkdWdWRHbG1hV1Z5SWpvaU1qYzNNVE00TURNNU1USWlMQ0ptYzNCSlpDSTZJbWR5WldWdVltRnVhMlp6Y0NKOWZTd2ljR0Y1WlhJaU9uc2ljR0Z5ZEhsSlpFbHVabThpT25zaWNHRnlkSGxKWkZSNWNHVWlPaUpOVTBsVFJFNGlMQ0p3WVhKMGVVbGtaVzUwYVdacFpYSWlPaUkwTkRFeU16UTFOamM0T1NJc0ltWnpjRWxrSWpvaWNHbHVhMkpoYm10bWMzQWlmU3dpY0dWeWMyOXVZV3hKYm1adklqcDdJbU52YlhCc1pYaE9ZVzFsSWpwN0ltWnBjbk4wVG1GdFpTSTZJa1pwY25OMGJtRnRaUzFVWlhOMElpd2liR0Z6ZEU1aGJXVWlPaUpNWVhOMGJtRnRaUzFVWlhOMEluMHNJbVJoZEdWUFprSnBjblJvSWpvaU1UazROQzB3TVMwd01TSjlmU3dpWVcxdmRXNTBJanA3SW1OMWNuSmxibU41SWpvaVZWTkVJaXdpWVcxdmRXNTBJam9pTVRBd0luMHNJblJ5WVc1ellXTjBhVzl1Vkhsd1pTSTZleUp6WTJWdVlYSnBieUk2SWxSU1FVNVRSa1ZTSWl3aWFXNXBkR2xoZEc5eUlqb2lVRUZaUlZJaUxDSnBibWwwYVdGMGIzSlVlWEJsSWpvaVEwOU9VMVZOUlZJaWZYMAA' K6_SCRIPT_FSPIOP_TRANSFERS_CONDITION='5m0gq_5dLQlTSSRKQmLpj0MZ1MtWLWgSu1oLGVTJyYs' @@ -44,3 +50,5 @@ K6_SCRIPT_FSPIOP_TRANSFERS_AMOUNT=2 K6_SCRIPT_FSPIOP_TRANSFERS_CURRENCY=USD K6_SCRIPT_CONFIG_FILE_NAME=test.json + +KAFKA_PARTITIONS_NUM=17