feat(mojaloop/#3398): performance char for als setup test-cases

feat(mojaloop/#3398): performance characterisation for ALS Setup test-cases - mojaloop/project#3398
- Added Docker Composers for
    - Perf <-- start-up services lean and mean for performance characterization purposes
    - Monitoring <-- provides monitoring stack for Prometheus, Grafana and exporters for Host, Docker, MySQL and Kafka, with associated Dashboards
    - Load <-- run load tests using K6
- Added new "Callback Handler" Simulator to provide static processing of Payee requests & Payee callbacks, with WS notifications hook for K6 iteration runners.
- Added Grafana Dashboards for the following:
    - Account-lookup-service
    - ML-API-Adapter
    - Central-Ledger
    - NodeJS Applications
    - K6 Test Runner
    - Docker Host (NodeExporter & CAdviser)
    - NodeJS
    - MySQL

.env

@@ -1,4 +1,22 @@
+## Mojaloop Docker Image Versions
+
 ML_API_ADAPTER_VERSION=v14.0.0
-ACCOUNT_LOOKUP_SERVICE_VERSION=v14.1.0
+## ALS initial baseline
+# ACCOUNT_LOOKUP_SERVICE_VERSION=v14.2.2
+## ALS published version with included fix: JSON.stringify disabled in logResponse function
+# ACCOUNT_LOOKUP_SERVICE_VERSION=v14.2.3
+## ALS with local fix: v14.2.3 + caching for validateParticipant and resolve Participants via Oracles
+ACCOUNT_LOOKUP_SERVICE_VERSION=v14.2.4-snapshot.3
 QUOTING_SERVICE_VERSION=v15.0.2
-CENTRAL_LEDGER_VERSION=v17.0.3
+CENTRAL_LEDGER_VERSION=v17.0.3
+
+
+## Monitoring Docker Image Versions
+GRAFANA_VERSION=10.0.2
+GRAFANA_RENDERER_VERSION=3.7.1
+PROMETHEUS_VERSION=v2.45.0
+NODE_EXPORTER_VERSION=latest
+CADVISER_VERSION=latest
+
+## Load Docker Image Versions
+K6_VERSION=0.45.0

.gitignore

@@ -0,0 +1,3 @@
+
+# General Ignore
+**/*IGNORE*

README.md

@@ -155,7 +155,7 @@ You can use this repo to run functional tests inside the CICD of a core service
 
 The following commands can be added to the CICD pipeline
 
-```
+```bash
 git clone --depth 1 --branch v0.0.2 https://github.com/mojaloop/ml-core-test-harness.git
 cd ml-core-test-harness
 
@@ -167,3 +167,61 @@ cat ttk-tests-console.log
 ls reports/ttk-func-tests-report.html reports/ttk-provisioning-report.html
 ```
 
+## Performance Characterization
+
+### Running ALS with dependencies
+
+```bash
+docker compose --project-name ml-core -f docker-compose-perf.yml --profile all-services --profile ttk-provisioning up -d
+```
+
+Stop Services
+
+```bash
+docker compose --project-name ml-core -f docker-compose-perf.yml --profile all-services down -v
+```
+
+> NOTE: `-v` argument is optional, and it will delete any volume data created by the monitoring docker compose
+
+### Monitoring
+
+Start Monitoring Services stack which uses:
+- [Prometheus](https://prometheus.io) for time series data store
+- [Grafana](https://grafana.com/) for visualization dashboards
+- [Node Exporter](https://github.com/prometheus/node_exporter) to instrument the Host machine
+- [CAdviser](https://github.com/google/cadvisor) to instrument the Docker containers running on Host machine
+
+```bash
+docker compose --project-name monitoring -f docker-compose-monitoring.yml up -d
+```
+
+Stop Monitoring Services
+
+```bash
+docker compose --project-name monitoring -f docker-compose-monitoring.yml down -v
+```
+
+> NOTE: `-v` argument is optional, and it will delete any volume data created by the monitoring docker compose
+
+TODO:
+- add note about network being created by docker-compose-perf.yml, or it can be done manually.
+
+### Load Tests
+
+[K6](https://k6.io) is being used to execute performance tests, with metrics being captured by [Prometheus](https://k6.io/docs/results-output/real-time/prometheus-remote-write) and displayed using [Grafana](https://k6.io/docs/results-output/real-time/prometheus-remote-write/#time-series-visualization).
+
+Tests can be defined in the [./docker/k6/scripts/test.js](./docker/k6/scripts/test.js), refer to [API load testing guide](https://k6.io/docs/testing-guides/api-load-testing/) for more information.
+
+Env configs are stored in the [./docker/k6/.env](./docker/k6/.env) environment configuration file, which can be referenced in by the [./docker/k6/scripts/test.js](./docker/k6/scripts/test.js).
+
+Start tests
+
+```bash
+docker compose --project-name load -f docker-compose-load.yml up
+```
+
+Cleanup tests
+
+```bash
+docker compose --project-name load -f docker-compose-load.yml down -v
+```

docker-compose-load.yml

@@ -0,0 +1,38 @@
+## Ref: https://github.com/grafana/xk6-output-prometheus-remote/blob/main/docker-compose.yml
+## Ref: https://k6.io/docs/results-output/real-time/prometheus-remote-write/
+
+version: '3.8'
+
+networks:
+  load-net:
+    name: load-net
+  monitoring-net:
+    name: monitoring-net
+    external: true
+  mojaloop-net:
+    name: mojaloop-net
+    external: true
+
+services:
+  k6:
+    image: grafana/k6:${K6_VERSION}
+    env_file:
+      - .env
+      - perf.env
+    command:
+      - run
+      # - '--help' ## Useful to list run options
+      - '--include-system-env-vars'
+      - '/scripts/index.js'
+    networks:
+      - load-net
+      - monitoring-net
+      - mojaloop-net
+    ports:
+      - "6565:6565"
+    environment: {}
+      # - K6_PROMETHEUS_RW_SERVER_URL=http://prometheus:9090/api/v1/write # Ref: https://k6.io/docs/results-output/real-time/prometheus-remote-write/#options
+      # - K6_PROMETHEUS_RW_TREND_AS_NATIVE_HISTOGRAM=true # Ref: https://k6.io/docs/results-output/real-time/prometheus-remote-write/#options
+      # - K6_OUT=experimental-prometheus-rw # Ref: https://k6.io/docs/results-output/real-time/prometheus-remote-write/
+    volumes:
+      - ./packages/k6-tests:/scripts

docker-compose-monitoring.yml

@@ -0,0 +1,162 @@
+version: "3.7"
+
+networks:
+  monitoring-net:
+    name: monitoring-net
+  mojaloop-net:
+    name: mojaloop-net
+    external: true
+
+volumes:
+    vol_prometheus_data: {}
+    vol_grafana_data: {}
+
+services:
+
+  prometheus:
+    image: prom/prometheus:${PROMETHEUS_VERSION}
+    volumes:
+      - ./docker/prometheus/:/etc/prometheus/
+      - vol_prometheus_data:/prometheus/
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--enable-feature=native-histograms' # Ref: https://prometheus.io/docs/concepts/metric_types/#histogram, https://k6.io/docs/results-output/real-time/prometheus-remote-write/#send-test-metrics-to-a-remote-write-endpoint
+      - '--web.enable-remote-write-receiver' # Ref: https://prometheus.io/docs/prometheus/latest/feature_flags/#remote-write-receiver
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+    ports:
+      - 9090:9090
+    links:
+      - cadvisor:cadvisor
+      #- alertmanager:alertmanager
+      #- pushgateway:pushgateway
+    depends_on:
+      - cadvisor
+      #- pushgateway
+    networks:
+      - mojaloop-net
+      - monitoring-net
+    restart: always
+    # deploy:
+    #   placement:
+    #     constraints:
+    #       - node.hostname == ${HOSTNAME}
+
+  node-exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /:/host:ro,rslave
+    command:
+      - '--path.rootfs=/host'
+      - '--path.procfs=/host/proc'
+      - '--path.sysfs=/host/sys'
+      - --collector.filesystem.ignored-mount-points
+      - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)"
+    ports:
+      - 9100:9100
+    networks:
+      - mojaloop-net
+      - monitoring-net
+    restart: always
+    deploy:
+      mode: global
+
+  # alertmanager:
+  #   image: prom/alertmanager
+  #   ports:
+  #     - 9093:9093
+  #   volumes:
+  #     - ./docker/alertmanager/:/etc/alertmanager/
+  #   networks:
+  #     - mojaloop-net
+  #     - monitoring-net
+  #   restart: always
+  #   command:
+  #     - '--config.file=/etc/alertmanager/config.yml'
+  #     - '--storage.path=/alertmanager'
+  #   # deploy:
+  #   #   placement:
+  #   #     constraints:
+  #   #       - node.hostname == ${HOSTNAME}
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:${CADVISER_VERSION}
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    ports:
+      - 8080:8080
+    networks:
+      - mojaloop-net
+      - monitoring-net
+    restart: always
+    deploy:
+      mode: global
+
+  grafana:
+    image: grafana/grafana:${GRAFANA_VERSION}
+    user: "472"
+    depends_on:
+      - prometheus
+    ports:
+      - 9999:3000
+    volumes:
+      - vol_grafana_data:/var/lib/grafana
+      - ./docker/grafana/provisioning/:/etc/grafana/provisioning/
+    env_file:
+      - ./docker/grafana/config.env
+    environment:
+      GF_RENDERING_SERVER_URL: http://renderer:8081/render
+      GF_RENDERING_CALLBACK_URL: http://grafana:3000/
+      GF_LOG_FILTERS: rendering:debug
+    networks:
+      - mojaloop-net
+      - monitoring-net
+    restart: always
+
+  ## Ref: https://github.com/grafana/grafana-image-renderer
+  renderer:
+    image: grafana/grafana-image-renderer:${GRAFANA_RENDERER_VERSION}
+    ports:
+      - 8081:8081
+    environment:
+      # ENABLE_METRICS: 'true'
+      # VIRTUAL_HOST: renderer.loc
+      HTTP_HOST: 0.0.0.0
+      # RENDERING_MODE: ${RENDERING_MODE}
+      # RENDERING_CLUSTERING_MODE: ${RENDERING_CLUSTERING_MODE}
+      # RENDERING_CLUSTERING_MAX_CONCURRENCY: ${RENDERING_CLUSTERING_MAX_CONCURRENCY}
+      # RENDERING_ARGS: --no-sandbox,--disable-setuid-sandbox,--disable-dev-shm-usage,--disable-accelerated-2d-canvas,--disable-gpu,--window-size=8000x8000
+    networks:
+      - monitoring-net
+    volumes:
+      - ./docker/renderer/config.json:/usr/src/app/config.json
+
+  mysqlexporter-als:
+    image: prom/mysqld-exporter:v0.15.0
+    # ports:
+    # - "9104:9104"
+    networks:
+      - mojaloop-net
+      - monitoring-net
+    environment:
+    - MYSQLD_EXPORTER_PASSWORD=password
+    command:
+      - --mysqld.address=mysql-als:3306
+      - --mysqld.username=exporter
+
+  kafka-exporter:
+    image: danielqsj/kafka-exporter:latest
+    ports:
+      - "9308:9308"
+    networks:
+      - mojaloop-net
+      - monitoring-net
+    command:
+      - --kafka.server=kafka:9092