Merge branch 'main' of https://github.com/mojaloop/ml-core-test-harness…

… into feat/support-iac-vm

.env

@@ -6,11 +6,13 @@ ML_API_ADAPTER_VERSION=v14.0.1
 ACCOUNT_LOOKUP_SERVICE_VERSION=v14.2.3
 ## ALS snapshot release with fix: v14.2.3 + caching for validateParticipant and resolve Participants via Oracles
 # ACCOUNT_LOOKUP_SERVICE_VERSION=v14.2.4-snapshot.3
-QUOTING_SERVICE_VERSION=v15.0.2
+QUOTING_SERVICE_VERSION=v15.1.0
 ## CL initial baseline with version included in Mojaloop v15.1.0 Helm Release
-CENTRAL_LEDGER_VERSION=v17.0.3
+CENTRAL_LEDGER_VERSION=v17.2.0
 ## CL snapshot release with included fix: JSON.stringify disabled in logResponse function
 # CENTRAL_LEDGER_VERSION=v17.0.4-snapshot.0
+## Central Ledger version with batch processing capability
+CENTRAL_LEDGER_BATCH_VERSION=v17.3.0-snapshot.5
 
 ## Testing & Sims
 TEST_SIMULATOR_VERSION=v11.1.3
@@ -38,8 +40,10 @@ CADVISER_VERSION=latest
 K6_VERSION=0.45.0
 
 ## Performance Testing Replicas
+QS_REPLICAS=1
 ALS_REPLICAS=1
 CENTRAL_LEDGER_GENERAL_REPLICAS=4
 CENTRAL_LEDGER_POSITION_REPLICAS=8
+CENTRAL_LEDGER_POSITION_BATCH_REPLICAS=0
 ML_API_ADAPTER_REPLICAS=4
 ML_NOTIFICATION_REPLICAS=4

README.md

@@ -211,6 +211,26 @@ docker compose --project-name ml-core -f docker-compose-perf.yml --profile trans
 
 > NOTE: `-v` argument is optional, and it will delete any volume data created by the monitoring docker compose
 
+
+### Running Services for Quotes characterization
+
+```bash
+docker compose --project-name ml-core -f docker-compose-perf.yml --profile quotes-test --profile 8dfsp --profile ttk-provisioning-quotes up -d
+```
+
+Stop Services
+
+```bash
+docker compose --project-name ml-core -f docker-compose-perf.yml --profile quotes-test --profile 8dfsp down -v
+```
+
+> NOTE: `-v` argument is optional, and it will delete any volume data created by the monitoring docker compose
+
+### Configuration for Transfers with batch support
+- Set CENTRAL_LEDGER_POSITION_BATCH_REPLICAS to desired count in `.env` file
+- Enable line `CLEDG_KAFKA__EVENT_TYPE_ACTION_TOPIC_MAP__POSITION__PREPARE=topic-transfer-position-batch` in `perf.env` file
+- Set `CENTRAL_LEDGER_VERSION` to `v17.2.0` or higher
+
 ### Monitoring
 
 Start Monitoring Services stack which uses:
@@ -242,6 +262,14 @@ Start monitoring with central ledger mysql exporter
 docker compose --project-name monitoring --profile transfers-test -f docker-compose-monitoring.yml up -d
 ```
 
+or
+
+```bash
+docker compose --project-name monitoring --profile quotes-test -f docker-compose-monitoring.yml up -d
+```
+
+since the quoting service uses the central ledger database.
+
 > NOTE: `-v` argument is optional, and it will delete any volume data created by the monitoring docker compose
 
 TODO:
@@ -256,7 +284,7 @@ Tests can be defined in the [./packages/k6-tests/scripts/test.js](./packages/k6-
 
 Env configs are stored in the [./perf.env](./perf.env) environment configuration file..
 
-Note: Transfer testing
+Note: Transfer testing and quote testing
 
 Depending on the profile you started the performance docker compose with i.e `--profile transfers-test --profile {2/4/8}dfsp`
 You will need to edit `K6_SCRIPT_FSPIOP_FSP_POOL` json string in `./perf.env` to contain 2/4/8 dfsps depending on your test.
@@ -283,6 +311,8 @@ env K6_SCRIPT_CONFIG_FILE_NAME=fspiopTransfers.json docker compose --project-nam
 env K6_SCRIPT_CONFIG_FILE_NAME=fspiopTransfersUnidirectional.json docker compose --project-name load -f docker-compose-load.yml up
 ( or )
 env K6_SCRIPT_CONFIG_FILE_NAME=fspiopDiscovery.json docker compose --project-name load -f docker-compose-load.yml up
+( or )
+env K6_SCRIPT_CONFIG_FILE_NAME=fspiopQuotes.json docker compose --project-name load -f docker-compose-load.yml up
 ```
 
 Cleanup tests

docker-compose-monitoring.yml

@@ -169,6 +169,7 @@ services:
       - --mysqld.username=exporter
     profiles:
       - transfers-test
+      - quotes-test
 
   kafka-exporter:
     image: danielqsj/kafka-exporter:${KAFKA_EXPORTER_VERSION}

docker-compose-perf.yml

@@ -118,8 +118,6 @@ services:
     depends_on:
       mysql-cl:
         condition: service_healthy
-      kafka:
-        condition: service_started
       # objstore:
       #   condition: service_started
     user: root
@@ -130,6 +128,7 @@ services:
       - agreement
       - all-services
       - transfers-test
+      - quotes-test
 
   central-ledger:
     image: mojaloop/central-ledger:${CENTRAL_LEDGER_VERSION}
@@ -177,6 +176,7 @@ services:
       - agreement
       - all-services
       - transfers-test
+      - quotes-test
 
   central-handler-prepare:
     image: mojaloop/central-ledger:${CENTRAL_LEDGER_VERSION}
@@ -266,6 +266,46 @@ services:
       - all-services
       - transfers-test
 
+  central-handler-position-batch:
+    image: mojaloop/central-ledger:${CENTRAL_LEDGER_BATCH_VERSION}
+    env_file:
+      - .env
+      - perf.env
+    deploy:
+      mode: replicated
+      replicas: ${CENTRAL_LEDGER_POSITION_BATCH_REPLICAS}
+    command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/central-handlers.js /opt/app/config/default.json && /opt/app/wait4/wait4.js central-ledger && node src/handlers/index.js handler --positionbatch"
+    # ports:
+    #   - "3001:3001"
+    volumes:
+       - ./docker/wait4:/opt/app/wait4
+       - ./docker/config-modifier:/opt/app/config-modifier
+    environment:
+      # - LOG_LEVEL=error
+      - CLEDG_MONGODB__DISABLED=true
+      - EVENT_SDK_TRACEID_PER_VENDOR=false
+    networks:
+      - mojaloop-net
+    depends_on:
+      central-ledger-migrator:
+        condition: service_completed_successfully
+      mysql-cl:
+        condition: service_healthy
+      kafka:
+        condition: service_started
+    healthcheck:
+      test: wget -q http://localhost:3001/health -O /dev/null || exit 1
+      timeout: 20s
+      retries: 30
+      interval: 30s
+      start_period: 30s
+    user: root
+    profiles:
+      - central-ledger
+      - transfer
+      - all-services
+      - transfers-test
+
   central-handler-fulfil:
     image: mojaloop/central-ledger:${CENTRAL_LEDGER_VERSION}
     env_file:
@@ -431,6 +471,35 @@ services:
       - all-services
       - transfers-test
 
+  quoting-service:
+    image: "mojaloop/quoting-service:${QUOTING_SERVICE_VERSION}"
+    command: sh -c "/opt/app/config-modifier/run.js /opt/app/config/default.json /opt/app/config-modifier/configs/quoting-service-perf.js /opt/app/config/default.json && node src/index.js"
+    deploy:
+      mode: replicated
+      replicas: ${QS_REPLICAS}
+    networks:
+      - mojaloop-net
+    # ports:
+    #   - "3002:3002"
+    depends_on:
+      mysql-cl:
+        condition: service_started
+      central-ledger:
+        condition: service_started
+    environment:
+      ## Allows TraceID to be propogated - https://github.com/mojaloop/event-sdk#configuration
+      - EVENT_SDK_TRACEID_PER_VENDOR=false
+    volumes:
+      - ./docker/config-modifier:/opt/app/config-modifier
+    healthcheck:
+      test: wget -q http://localhost:3002/health -O /dev/null || exit 1
+      timeout: 20s
+      retries: 30
+      interval: 15s
+    user: root
+    profiles:
+      - quotes-test
+
   # objstore:
   #   image: mongo:${DEP_MONGO_VERSION}
   #   container_name: objstore
@@ -469,6 +538,7 @@ services:
       echo -e 'Creating kafka topics'
       kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-prepare --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM
       kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-position --replication-factor 1 --partitions $$KAFKA_POSITION_PARTITIONS_NUM
+      kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-position-batch --replication-factor 1 --partitions $$KAFKA_POSITION_BATCH_PARTITIONS_NUM
       kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-fulfil --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM
       kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-notification-event --replication-factor 1 --partitions $$KAFKA_NOTIFICATION_PARTITIONS_NUM
       kafka-topics.sh --bootstrap-server kafka:29092 --create --if-not-exists --topic topic-transfer-get --replication-factor 1 --partitions $$KAFKA_GENERAL_PARTITIONS_NUM
@@ -523,6 +593,7 @@ services:
       - agreement
       - all-services
       - transfers-test
+      - quotes-test
 
   mysql-cl:
     image: mysql/mysql-server:${DEP_MYSQL_VERSION}
@@ -554,6 +625,7 @@ services:
       - agreement
       - all-services
       - transfers-test
+      - quotes-test
     ## In memory storage - Ref: https://docs.docker.com/storage/tmpfs/
     tmpfs:
       - /var/lib/mysql
@@ -664,6 +736,7 @@ services:
       - all-services
       - als-test
       - transfers-test
+      - quotes-test
 
   mojaloop-testing-toolkit-ui:
     image: mojaloop/ml-testing-toolkit-ui:${TEST_TTK_UI_VERSION}
@@ -685,6 +758,7 @@ services:
       - all-services
       - als-test
       - transfers-test
+      - quotes-test
 
   ttk-provisioning-als:
     image: mojaloop/ml-testing-toolkit-client-lib:${TEST_TTK_CLI_VERSION}
@@ -771,6 +845,30 @@ services:
     profiles:
       - ttk-provisioning-remote-k8s
 
+  ttk-provisioning-quotes:
+    image: mojaloop/ml-testing-toolkit-client-lib:${TEST_TTK_CLI_VERSION}
+    env_file:
+      - .env
+      - perf.env
+    volumes:
+      - "./docker/ml-testing-toolkit/test-cases/collections:/opt/app/collections"
+      - "./docker/ml-testing-toolkit/test-cases/environments:/opt/app/environments"
+      - "./reports:/opt/app/reports"
+    depends_on:
+      mojaloop-testing-toolkit:
+        condition: service_healthy
+      central-ledger:
+        condition: service_healthy
+    networks:
+      - mojaloop-net
+    command:
+      - sh
+      - -c
+      - "npm run cli -- -u http://mojaloop-testing-toolkit:5050 -l 2 -i collections/provisioning -e environments/default-env.json --save-report --report-format html --report-target file://reports/ttk-provisioning-report.html"
+    user: root
+    profiles:
+      - ttk-provisioning-quotes
+
   callback-handler-svc-oracle-sim:
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}
     deploy:
@@ -837,6 +935,7 @@ services:
       - all-services
       - als-test
       - transfers-test
+      - quotes-test
 
   sim-perffsp1:
     image: mojaloop/callback-handler-svc:${TEST_CALLBACK_HAND_SVC_VERSION}

docker/config-modifier/configs/central-handlers.js

@@ -75,6 +75,16 @@ module.exports = {
               "metadata.broker.list": "kafka:29092"
             }
           }
+        },
+        "POSITION_BATCH": {
+          "config": {
+            "options": {
+              "batchSize": 50
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "kafka:29092"
+            }
+          }
         }
       },
       "ADMIN": {

docker/config-modifier/configs/quoting-service-perf.js

@@ -0,0 +1,8 @@
+module.exports = {
+  "HOSTNAME": "http://quoting-service",
+  "DATABASE": {
+    "HOST": "mysql-cl"
+  },
+  "SIMPLE_ROUTING_MODE": false,
+  "SWITCH_ENDPOINT": "http://callback-handler-svc-cl-sim:3001/admin"
+}