Use cli tool to run benchmarks instead of rake

.github/workflows/benchmarks.yml

@@ -27,79 +27,37 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y gconf-service libgbm-dev libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
-          libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
-          libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 \
-          libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \
-          libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libnss3 lsb-release xdg-utils wget \
-          libudev-dev cargo npm imagemagick libmagickwand-dev cmake jq
+        run: ./scripts/benchmarks/setup_dependencies.sh
 
       - name: Install Node.js
         uses: actions/setup-node@v4
         with:
           node-version: 22.4.0
 
       - name: Install Ruby and related tools
-        run: |
-          whoami
-          npm install -g corepack
-          corepack enable
-          curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
-          echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
-          sudo apt-get update && sudo apt-get install -y yarn software-properties-common
-          apt-add-repository -y ppa:rael-gc/rvm
-          sudo apt-get update && sudo apt-get install -y rvm
-          echo 'source "/etc/profile.d/rvm.sh"' >> ~/.bashrc
-          source /etc/profile.d/rvm.sh
-          rvm install ruby-3.1.2
-          ruby --version
-          sudo chown -R $(whoami) /usr/local
-          rvm use 3.1.2 --default
+        run: ./scripts/benchmarks/setup_dependencies.sh
 
       - name: Install cargo tools
         run: |
           cargo install nj-cli wasm-pack --locked
-
-      - name: Prepare environment
-        run: |
           npm install -g tslib
-          export PATH="/usr/share/rvm:$PATH"
-          gem install dotenv json octokit tmpdir fileutils
-
-      - name: Get PR details
-        if: github.event_name == 'workflow_dispatch'
-        uses: actions/github-script@v7
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          script: |
-            const fs = require('fs');
-            const pr = await github.rest.pulls.get({
-              owner: 'esrlabs',
-              repo: 'chipmunk',
-              pull_number: ${{ github.event.inputs.pr_id }}
-            });
-            fs.appendFileSync(process.env.GITHUB_ENV, `REPO_OWNER=${pr.data.user.login}\n`);
-            fs.appendFileSync(process.env.GITHUB_ENV, `REPO_NAME=${pr.data.base.repo.name}\n`);
 
       - name: Run Jasmine performance tests
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
           export PATH="/root/.cargo/bin:$PATH"
           source ~/.bashrc
-          if [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then
-            ruby scripts/tools/run_benchmarks.rb PR~${{ github.event.inputs.pr_id }}
-          else
-            export REPO_OWNER='esrlabs'
-            export REPO_NAME='chipmunk'
-            ruby scripts/tools/run_benchmarks.rb 1
+          if [[ "${{ github.event_name }}" == 'workflow_dispatch' && "${{ github.event.inputs.pr_id }}" =~ ^[0-9]+$ ]]; then
+            ruby scripts/benchmarks/process.rb PR~${{ github.event.inputs.pr_id }}
+          elif [[ "${{ github.event_name }}" == 'workflow_dispatch' && "${{ github.event.inputs.pr_id }}" -eq 0 ]]; then
+            echo "Running manually"
+            ruby scripts/benchmarks/process.rb 1
+          elif [[ "${{ github.event_name }}" != 'workflow_dispatch' ]]; then
+            ruby scripts/benchmarks/process.rb 1
           fi
 
       - name: List files in the results folder
-        run: |
-          ls -la /chipmunk/chipmunk_performance_results
+        run: ls -la /chipmunk/chipmunk_performance_results
 
   push_data_to_chipmunk_docs:
     name: Move benchmark data to chipmunk-docs repository

application/apps/rustcore/ts-bindings/spec/benchmarks.json

@@ -5,42 +5,42 @@
             "test1": {
                 "alias": "Observe - grab content (text)",
                 "file": "test_files/temp_readings3.txt",
-                "expectation_ms": 10000
+                "expectation_ms": 2000
             },
             "test2": {
                 "alias": "Observe - grab content (dlt)",
                 "file": "test_files/FzgProg_SP21.dlt",
-                "expectation_ms": 60000
+                "expectation_ms": 120000
             },
             "test3": {
                 "alias": "Observe - grab content (pcapng)",
                 "file": "test_files/someip.pcapng",
-                "expectation_ms": 1000
+                "expectation_ms": 2000
             },
             "test4": {
                 "alias": "Stream - startup measurement",
                 "file": "",
-                "expectation_ms": 1000
+                "expectation_ms": 2000
             },
             "test5": {
                 "alias": "Stream - shutdown measurement",
                 "file": "",
-                "expectation_ms": 10000
+                "expectation_ms": 20000
             },
             "test6": {
                 "alias": "Stream - Open 50 sessions",
                 "file": "",
-                "expectation_ms": 10000
+                "expectation_ms": 20000
             },
             "test7": {
                 "alias": "Indexes - Switch to breadcrumb mode",
                 "file": "test_files/indexing_access_huge.log",
-                "expectation_ms": 15000
+                "expectation_ms": 30000
             },
             "test8": {
                 "alias": "Assign & single search",
                 "file": "test_files/indexing_access_huge.log",
-                "expectation_ms": 1000
+                "expectation_ms": 3000
             },
             "test9": {
                 "alias": "Assign & multiple search",

scripts/tools/run_benchmarks.rb → scripts/benchmarks/process.rb

@@ -3,19 +3,18 @@
 require 'fileutils'
 require 'json'
 
+ENV['REPO_OWNER'] = 'esrlabs'
+ENV['REPO_NAME'] = 'chipmunk'
 
-ENV['REPO_OWNER'] ||= 'esrlabs'
-ENV['REPO_NAME'] ||= 'chipmunk'
-
-RAKE_COMMANDS = [
-  'rake clobber',
-  'rake bindings:test:benchmark'
+COMMANDS = [
+  'cargo install --path=cli',
+  'cargo chipmunk test wrapper -s spec/build/spec/session.benchmark.spec.js -u print'
 ]
 
 SHELL_SCRIPT_PATH = 'application/apps/rustcore/ts-bindings/spec'
 
 def usage
-  puts "Usage: ruby scripts/tools/run_benchmarks.rb <number_of_releases>/<start_tag-end_tag>/PR~<PR_NUMBER>"
+  puts "Usage: ruby scripts/benchmarks/process.rb <number_of_releases>/<start_tag-end_tag>/PR~<PR_NUMBER>"
   exit(1)
 end
 
@@ -42,7 +41,7 @@ def compute_average_of_benchmarks(result_path)
 client = Octokit::Client.new
 
 def fetch_pull_request(client, pr_number)
-  client.pull_request("esrlabs/chipmunk", pr_number)
+  client.pull_request("#{ENV['REPO_OWNER']}/#{ENV['REPO_NAME']}", pr_number)
 end
 
 def fetch_releases(client)
@@ -77,7 +76,7 @@ def set_environment_vars
 def clone_and_setup_repo(branch_or_tag_name, temp_dir)
   system("git clone --depth 1 --branch #{branch_or_tag_name} https://github.com/#{ENV['REPO_OWNER']}/#{ENV['REPO_NAME']}.git #{temp_dir}")
   FileUtils.cp_r("#{SHELL_SCRIPT_PATH}/.", "#{temp_dir}/#{SHELL_SCRIPT_PATH}/.", verbose: true)
-  FileUtils.cp_r("scripts/elements/bindings.rb", "#{temp_dir}/scripts/elements/bindings.rb", verbose: true)
+  FileUtils.cp_r("cli", "#{temp_dir}/", verbose: true)
 end
 
 def process_release_or_pr(branch_or_tag_name, identifier, env_vars)
@@ -91,16 +90,21 @@ def process_release_or_pr(branch_or_tag_name, identifier, env_vars)
         ENV['PERFORMANCE_RESULTS'] = "Benchmark_#{identifier}.json"
         system("corepack enable")
         system("yarn cache clean")
+        system(COMMANDS[0])
 
         next unless File.exist?("#{SHELL_SCRIPT_PATH}/#{env_vars['JASMIN_TEST_CONFIGURATION'].gsub('./spec/', '')}")
 
         puts "Benchmark.json file available."
 
         FileUtils.rm(result_path, verbose: true) if File.exist?(result_path)
-
-        RAKE_COMMANDS.each do |command|
-          puts "Running #{command} for #{identifier}"
-          system(command)
+        iterations = 6
+        for i in 1..iterations do
+          puts "Running #{COMMANDS[1]} for #{identifier} - iteration #{i}"
+          begin
+            system(COMMANDS[1])
+          rescue
+            next
+          end
         end
       end
 
@@ -156,6 +160,8 @@ def update_performance_data(data, data_file_path)
   pr_number = parsed_arg[:value]
   pull_request = fetch_pull_request(client, pr_number)
   branch_name = pull_request.head.ref
+  ENV['REPO_OWNER'] = pull_request.head.user.login
+  ENV['REPO_NAME'] = pull_request.head.repo.name
   puts "Running benchmarks for the pull request: #{pull_request.title} (#{branch_name})"
   process_release_or_pr(branch_name, "PR_#{pr_number}", env_vars)
 when :number_of_releases

scripts/benchmarks/setup_dependencies.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Multiple dependencies required to be explicitly installed since self hosted runners do not support some github actions.
+sudo apt-get update
+sudo apt-get install -y gconf-service libgbm-dev libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
+libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
+libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 \
+libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \
+libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libnss3 lsb-release xdg-utils wget \
+libudev-dev cargo npm imagemagick libmagickwand-dev cmake jq

scripts/benchmarks/setup_ruby.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Need to install ruby separately because github action for installing ruby does not work on self hosted runner.
+whoami
+npm install -g corepack
+corepack enable
+curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+sudo apt-get update && sudo apt-get install -y yarn software-properties-common
+apt-add-repository -y ppa:rael-gc/rvm
+sudo apt-get update && sudo apt-get install -y rvm
+echo 'source "/etc/profile.d/rvm.sh"' >> ~/.bashrc
+source /etc/profile.d/rvm.sh
+rvm install ruby-3.1.2
+ruby --version
+sudo chown -R $(whoami) /usr/local
+rvm use 3.1.2 --default
+export PATH="/usr/share/rvm:$PATH"
+gem install dotenv json octokit tmpdir fileutils