diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 5d75f8432..19ead8623 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -27,14 +27,7 @@ jobs: uses: actions/checkout@v3 - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y gconf-service libgbm-dev libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \ - libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \ - libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 \ - libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ - libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libnss3 lsb-release xdg-utils wget \ - libudev-dev cargo npm imagemagick libmagickwand-dev cmake jq + run: ./scripts/benchmarks/setup_dependencies.sh - name: Install Node.js uses: actions/setup-node@v4 @@ -42,64 +35,29 @@ jobs: node-version: 22.4.0 - name: Install Ruby and related tools - run: | - whoami - npm install -g corepack - corepack enable - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list - sudo apt-get update && sudo apt-get install -y yarn software-properties-common - apt-add-repository -y ppa:rael-gc/rvm - sudo apt-get update && sudo apt-get install -y rvm - echo 'source "/etc/profile.d/rvm.sh"' >> ~/.bashrc - source /etc/profile.d/rvm.sh - rvm install ruby-3.1.2 - ruby --version - sudo chown -R $(whoami) /usr/local - rvm use 3.1.2 --default + run: ./scripts/benchmarks/setup_dependencies.sh - name: Install cargo tools run: | cargo install nj-cli wasm-pack --locked - - - name: Prepare environment - run: | npm install -g tslib - export PATH="/usr/share/rvm:$PATH" - gem install dotenv json octokit tmpdir fileutils - - - name: Get PR details - if: github.event_name == 'workflow_dispatch' - uses: actions/github-script@v7 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - script: | - const fs = require('fs'); - const pr = await github.rest.pulls.get({ - owner: 'esrlabs', - repo: 'chipmunk', - pull_number: ${{ github.event.inputs.pr_id }} - }); - fs.appendFileSync(process.env.GITHUB_ENV, `REPO_OWNER=${pr.data.user.login}\n`); - fs.appendFileSync(process.env.GITHUB_ENV, `REPO_NAME=${pr.data.base.repo.name}\n`); - name: Run Jasmine performance tests run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y export PATH="/root/.cargo/bin:$PATH" source ~/.bashrc - if [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then - ruby scripts/tools/run_benchmarks.rb PR~${{ github.event.inputs.pr_id }} - else - export REPO_OWNER='esrlabs' - export REPO_NAME='chipmunk' - ruby scripts/tools/run_benchmarks.rb 1 + if [[ "${{ github.event_name }}" == 'workflow_dispatch' && "${{ github.event.inputs.pr_id }}" =~ ^[0-9]+$ ]]; then + ruby scripts/benchmarks/process.rb PR~${{ github.event.inputs.pr_id }} + elif [[ "${{ github.event_name }}" == 'workflow_dispatch' && "${{ github.event.inputs.pr_id }}" -eq 0 ]]; then + echo "Running manually" + ruby scripts/benchmarks/process.rb 1 + elif [[ "${{ github.event_name }}" != 'workflow_dispatch' ]]; then + ruby scripts/benchmarks/process.rb 1 fi - name: List files in the results folder - run: | - ls -la /chipmunk/chipmunk_performance_results + run: ls -la /chipmunk/chipmunk_performance_results push_data_to_chipmunk_docs: name: Move benchmark data to chipmunk-docs repository diff --git a/application/apps/rustcore/ts-bindings/spec/benchmarks.json b/application/apps/rustcore/ts-bindings/spec/benchmarks.json index bb7478f43..8df4021d1 100644 --- a/application/apps/rustcore/ts-bindings/spec/benchmarks.json +++ b/application/apps/rustcore/ts-bindings/spec/benchmarks.json @@ -5,42 +5,42 @@ "test1": { "alias": "Observe - grab content (text)", "file": "test_files/temp_readings3.txt", - "expectation_ms": 10000 + "expectation_ms": 2000 }, "test2": { "alias": "Observe - grab content (dlt)", "file": "test_files/FzgProg_SP21.dlt", - "expectation_ms": 60000 + "expectation_ms": 120000 }, "test3": { "alias": "Observe - grab content (pcapng)", "file": "test_files/someip.pcapng", - "expectation_ms": 1000 + "expectation_ms": 2000 }, "test4": { "alias": "Stream - startup measurement", "file": "", - "expectation_ms": 1000 + "expectation_ms": 2000 }, "test5": { "alias": "Stream - shutdown measurement", "file": "", - "expectation_ms": 10000 + "expectation_ms": 20000 }, "test6": { "alias": "Stream - Open 50 sessions", "file": "", - "expectation_ms": 10000 + "expectation_ms": 20000 }, "test7": { "alias": "Indexes - Switch to breadcrumb mode", "file": "test_files/indexing_access_huge.log", - "expectation_ms": 15000 + "expectation_ms": 30000 }, "test8": { "alias": "Assign & single search", "file": "test_files/indexing_access_huge.log", - "expectation_ms": 1000 + "expectation_ms": 3000 }, "test9": { "alias": "Assign & multiple search", diff --git a/scripts/tools/run_benchmarks.rb b/scripts/benchmarks/process.rb similarity index 88% rename from scripts/tools/run_benchmarks.rb rename to scripts/benchmarks/process.rb index 3d13a6fc0..ce1873c32 100644 --- a/scripts/tools/run_benchmarks.rb +++ b/scripts/benchmarks/process.rb @@ -3,19 +3,18 @@ require 'fileutils' require 'json' +ENV['REPO_OWNER'] = 'esrlabs' +ENV['REPO_NAME'] = 'chipmunk' -ENV['REPO_OWNER'] ||= 'esrlabs' -ENV['REPO_NAME'] ||= 'chipmunk' - -RAKE_COMMANDS = [ - 'rake clobber', - 'rake bindings:test:benchmark' +COMMANDS = [ + 'cargo install --path=cli', + 'cargo chipmunk test wrapper -s spec/build/spec/session.benchmark.spec.js -u print' ] SHELL_SCRIPT_PATH = 'application/apps/rustcore/ts-bindings/spec' def usage - puts "Usage: ruby scripts/tools/run_benchmarks.rb //PR~" + puts "Usage: ruby scripts/benchmarks/process.rb //PR~" exit(1) end @@ -42,7 +41,7 @@ def compute_average_of_benchmarks(result_path) client = Octokit::Client.new def fetch_pull_request(client, pr_number) - client.pull_request("esrlabs/chipmunk", pr_number) + client.pull_request("#{ENV['REPO_OWNER']}/#{ENV['REPO_NAME']}", pr_number) end def fetch_releases(client) @@ -77,7 +76,7 @@ def set_environment_vars def clone_and_setup_repo(branch_or_tag_name, temp_dir) system("git clone --depth 1 --branch #{branch_or_tag_name} https://github.com/#{ENV['REPO_OWNER']}/#{ENV['REPO_NAME']}.git #{temp_dir}") FileUtils.cp_r("#{SHELL_SCRIPT_PATH}/.", "#{temp_dir}/#{SHELL_SCRIPT_PATH}/.", verbose: true) - FileUtils.cp_r("scripts/elements/bindings.rb", "#{temp_dir}/scripts/elements/bindings.rb", verbose: true) + FileUtils.cp_r("cli", "#{temp_dir}/", verbose: true) end def process_release_or_pr(branch_or_tag_name, identifier, env_vars) @@ -91,16 +90,21 @@ def process_release_or_pr(branch_or_tag_name, identifier, env_vars) ENV['PERFORMANCE_RESULTS'] = "Benchmark_#{identifier}.json" system("corepack enable") system("yarn cache clean") + system(COMMANDS[0]) next unless File.exist?("#{SHELL_SCRIPT_PATH}/#{env_vars['JASMIN_TEST_CONFIGURATION'].gsub('./spec/', '')}") puts "Benchmark.json file available." FileUtils.rm(result_path, verbose: true) if File.exist?(result_path) - - RAKE_COMMANDS.each do |command| - puts "Running #{command} for #{identifier}" - system(command) + iterations = 6 + for i in 1..iterations do + puts "Running #{COMMANDS[1]} for #{identifier} - iteration #{i}" + begin + system(COMMANDS[1]) + rescue + next + end end end @@ -156,6 +160,8 @@ def update_performance_data(data, data_file_path) pr_number = parsed_arg[:value] pull_request = fetch_pull_request(client, pr_number) branch_name = pull_request.head.ref + ENV['REPO_OWNER'] = pull_request.head.user.login + ENV['REPO_NAME'] = pull_request.head.repo.name puts "Running benchmarks for the pull request: #{pull_request.title} (#{branch_name})" process_release_or_pr(branch_name, "PR_#{pr_number}", env_vars) when :number_of_releases diff --git a/scripts/benchmarks/setup_dependencies.sh b/scripts/benchmarks/setup_dependencies.sh new file mode 100755 index 000000000..ed7dda41e --- /dev/null +++ b/scripts/benchmarks/setup_dependencies.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Multiple dependencies required to be explicitly installed since self hosted runners do not support some github actions. +sudo apt-get update +sudo apt-get install -y gconf-service libgbm-dev libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \ +libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \ +libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 \ +libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ +libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libnss3 lsb-release xdg-utils wget \ +libudev-dev cargo npm imagemagick libmagickwand-dev cmake jq \ No newline at end of file diff --git a/scripts/benchmarks/setup_ruby.sh b/scripts/benchmarks/setup_ruby.sh new file mode 100755 index 000000000..f2741a0a2 --- /dev/null +++ b/scripts/benchmarks/setup_ruby.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Need to install ruby separately because github action for installing ruby does not work on self hosted runner. +whoami +npm install -g corepack +corepack enable +curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list +sudo apt-get update && sudo apt-get install -y yarn software-properties-common +apt-add-repository -y ppa:rael-gc/rvm +sudo apt-get update && sudo apt-get install -y rvm +echo 'source "/etc/profile.d/rvm.sh"' >> ~/.bashrc +source /etc/profile.d/rvm.sh +rvm install ruby-3.1.2 +ruby --version +sudo chown -R $(whoami) /usr/local +rvm use 3.1.2 --default +export PATH="/usr/share/rvm:$PATH" +gem install dotenv json octokit tmpdir fileutils \ No newline at end of file