Merge branch 'master' into umbrella-authority-discovery

.cargo/config.toml

@@ -9,3 +9,7 @@ rustdocflags = [
 CC_x86_64_unknown_linux_musl = { value = ".cargo/musl-gcc", force = true, relative = true }
 CXX_x86_64_unknown_linux_musl = { value = ".cargo/musl-g++", force = true, relative = true }
 CARGO_WORKSPACE_ROOT_DIR = { value = "", relative = true }
+
+[net]
+retry = 5
+# git-fetch-with-cli = true # commented because there is a risk that a runner can be banned by github

.config/lychee.toml

@@ -28,7 +28,7 @@ exclude = [
 	"http://visitme/",
 	"https://visitme/",
 
-	# TODO <https://github.com/paritytech/polkadot-sdk/issues/134>
+	# TODO meta issue: <https://github.com/paritytech/polkadot-sdk/issues/134>
 	"https://docs.substrate.io/main-docs/build/custom-rpc/#public-rpcs",
 	"https://docs.substrate.io/rustdocs/latest/sp_api/macro.decl_runtime_apis.html",
 	"https://github.com/ipfs/js-ipfs-bitswap/blob/",
@@ -50,8 +50,10 @@ exclude = [
 	"https://w3f.github.io/parachain-implementers-guide/runtime/session_info.html",
 
 	# Behind a captcha (code 403):
+	"https://chainlist.org/chain/*",
 	"https://iohk.io/en/blog/posts/2023/11/03/partner-chains-are-coming-to-cardano/",
 	"https://www.reddit.com/r/rust/comments/3spfh1/does_collect_allocate_more_than_once_while/",
+
 	# 403 rate limited:
 	"https://etherscan.io/block/11090290",
 	"https://subscan.io/",

.config/taplo.toml

@@ -40,3 +40,10 @@ keys = ["workspace.dependencies"]
 
 [rule.formatting]
 reorder_keys = true
+
+[[rule]]
+include = ["**/Cargo.toml"]
+keys = ["build-dependencies", "dependencies", "dev-dependencies"]
+
+[rule.formatting]
+reorder_keys = true

.github/scripts/common/lib.sh

@@ -297,7 +297,6 @@ fetch_release_artifacts_from_s3() {
   pwd
   ls -al --color
   popd > /dev/null
-  unset OUTPUT_DIR
 }
 
 # Pass the name of the binary as input, it will

.github/scripts/generate-prdoc.py

@@ -36,6 +36,21 @@ def from_pr_number(n, audience, bump, force):
 
 	create_prdoc(n, audience, pr.title, pr.body, patch, bump, force)
 
+def translate_audience(audience):
+	aliases = {
+		'runtime_dev': 'Runtime Dev',
+		'runtime_user': 'Runtime Operator',
+		'node_dev': 'Node Dev',
+		'node_user': 'Node User',
+	}
+
+	if audience in aliases:
+		to = aliases[audience]
+		print(f"Translated audience '{audience}' to '{to}'")
+		audience = to
+
+	return audience
+
 def create_prdoc(pr, audience, title, description, patch, bump, force):
 	path = f"prdoc/pr_{pr}.prdoc"
 
@@ -49,6 +64,7 @@ def create_prdoc(pr, audience, title, description, patch, bump, force):
 		print(f"No preexisting PrDoc for PR {pr}")
 
 	prdoc = { "title": title, "doc": [{}], "crates": [] }
+	audience = translate_audience(audience)
 
 	prdoc["doc"][0]["audience"] = audience
 	prdoc["doc"][0]["description"] = description
@@ -117,7 +133,7 @@ def setup_parser(parser=None, pr_required=True):
 		parser = argparse.ArgumentParser()
 	parser.add_argument("--pr", type=int, required=pr_required, help="The PR number to generate the PrDoc for.")
 	parser.add_argument("--audience", type=str, nargs='*', choices=allowed_audiences, default=["todo"], help="The audience of whom the changes may concern. Example: --audience runtime_dev node_dev")
-	parser.add_argument("--bump", type=str, default="major", choices=["patch", "minor", "major", "silent", "ignore", "no_change"], help="A default bump level for all crates. Example: --bump patch")
+	parser.add_argument("--bump", type=str, default="major", choices=["patch", "minor", "major", "silent", "ignore", "none"], help="A default bump level for all crates. Example: --bump patch")
 	parser.add_argument("--force", action="store_true", help="Whether to overwrite any existing PrDoc.")
 	return parser
 

.github/workflows/benchmarks-networking.yml

@@ -0,0 +1,111 @@
+name: Networking Benchmarks
+
+on:
+  push:
+    branches:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  preflight:
+    uses: ./.github/workflows/reusable-preflight.yml
+
+  build:
+    timeout-minutes: 50
+    needs: [preflight]
+    runs-on: ${{ needs.preflight.outputs.RUNNER_BENCHMARK }}
+    container:
+      image: ${{ needs.preflight.outputs.IMAGE }}
+    strategy:
+      fail-fast: false
+      matrix:
+        features:
+          [
+            { bench: "notifications_protocol" },
+            { bench: "request_response_protocol" },
+          ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run Benchmarks
+        id: run-benchmarks
+        run: |
+          mkdir -p ./charts
+          forklift cargo bench -p sc-network --bench ${{ matrix.features.bench }} -- --output-format bencher | grep "^test" | tee ./charts/${{ matrix.features.bench }}.txt || echo "Benchmarks failed"
+          ls -lsa ./charts
+
+      - name: Upload artifacts
+        uses: actions/[email protected]
+        with:
+          name: ${{ matrix.features.bench }}-${{ github.sha }}
+          path: ./charts
+
+  publish-benchmarks:
+    timeout-minutes: 60
+    needs: [build]
+    if: github.ref == 'refs/heads/master'
+    environment: subsystem-benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+          fetch-depth: 0
+
+      - run: git checkout master --
+
+      - name: Download artifacts
+        uses: actions/[email protected]
+        with:
+          name: notifications_protocol-${{ github.sha }}
+          path: ./charts
+
+      - name: Download artifacts
+        uses: actions/[email protected]
+        with:
+          name: request_response_protocol-${{ github.sha }}
+          path: ./charts
+
+      - name: Setup git
+        run: |
+          # Fixes "detected dubious ownership" error in the ci
+          git config --global --add safe.directory '*'
+          ls -lsR ./charts
+
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ secrets.POLKADOTSDK_GHPAGES_APP_ID }}
+          private-key: ${{ secrets.POLKADOTSDK_GHPAGES_APP_KEY }}
+
+      - name: Generate ${{ env.BENCH }}
+        env:
+          BENCH: notifications_protocol
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "cargo"
+          name: ${{ env.BENCH }}
+          output-file-path: ./charts/${{ env.BENCH }}.txt
+          benchmark-data-dir-path: ./bench/${{ env.BENCH }}
+          github-token: ${{ steps.app-token.outputs.token }}
+          auto-push: true
+
+      - name: Generate ${{ env.BENCH }}
+        env:
+          BENCH: request_response_protocol
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "cargo"
+          name: ${{ env.BENCH }}
+          output-file-path: ./charts/${{ env.BENCH }}.txt
+          benchmark-data-dir-path: ./bench/${{ env.BENCH }}
+          github-token: ${{ steps.app-token.outputs.token }}
+          auto-push: true

.github/workflows/build-misc.yml

@@ -20,7 +20,7 @@ jobs:
     uses: ./.github/workflows/reusable-preflight.yml
 
   build-runtimes-polkavm:
-    timeout-minutes: 20
+    timeout-minutes: 60
     needs: [preflight]
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     container:
@@ -38,11 +38,7 @@ jobs:
         env:
           SUBSTRATE_RUNTIME_TARGET: riscv
         id: required
-        run: |
-          forklift cargo check -p minimal-template-runtime
-          forklift cargo check -p westend-runtime
-          forklift cargo check -p rococo-runtime
-          forklift cargo check -p polkadot-test-runtime
+        run: forklift cargo check -p minimal-template-runtime -p westend-runtime -p rococo-runtime -p polkadot-test-runtime
       - name: Stop all workflows if failed
         if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper

.github/workflows/check-semver.yml

@@ -2,7 +2,7 @@ name: Check semver
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
   workflow_dispatch:
   merge_group:
 
@@ -62,21 +62,29 @@ jobs:
 
           echo "PRDOC_EXTRA_ARGS=--max-bump minor" >> $GITHUB_ENV
 
+      - name: Echo Skip
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'R0-silent') }}
+        run: echo "Skipping this PR because it is labeled as R0-silent."
+
       - name: Rust Cache
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'R0-silent') }}
         uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
         with:
           cache-on-failure: true
 
       - name: Rust compilation prerequisites
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'R0-silent') }}
         run: |
           rustup default $TOOLCHAIN
           rustup component add rust-src --toolchain $TOOLCHAIN
 
       - name: install parity-publish
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'R0-silent') }}
         # Set the target dir to cache the build.
-        run: CARGO_TARGET_DIR=./target/ cargo install [email protected].2 --locked -q
+        run: CARGO_TARGET_DIR=./target/ cargo install [email protected].3 --locked -q
 
       - name: check semver
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'R0-silent') }}
         run: |
           if [ -z "$PR" ]; then
             echo "Skipping master/merge queue"

.github/workflows/cmd.yml

@@ -19,10 +19,10 @@ jobs:
     steps:
       - name: Generate token
         id: generate_token
-        uses: tibdex/github-app-token@v2.1.0
+        uses: actions/create-github-app-token@v1
         with:
-          app_id: ${{ secrets.CMD_BOT_APP_ID }}
-          private_key: ${{ secrets.CMD_BOT_APP_KEY }}
+          app-id: ${{ secrets.CMD_BOT_APP_ID }}
+          private-key: ${{ secrets.CMD_BOT_APP_KEY }}
 
       - name: Check if user is a member of the organization
         id: is-member
@@ -292,9 +292,17 @@ jobs:
       image: ${{ needs.set-image.outputs.IMAGE }}
     timeout-minutes: 1440 # 24 hours per runtime
     steps:
+      - name: Generate token
+        uses: actions/create-github-app-token@v1
+        id: generate_token
+        with:
+          app-id: ${{ secrets.CMD_BOT_APP_ID }}
+          private-key: ${{ secrets.CMD_BOT_APP_KEY }}
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
+          token: ${{ steps.generate_token.outputs.token }}
           repository: ${{ needs.get-pr-branch.outputs.repo }}
           ref: ${{ needs.get-pr-branch.outputs.pr-branch }}
 
@@ -392,19 +400,41 @@ jobs:
           name: command-output
           path: /tmp/cmd/command_output.log
 
+      # Generate token for commit, as the earlier token expires after 1 hour, while cmd can take longer
+      - name: Generate token for commit
+        uses: actions/create-github-app-token@v1
+        id: generate_token_commit
+        with:
+          app-id: ${{ secrets.CMD_BOT_APP_ID }}
+          private-key: ${{ secrets.CMD_BOT_APP_KEY }}
+
       - name: Commit changes
         run: |
           if [ -n "$(git status --porcelain)" ]; then
-            git config --local user.email "[email protected]"
-            git config --local user.name "GitHub Action"
+            git config --global user.name command-bot
+            git config --global user.email "<>"
+            git config --global pull.rebase false
+
+            # Push the results to the target branch
+            git remote add \
+              github \
+              "https://x-access-token:${{ steps.generate_token_commit.outputs.token }}@github.com/${{ needs.get-pr-branch.outputs.repo }}.git" || :
+
+            push_changes() {
+              git push github "HEAD:${{ needs.get-pr-branch.outputs.pr-branch }}"
+            }
 
             git add .
             git restore --staged Cargo.lock # ignore changes in Cargo.lock
             git commit -m "Update from ${{ github.actor }} running command '${{ steps.get-pr-comment.outputs.group2 }}'" || true
             
-            git pull --rebase origin ${{ needs.get-pr-branch.outputs.pr-branch }}
-            
-            git push origin ${{ needs.get-pr-branch.outputs.pr-branch }}
+            # Attempt to push changes
+            if ! push_changes; then
+              echo "Push failed, trying to rebase..."
+              git pull --rebase github "${{ needs.get-pr-branch.outputs.pr-branch }}"
+              # After successful rebase, try pushing again
+              push_changes
+            fi
           else
             echo "Nothing to commit";
           fi

.github/workflows/command-prdoc.yml

@@ -14,7 +14,7 @@ on:
         required: true
         options:
           - "TODO"
-          - "no_change"
+          - "none"
           - "patch"
           - "minor"
           - "major"

.github/workflows/misc-sync-templates.yml

@@ -21,6 +21,10 @@ on:
       stable_release_branch:
         description: 'Stable release branch, e.g. stable2407'
         required: true
+      debug:
+        description: Enable runner debug logging
+        required: false
+        default: false
 
 jobs:
   sync-templates:
@@ -86,7 +90,7 @@ jobs:
           EOF
 
           [ ${{ matrix.template }} != "solochain" ] && echo "# Leave out the node compilation from regular template usage." \
-            && echo "\"default-members\" = [\"pallets/template\", \"runtime\"]" >> Cargo.toml
+            && echo "default-members = [\"pallets/template\", \"runtime\"]" >> Cargo.toml
           [ ${{ matrix.template }} == "solochain" ] && echo "# The node isn't yet replaceable by Omni Node."
           cat << EOF >> Cargo.toml
           members = [
@@ -115,8 +119,9 @@ jobs:
           toml set templates/${{ matrix.template }}/Cargo.toml 'workspace.package.edition' "$(toml get --raw Cargo.toml 'workspace.package.edition')" > Cargo.temp
           mv Cargo.temp ./templates/${{ matrix.template }}/Cargo.toml
         working-directory: polkadot-sdk
+
       - name: Print the result Cargo.tomls for debugging
-        if: runner.debug == '1'
+        if: ${{ github.event.inputs.debug }}
         run: find . -type f -name 'Cargo.toml' -exec cat {} \;
         working-directory: polkadot-sdk/templates/${{ matrix.template }}/
 
@@ -142,6 +147,12 @@ jobs:
           done;
         working-directory: "${{ env.template-path }}"
 
+      - name: Print the result Cargo.tomls for debugging after copying required workspace dependencies
+        if: ${{ github.event.inputs.debug }}
+        run: find . -type f -name 'Cargo.toml' -exec cat {} \;
+        working-directory: polkadot-sdk/templates/${{ matrix.template }}/
+
+
       # 3. Verify the build. Push the changes or create a PR.
 
       # We've run into out-of-disk error when compiling in the next step, so we free up some space this way.