Merge branch 'master' into ao-fix-issue-64

* master: (256 commits)
  fix chunk fetching network compatibility zombienet test (#6988)
  chore: delete repeat words (#7034)
  Print taplo version in CI (#7041)
  Implement cumulus StorageWeightReclaim as wrapping transaction extension + frame system ReclaimWeight (#6140)
  Make `TransactionExtension` tuple of tuple transparent for implication (#7028)
  Replace duplicated whitelist with whitelisted_storage_keys (#7024)
  [WIP] Fix networking-benchmarks (#7036)
  [docs] Fix release naming (#7032)
  migrate pallet-mixnet to umbrella crate (#6986)
  Improve remote externalities logging (#7021)
  Fix polkadot sdk doc. (#7022)
  Remove warning log from frame-omni-bencher CLI (#7020)
  [pallet-revive] fix file case (#6981)
  Add workflow for networking benchmarks (#7029)
  [CI] Skip SemVer on R0-silent and update docs (#6285)
  correct path in cumulus README (#7001)
  sync: Send already connected peers to new subscribers (#7011)
  Excluding chainlink domain for link checker CI (#6524)
  pallet-bounties: Fix benchmarks for 0 ED (#7013)
  Log peerset set ID -> protocol name mapping (#7005)
  ...

Author: ordian

.cargo/config.toml

@@ -9,3 +9,7 @@ rustdocflags = [
 CC_x86_64_unknown_linux_musl = { value = ".cargo/musl-gcc", force = true, relative = true }
 CXX_x86_64_unknown_linux_musl = { value = ".cargo/musl-g++", force = true, relative = true }
 CARGO_WORKSPACE_ROOT_DIR = { value = "", relative = true }
+
+[net]
+retry = 5
+# git-fetch-with-cli = true # commented because there is a risk that a runner can be banned by github

.config/lychee.toml

@@ -28,7 +28,7 @@ exclude = [
 	"http://visitme/",
 	"https://visitme/",
 
-	# TODO <https://github.com/paritytech/polkadot-sdk/issues/134>
+	# TODO meta issue: <https://github.com/paritytech/polkadot-sdk/issues/134>
 	"https://docs.substrate.io/main-docs/build/custom-rpc/#public-rpcs",
 	"https://docs.substrate.io/rustdocs/latest/sp_api/macro.decl_runtime_apis.html",
 	"https://github.com/ipfs/js-ipfs-bitswap/blob/",
@@ -50,8 +50,10 @@ exclude = [
 	"https://w3f.github.io/parachain-implementers-guide/runtime/session_info.html",
 
 	# Behind a captcha (code 403):
+	"https://chainlist.org/chain/*",
 	"https://iohk.io/en/blog/posts/2023/11/03/partner-chains-are-coming-to-cardano/",
 	"https://www.reddit.com/r/rust/comments/3spfh1/does_collect_allocate_more_than_once_while/",
+
 	# 403 rate limited:
 	"https://etherscan.io/block/11090290",
 	"https://subscan.io/",

.config/nextest.toml

@@ -21,7 +21,6 @@ retries = 5
 # The number of threads to run tests with. Supported values are either an integer or
 # the string "num-cpus". Can be overridden through the `--test-threads` option.
 # test-threads = "num-cpus"
-
 test-threads = 20
 
 # The number of threads required for each test. This is generally used in overrides to
@@ -124,3 +123,10 @@ serial-integration = { max-threads = 1 }
 [[profile.default.overrides]]
 filter = 'test(/(^ui$|_ui|ui_)/)'
 test-group = 'serial-integration'
+
+# Running eth-rpc tests sequentially
+# These tests rely on a shared resource (the RPC and Node) 
+# and would cause race conditions due to transaction nonces if run in parallel.
+[[profile.default.overrides]]
+filter = 'package(pallet-revive-eth-rpc) and test(/^tests::/)'
+test-group = 'serial-integration'

.config/taplo.toml

@@ -40,3 +40,10 @@ keys = ["workspace.dependencies"]
 
 [rule.formatting]
 reorder_keys = true
+
+[[rule]]
+include = ["**/Cargo.toml"]
+keys = ["build-dependencies", "dependencies", "dev-dependencies"]
+
+[rule.formatting]
+reorder_keys = true

.github/actions/workflow-stopper/action.yml

@@ -0,0 +1,28 @@
+name: "stop all workflows"
+description: "Action stops all workflows in a PR to save compute resources."
+inputs:
+  app-id:
+    description: "App id"
+    required: true
+  app-key:
+    description: "App token"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Worfklow stopper - Generate token
+      uses: actions/create-github-app-token@v1
+      id: app-token
+      with:
+        app-id: ${{ inputs.app-id }}
+        private-key: ${{ inputs.app-key }}
+        owner: "paritytech"
+        repositories: "workflow-stopper"
+    - name: Workflow stopper - Stop all workflows
+      uses: octokit/request-action@v2.x
+      with:
+        route: POST /repos/paritytech/workflow-stopper/actions/workflows/stopper.yml/dispatches
+        ref: main
+        inputs: '${{ format(''{{ "github_sha": "{0}", "github_repository": "{1}", "github_ref_name": "{2}", "github_workflow_id": "{3}", "github_job_name": "{4}" }}'', github.event.pull_request.head.sha, github.repository, github.ref_name, github.run_id, github.job) }}'
+      env:
+        GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}

.github/env

@@ -1 +1 @@
-IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"
+IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-11-19-v202411281558"

.github/scripts/check-missing-readme-generation.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+echo "Running script relative to `pwd`"
+# Find all README.docify.md files
+DOCIFY_FILES=$(find . -name "README.docify.md")
+
+# Initialize a variable to track directories needing README regeneration
+NEED_REGENERATION=""
+
+for file in $DOCIFY_FILES; do
+  echo "Processing $file"
+  
+  # Get the directory containing the docify file
+  DIR=$(dirname "$file")
+  
+  # Go to the directory and run cargo build
+  cd "$DIR"
+  cargo check --features generate-readme || { echo "Readme generation for $DIR failed. Ensure the crate compiles successfully and has a `generate-readme` feature which guards markdown compilation in the crate as follows: https://docs.rs/docify/latest/docify/macro.compile_markdown.html#conventions." && exit 1; }
+  
+  # Check if README.md has any uncommitted changes
+  git diff --exit-code README.md
+  
+  if [ $? -ne 0 ]; then
+    echo "Error: Found uncommitted changes in $DIR/README.md"
+    NEED_REGENERATION="$NEED_REGENERATION $DIR"
+  fi
+  
+  # Return to the original directory
+  cd - > /dev/null
+done
+
+# Check if any directories need README regeneration
+if [ -n "$NEED_REGENERATION" ]; then
+  echo "The following directories need README regeneration:"
+  echo "$NEED_REGENERATION"
+  exit 1
+fi

.github/scripts/cmd/cmd.py

@@ -58,7 +58,7 @@ def setup_logging():
  %(prog)s --runtime westend rococo --pallet pallet_balances pallet_multisig --quiet --clean
 '''
 
-parser_bench = subparsers.add_parser('bench', help='Runs benchmarks', epilog=bench_example, formatter_class=argparse.RawDescriptionHelpFormatter)
+parser_bench = subparsers.add_parser('bench', help='Runs benchmarks (old CLI)', epilog=bench_example, formatter_class=argparse.RawDescriptionHelpFormatter)
 
 for arg, config in common_args.items():
     parser_bench.add_argument(arg, **config)
@@ -67,6 +67,35 @@ def setup_logging():
 parser_bench.add_argument('--pallet', help='Pallet(s) space separated', nargs='*', default=[])
 parser_bench.add_argument('--fail-fast', help='Fail fast on first failed benchmark', action='store_true')
 
+
+"""
+BENCH OMNI
+"""
+
+bench_example = '''**Examples**:
+ Runs all benchmarks 
+ %(prog)s
+
+ Runs benchmarks for pallet_balances and pallet_multisig for all runtimes which have these pallets. **--quiet** makes it to output nothing to PR but reactions
+ %(prog)s --pallet pallet_balances pallet_xcm_benchmarks::generic --quiet
+ 
+ Runs bench for all pallets for westend runtime and fails fast on first failed benchmark
+ %(prog)s --runtime westend --fail-fast
+ 
+ Does not output anything and cleans up the previous bot's & author command triggering comments in PR 
+ %(prog)s --runtime westend rococo --pallet pallet_balances pallet_multisig --quiet --clean
+'''
+
+parser_bench_old = subparsers.add_parser('bench-omni', help='Runs benchmarks (frame omni bencher)', epilog=bench_example, formatter_class=argparse.RawDescriptionHelpFormatter)
+
+for arg, config in common_args.items():
+    parser_bench_old.add_argument(arg, **config)
+
+parser_bench_old.add_argument('--runtime', help='Runtime(s) space separated', choices=runtimeNames, nargs='*', default=runtimeNames)
+parser_bench_old.add_argument('--pallet', help='Pallet(s) space separated', nargs='*', default=[])
+parser_bench_old.add_argument('--fail-fast', help='Fail fast on first failed benchmark', action='store_true')
+
+
 """
 FMT
 """
@@ -98,12 +127,12 @@ def main():
 
     print(f'args: {args}')
 
-    if args.command == 'bench':
+    if args.command == 'bench-omni':
         runtime_pallets_map = {}
         failed_benchmarks = {}
         successful_benchmarks = {}
 
-        profile = "release"
+        profile = "production"
 
         print(f'Provided runtimes: {args.runtime}')
         # convert to mapped dict
@@ -113,11 +142,22 @@ def main():
 
         # loop over remaining runtimes to collect available pallets
         for runtime in runtimesMatrix.values():
-            os.system(f"forklift cargo build -p {runtime['package']} --profile {profile} --features={runtime['bench_features']}")
+            build_command = f"forklift cargo build -p {runtime['package']} --profile {profile} --features={runtime['bench_features']}"
+            print(f'-- building "{runtime["name"]}" with `{build_command}`')
+            os.system(build_command)
             print(f'-- listing pallets for benchmark for {runtime["name"]}')
             wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm"
-            output = os.popen(
-                f"frame-omni-bencher v1 benchmark pallet --no-csv-header --no-storage-info --no-min-squares --no-median-slopes --all --list --runtime={wasm_file} {runtime['bench_flags']}").read()
+            list_command = f"frame-omni-bencher v1 benchmark pallet " \
+                f"--no-csv-header " \
+                f"--no-storage-info " \
+                f"--no-min-squares " \
+                f"--no-median-slopes " \
+                f"--all " \
+                f"--list " \
+                f"--runtime={wasm_file} " \
+                f"{runtime['bench_flags']}"
+            print(f'-- running: {list_command}')
+            output = os.popen(list_command).read()
             raw_pallets = output.strip().split('\n')
 
             all_pallets = set()
@@ -230,6 +270,149 @@ def main():
             print_and_log('✅ Successful benchmarks of runtimes/pallets:')
             for runtime, pallets in successful_benchmarks.items():
                 print_and_log(f'-- {runtime}: {pallets}')
+    
+    if args.command == 'bench':
+        runtime_pallets_map = {}
+        failed_benchmarks = {}
+        successful_benchmarks = {}
+
+        profile = "production"
+
+        print(f'Provided runtimes: {args.runtime}')
+        # convert to mapped dict
+        runtimesMatrix = list(filter(lambda x: x['name'] in args.runtime, runtimesMatrix))
+        runtimesMatrix = {x['name']: x for x in runtimesMatrix}
+        print(f'Filtered out runtimes: {runtimesMatrix}')
+
+        # loop over remaining runtimes to collect available pallets
+        for runtime in runtimesMatrix.values():
+            build_command = f"forklift cargo build -p {runtime['old_package']} --profile {profile} --features={runtime['bench_features']} --locked"
+            print(f'-- building {runtime["name"]} with `{build_command}`')
+            os.system(build_command)
+            
+            chain = runtime['name'] if runtime['name'] == 'dev' else f"{runtime['name']}-dev"
+
+            machine_test = f"target/{profile}/{runtime['old_bin']} benchmark machine --chain={chain}"
+            print(f"Running machine test for `{machine_test}`")
+            os.system(machine_test)
+
+            print(f'-- listing pallets for benchmark for {chain}')
+            list_command = f"target/{profile}/{runtime['old_bin']} " \
+                f"benchmark pallet " \
+                f"--no-csv-header " \
+                f"--no-storage-info " \
+                f"--no-min-squares " \
+                f"--no-median-slopes " \
+                f"--all " \
+                f"--list " \
+                f"--chain={chain}"
+            print(f'-- running: {list_command}')
+            output = os.popen(list_command).read()
+            raw_pallets = output.strip().split('\n')
+
+            all_pallets = set()
+            for pallet in raw_pallets:
+                if pallet:
+                    all_pallets.add(pallet.split(',')[0].strip())
+
+            pallets = list(all_pallets)
+            print(f'Pallets in {runtime["name"]}: {pallets}')
+            runtime_pallets_map[runtime['name']] = pallets
+
+        print(f'\n')
+
+        # filter out only the specified pallets from collected runtimes/pallets
+        if args.pallet:
+            print(f'Pallets: {args.pallet}')
+            new_pallets_map = {}
+            # keep only specified pallets if they exist in the runtime
+            for runtime in runtime_pallets_map:
+                if set(args.pallet).issubset(set(runtime_pallets_map[runtime])):
+                    new_pallets_map[runtime] = args.pallet
+
+            runtime_pallets_map = new_pallets_map
+
+        print(f'Filtered out runtimes & pallets: {runtime_pallets_map}\n')
+
+        if not runtime_pallets_map:
+            if args.pallet and not args.runtime:
+                print(f"No pallets {args.pallet} found in any runtime")
+            elif args.runtime and not args.pallet:
+                print(f"{args.runtime} runtime does not have any pallets")
+            elif args.runtime and args.pallet:
+                print(f"No pallets {args.pallet} found in {args.runtime}")
+            else:
+                print('No runtimes found')
+            sys.exit(1)
+
+        for runtime in runtime_pallets_map:
+            for pallet in runtime_pallets_map[runtime]:
+                config = runtimesMatrix[runtime]
+                header_path = os.path.abspath(config['header'])
+                template = None
+
+                chain = config['name'] if runtime == 'dev' else f"{config['name']}-dev"
+
+                print(f'-- config: {config}')
+                if runtime == 'dev':
+                    # to support sub-modules (https://github.com/paritytech/command-bot/issues/275)
+                    search_manifest_path = f"cargo metadata --locked --format-version 1 --no-deps | jq -r '.packages[] | select(.name == \"{pallet.replace('_', '-')}\") | .manifest_path'"
+                    print(f'-- running: {search_manifest_path}')
+                    manifest_path = os.popen(search_manifest_path).read()
+                    if not manifest_path:
+                        print(f'-- pallet {pallet} not found in dev runtime')
+                        if args.fail_fast:
+                            print_and_log(f'Error: {pallet} not found in dev runtime')
+                            sys.exit(1)
+                    package_dir = os.path.dirname(manifest_path)
+                    print(f'-- package_dir: {package_dir}')
+                    print(f'-- manifest_path: {manifest_path}')
+                    output_path = os.path.join(package_dir, "src", "weights.rs")
+                    template = config['template']
+                else:
+                    default_path = f"./{config['path']}/src/weights"
+                    xcm_path = f"./{config['path']}/src/weights/xcm"
+                    output_path = default_path
+                    if pallet.startswith("pallet_xcm_benchmarks"):
+                        template = config['template']
+                        output_path = xcm_path
+
+                print(f'-- benchmarking {pallet} in {runtime} into {output_path}')
+                cmd = f"target/{profile}/{config['old_bin']} benchmark pallet " \
+                    f"--extrinsic=* " \
+                    f"--chain={chain} " \
+                    f"--pallet={pallet} " \
+                    f"--header={header_path} " \
+                    f"--output={output_path} " \
+                    f"--wasm-execution=compiled " \
+                    f"--steps=50 " \
+                    f"--repeat=20 " \
+                    f"--heap-pages=4096 " \
+                    f"{f'--template={template} ' if template else ''}" \
+                    f"--no-storage-info --no-min-squares --no-median-slopes "
+                print(f'-- Running: {cmd} \n')
+                status = os.system(cmd)
+
+                if status != 0 and args.fail_fast:
+                    print_and_log(f'❌ Failed to benchmark {pallet} in {runtime}')
+                    sys.exit(1)
+
+                # Otherwise collect failed benchmarks and print them at the end
+                # push failed pallets to failed_benchmarks
+                if status != 0:
+                    failed_benchmarks[f'{runtime}'] = failed_benchmarks.get(f'{runtime}', []) + [pallet]
+                else:
+                    successful_benchmarks[f'{runtime}'] = successful_benchmarks.get(f'{runtime}', []) + [pallet]
+
+        if failed_benchmarks:
+            print_and_log('❌ Failed benchmarks of runtimes/pallets:')
+            for runtime, pallets in failed_benchmarks.items():
+                print_and_log(f'-- {runtime}: {pallets}')
+
+        if successful_benchmarks:
+            print_and_log('✅ Successful benchmarks of runtimes/pallets:')
+            for runtime, pallets in successful_benchmarks.items():
+                print_and_log(f'-- {runtime}: {pallets}')
 
     elif args.command == 'fmt':
         command = f"cargo +nightly fmt"