Add --attempt-cache-bust to evmtool benchmark subcommand

CHANGELOG.md

@@ -13,9 +13,12 @@
 - Improve transaction simulation and gas estimation when no gas pricing is present [#8888](https://github.com/hyperledger/besu/pull/8888)
 - Add option to trace reference tests during execution [#8878](https://github.com/hyperledger/besu/pull/8878)
 - Expose methods to query hardfork by block header or for the next block in the Plugin API [#8909](https://github.com/hyperledger/besu/pull/8909)
-- Enable decoding for large RPC requests [#8877](https://github.com/hyperledger/besu/pull/8877)
 - Generate distribution dependencies catalog [#8987](https://github.com/hyperledger/besu/pull/8987)
+
+#### Performance
 - Improve the sync performance by not RLP decoding bodies during sync. This means we are using less memory and CPU, allowing us to increase the parallelism of the download pipeline, which has been increased from 4 to 8. Can be reduced again with  `--Xsynchronizer-downloader-parallelism=4` [#8959]
+- Enable decoding for large RPC requests [#8877](https://github.com/hyperledger/besu/pull/8877)
+- Add --attempt-cache-bust to evmtool benchmark subcommand [#8985](https://github.com/hyperledger/besu/pull/8985)
 
 #### Fusaka devnets
 - EIP-7910 - `eth_config` JSON-RPC Method [#8417](https://github.com/hyperledger/besu/pull/8417), [#8946](https://github.com/hyperledger/besu/pull/8946)

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BenchmarkSubCommand.java

@@ -141,6 +141,14 @@ enum Benchmark {
       scope = LOCAL)
   Optional<Integer> warmTime = Optional.empty();
 
+  @Option(
+      names = {"--attempt-cache-bust"},
+      description =
+          "Run each test case within each warmup and exec iteration. This attempts to warm the code without warming the data, i.e. avoid warming CPU caches. Benchmark must have sufficient number and variety of test cases to be effective. --warm-time, --exec-time and --async-profiler are ignored.",
+      scope = LOCAL,
+      negatable = true)
+  Boolean attemptCacheBust = false;
+
   @Parameters(description = "One or more of ${COMPLETION-CANDIDATES}.")
   EnumSet<Benchmark> benchmarks = EnumSet.noneOf(Benchmark.class);
 
@@ -177,7 +185,8 @@ public void run() {
             execIterations,
             execTime,
             warmIterations,
-            warmTime);
+            warmTime,
+            attemptCacheBust);
     for (var benchmark : benchmarksToRun) {
       output.println("\nBenchmarks for " + benchmark + " on fork " + parentCommand.getFork());
       BenchmarkExecutor executor = benchmark.executorBuilder.create(output, benchmarkConfig);

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/AltBN128Benchmark.java

@@ -22,7 +22,7 @@
 
 import java.io.PrintStream;
 import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.SequencedMap;
 
 import org.apache.tuweni.bytes.Bytes;
 
@@ -57,21 +57,36 @@ public void runBenchmark(final Boolean attemptNative, final String fork) {
   }
 
   private void benchmarkAdd(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
     testCases.put(
-        "add",
+        "EcAdd",
         Bytes.fromHexString(
             "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa9"
                 + "01e0559bacb160664764a357af8a9fe70baa9258e0b959273ffc5718c6d4cc7c"
                 + "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa9"
                 + "2e83f8d734803fc370eba25ed1f6b8768bd6d83887b87165fc2434fe11a830cb"));
     testCases.put(
-        "AddMarius",
+        "EcAddMarius",
         Bytes.fromHexString(
             "1581d9d4d7eb0e3cdc75739f7098479097d579573f23e70b07cbd40a"
                 + "5d97bbc118da69b1d2cb7b89fdb3bba2524bf135453ce828faea190d8f4d48d572cbe3fe20"
                 + "34b5942fbdd612f2553a9fd9fa5eb4c3e5ce6a34ed100f62de0e380f4e67f8016027893e1f"
                 + "5082bdc48651667776e2b1e32a85edd33ff430eef15e8e68b3d460"));
+    testCases.put(
+        "EcAddAmez1",
+        Bytes.fromHexString(
+            "1f69350cfea1bdf51ecec3a0bdf57732411418502904eb86c10901af310b9ca0291547261fc8bbcf961534f3f3d282d4ab3bfa0b9f3b5ecb3e575d42fd13e25c"
+                + "2034b5942fbdd612f2553a9fd9fa5eb4c3e5ce6a34ed100f62de0e380f4e67f8016027893e1f5082bdc48651667776e2b1e32a85edd33ff430eef15e8e68b3d4"));
+    testCases.put(
+        "EcAddAmez2",
+        Bytes.fromHexString(
+            "03adc0948243eac87e40fd1d4a3b5693c651abd30a88b93f3dbe3f571791c4120e9f7fa4c9ae503399c7a4f0ee0e91961b759d94e49c8ea0d04b448dbc6f3d16"
+                + "2034b5942fbdd612f2553a9fd9fa5eb4c3e5ce6a34ed100f62de0e380f4e67f8016027893e1f5082bdc48651667776e2b1e32a85edd33ff430eef15e8e68b3d4"));
+    testCases.put(
+        "EcAddAmez3",
+        Bytes.fromHexString(
+            "012b7638324563dc328b870d414807ed27426354bc83f22c42690f717ae7137e19dffc74cd0183d631cb39f5f58d2846744119913519373ebb95f2654a989390"
+                + "2034b5942fbdd612f2553a9fd9fa5eb4c3e5ce6a34ed100f62de0e380f4e67f8016027893e1f5082bdc48651667776e2b1e32a85edd33ff430eef15e8e68b3d4"));
 
     PrecompiledContract contract =
         EvmSpec.evmSpec(forkVersion).getPrecompileContractRegistry().get(Address.ALTBN128_ADD);
@@ -80,7 +95,7 @@ private void benchmarkAdd(final EvmSpecVersion forkVersion) {
   }
 
   private void benchmarkMul(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
     testCases.put(
         "mul1",
         Bytes.fromHexString(
@@ -102,7 +117,7 @@ private void benchmarkMul(final EvmSpecVersion forkVersion) {
   }
 
   private void benchmarkPairings(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
     testCases.put(
         "2 pairings",
         Bytes.fromHexString(

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BLS12Benchmark.java

@@ -22,7 +22,7 @@
 
 import java.io.PrintStream;
 import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.SequencedMap;
 
 import org.apache.tuweni.bytes.Bytes;
 
@@ -172,7 +172,7 @@ public void runBenchmark(final Boolean attemptNative, final String fork) {
   }
 
   private void benchmarkG1Add(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
     for (int i = 0; i < g1PointPairs.length - 1; i++) {
       testCases.put("G1 Add " + i, Bytes.fromHexString(g1PointPairs[i] + g1PointPairs[i + 1]));
     }
@@ -184,7 +184,7 @@ private void benchmarkG1Add(final EvmSpecVersion forkVersion) {
   }
 
   private void benchmarkG1MultiExp32Pairs(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
 
     // add test cases for 2, 4, 8, 16, and 32 point/scalar pairs
     for (int i = 1; i <= 5; i++) {
@@ -202,7 +202,7 @@ private void benchmarkG1MultiExp32Pairs(final EvmSpecVersion forkVersion) {
   }
 
   private void benchmarkMapFpToG1(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
     for (int i = 0; i < g1PointPairs.length; i++) {
       testCases.put("Map Fp to G1 " + i, Bytes.fromHexString(g1PointPairs[i].substring(0, 128)));
     }
@@ -216,7 +216,7 @@ private void benchmarkMapFpToG1(final EvmSpecVersion forkVersion) {
   }
 
   private void benchmarkG2Add(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
     for (int i = 0; i < g2PointPairs.length - 1; i++) {
       testCases.put("G2 Add " + i, Bytes.fromHexString(g2PointPairs[i] + g2PointPairs[i + 1]));
     }
@@ -228,7 +228,7 @@ private void benchmarkG2Add(final EvmSpecVersion forkVersion) {
   }
 
   private void benchmarkG2MultiExp32Pairs(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
 
     // add test cases for 2, 4, 8, 16, and 32 point/scalar pairs
     for (int i = 1; i <= 5; i++) {
@@ -246,7 +246,7 @@ private void benchmarkG2MultiExp32Pairs(final EvmSpecVersion forkVersion) {
   }
 
   private void benchmarkMapFp2ToG2(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
     for (int i = 0; i < g2PointPairs.length; i++) {
       testCases.put("Map Fp2 to G2 " + i, Bytes.fromHexString(g2PointPairs[i].substring(0, 256)));
     }
@@ -260,7 +260,7 @@ private void benchmarkMapFp2ToG2(final EvmSpecVersion forkVersion) {
   }
 
   private void benchmarkBlsPairing(final EvmSpecVersion forkVersion) {
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
 
     // add test cases for 2, 4, 8, 16, and 32 point/scalar pairs
     for (int i = 1; i <= 5; i++) {

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkConfig.java

@@ -27,6 +27,7 @@
  * @param execTime run for an unbounded amount of time.
  * @param warmIterations warm up for an unbounded number of iterations.
  * @param warmTime warm up for an unbounded amount of time.
+ * @param attemptCacheBust if true, run each test case within each iteration
  */
 public record BenchmarkConfig(
     boolean useNative,
@@ -36,4 +37,5 @@ public record BenchmarkConfig(
     Optional<Integer> execIterations,
     Optional<Integer> execTime,
     Optional<Integer> warmIterations,
-    Optional<Integer> warmTime) {}
+    Optional<Integer> warmTime,
+    boolean attemptCacheBust) {}

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkExecutor.java

@@ -38,8 +38,12 @@
 import org.hyperledger.besu.evm.precompile.PrecompiledContract;
 
 import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
+import java.util.SequencedMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
@@ -54,8 +58,8 @@ public abstract class BenchmarkExecutor {
   private static final int MAX_WARMUP_TIME_IN_SECONDS = 3;
   private static final long GAS_PER_SECOND_STANDARD = 100_000_000L;
 
-  static final int MATH_WARMUP = 100_000;
-  static final int MATH_ITERATIONS = 100_000;
+  static final int MATH_WARMUP = 20_000;
+  static final int MATH_ITERATIONS = 1_000;
 
   /** Where to write the output of the benchmarks. */
   protected final PrintStream output;
@@ -129,10 +133,19 @@ public BenchmarkExecutor(
                             : Integer.MAX_VALUE));
     this.output = output;
     this.precompileTableHeader =
-        () ->
-            output.printf(
-                "%-30s | %12s | %12s | %15s | %15s%n",
-                "", "Actual cost", "Derived Cost", "Iteration time", "Throughput");
+        () -> {
+          if (benchmarkConfig.attemptCacheBust())
+            output.println("--attempt-cache-bust=true (--warm-time and --exec-time ignored)");
+          output.printf("--warm-iterations=%d%n", warmIterations);
+          if (!benchmarkConfig.attemptCacheBust())
+            output.printf("--warm-time=%ss%n", warmTimeInNano / 1.0e9D);
+          output.printf("--exec-iterations=%d%n", execIterations);
+          if (!benchmarkConfig.attemptCacheBust())
+            output.printf("--exec-time=%ss%n", execTimeInNano / 1.0e9D);
+          output.printf(
+              "%-30s | %12s | %12s | %15s | %15s%n",
+              "", "Actual cost", "Derived Cost", "Iteration time", "Throughput");
+        };
     this.config = benchmarkConfig;
     assert warmIterations <= 0;
     assert execIterations <= 0;
@@ -147,7 +160,7 @@ public BenchmarkExecutor(
    * @param evmSpecVersion EVM specification version to run the precompile for.
    */
   public void precompile(
-      final Map<String, Bytes> testCases,
+      final SequencedMap<String, Bytes> testCases,
       final PrecompiledContract contract,
       final EvmSpecVersion evmSpecVersion) {
 
@@ -156,15 +169,71 @@ public void precompile(
           "contract is unsupported on " + evmSpecVersion + " fork");
     }
 
+    Optional<Pattern> maybePattern = config.testCasePattern().map(Pattern::compile);
+    LinkedHashMap<String, Bytes> filteredTestCases = new LinkedHashMap<>();
+    testCases.forEach(
+        (k, v) -> {
+          if (maybePattern.map(p -> p.matcher(k).find()).orElse(true)) {
+            filteredTestCases.put(k, v);
+          }
+        });
+
+    if (config.attemptCacheBust()) {
+      runPrecompileAttemptCacheBust(filteredTestCases, contract);
+    } else {
+      runPrecompile(filteredTestCases, contract);
+    }
+  }
+
+  private void runPrecompileAttemptCacheBust(
+      final Map<String, Bytes> testCases, final PrecompiledContract contract) {
+
+    // Warmup all test cases in serial inside one warmup iteration
+    // avoid using warmTime as it is now dependent on the number of test cases
+    for (int i = 0; i < warmIterations; i++) {
+      for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
+        contract.computePrecompile(testCase.getValue(), fakeFrame);
+      }
+    }
+
+    // Also run all test cases in serial inside one iteration
+    Map<String, Long> totalElapsedByTestName = new HashMap<>();
+    int executions = 0;
+    while (executions < execIterations) {
+      for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
+        final long iterationStart = System.nanoTime();
+        final var result = contract.computePrecompile(testCase.getValue(), fakeFrame);
+        final long iterationElapsed = System.nanoTime() - iterationStart;
+        if (result.output() != null) {
+          // adds iterationElapsed if absent, or sums with existing value
+          totalElapsedByTestName.merge(testCase.getKey(), iterationElapsed, Long::sum);
+        } else {
+          throw new IllegalArgumentException("Input is Invalid for " + testCase.getValue());
+        }
+      }
+      executions++;
+    }
+
     for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
-      if (config.testCasePattern().isPresent()
-          && !Pattern.compile(config.testCasePattern().get()).matcher(testCase.getKey()).find()) {
-        continue;
+      if (totalElapsedByTestName.containsKey(testCase.getKey())) {
+        final double execTime =
+            totalElapsedByTestName.get(testCase.getKey()) / 1.0e9D / execIterations;
+        // log the performance of the precompile
+        long gasCost = contract.gasRequirement(testCases.get(testCase.getKey()));
+        logPrecompilePerformance(testCase.getKey(), gasCost, execTime);
+      } else {
+        output.printf("%s Test case missing from results%n", testCase.getKey());
       }
+    }
+  }
 
+  private void runPrecompile(
+      final Map<String, Bytes> testCases, final PrecompiledContract contract) {
+
+    // Fully warmup and execute, test case by test case
+    for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
       final double execTime =
           runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), contract);
-
       long gasCost = contract.gasRequirement(testCase.getValue());
       logPrecompilePerformance(testCase.getKey(), gasCost, execTime);
     }
@@ -181,14 +250,18 @@ public void precompile(
   protected double runPrecompileBenchmark(
       final String testName, final Bytes arg, final PrecompiledContract contract) {
     if (contract.computePrecompile(arg, fakeFrame).output() == null) {
-      throw new RuntimeException("Input is Invalid");
+      throw new IllegalArgumentException("Input is Invalid for " + testName);
     }
 
-    long startNanoTime = System.nanoTime();
-    for (int i = 0; i < warmIterations && System.nanoTime() - startNanoTime < warmTimeInNano; i++) {
+    // Warmup individual test case fully, which may have side effect of warming cpu caches
+    long startWarmNanoTime = System.nanoTime();
+    for (int i = 0;
+        i < warmIterations && System.nanoTime() - startWarmNanoTime < warmTimeInNano;
+        i++) {
       contract.computePrecompile(arg, fakeFrame);
     }
 
+    // Iterations
     final AtomicReference<AsyncProfiler> asyncProfiler = new AtomicReference<>();
     config
         .asyncProfilerOptions()
@@ -206,7 +279,6 @@ protected double runPrecompileBenchmark(
 
     int executions = 0;
     long totalElapsed = 0;
-
     while (executions < execIterations && totalElapsed < execTimeInNano) {
       long iterationStart = System.nanoTime();
       contract.computePrecompile(arg, fakeFrame);
@@ -224,7 +296,7 @@ protected double runPrecompileBenchmark(
       }
     }
 
-    return (totalElapsed / 1.0e9D) / executions;
+    return totalElapsed / 1.0e9D / executions;
   }
 
   /**

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/ECRecoverBenchmark.java

@@ -24,6 +24,7 @@
 import java.io.PrintStream;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.SequencedMap;
 
 import org.apache.tuweni.bytes.Bytes;
 
@@ -44,7 +45,7 @@ public ECRecoverBenchmark(final PrintStream output, final BenchmarkConfig benchm
   public void runBenchmark(final Boolean attemptNative, final String fork) {
     EvmSpecVersion evmSpecVersion = EvmSpecVersion.fromName(fork);
 
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
     testCases.put(
         "0x0c65a9d9ffc02c7c99e36e32ce0f950c7804ceda",
         Bytes.fromHexString(

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/KZGPointEvalBenchmark.java

@@ -22,7 +22,7 @@
 
 import java.io.PrintStream;
 import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.SequencedMap;
 
 import org.apache.tuweni.bytes.Bytes;
 
@@ -47,7 +47,7 @@ public void runBenchmark(final Boolean attemptNative, final String fork) {
     }
     output.println("Native KZGPointEval");
 
-    final Map<String, Bytes> testCases = new LinkedHashMap<>();
+    final SequencedMap<String, Bytes> testCases = new LinkedHashMap<>();
     testCases.put(
         "kzg-verify",
         Bytes.fromHexString(