Skip to content

Commit 227998b

Browse files
bshastryclaude
andcommitted
Add test summary reporting to block-test command
The block-test command now tracks and displays a summary of test execution results, including total tests, passed/failed counts, and a list of failed tests with their failure reasons. This improves usability when running large test suites by providing a clear overview of test outcomes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> Signed-off-by: Bhargava Shastry <bhargava.shastry@ethereum.org>
1 parent 7cfe414 commit 227998b

File tree

1 file changed

+79
-5
lines changed

1 file changed

+79
-5
lines changed

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainTestSubCommand.java

Lines changed: 79 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,11 @@
4343
import java.io.InputStreamReader;
4444
import java.nio.file.Path;
4545
import java.util.ArrayList;
46+
import java.util.LinkedHashMap;
4647
import java.util.List;
4748
import java.util.Map;
4849
import java.util.concurrent.TimeUnit;
50+
import java.util.concurrent.atomic.AtomicInteger;
4951

5052
import com.fasterxml.jackson.core.JsonProcessingException;
5153
import com.fasterxml.jackson.databind.JavaType;
@@ -92,6 +94,41 @@ public class BlockchainTestSubCommand implements Runnable {
9294
// picocli does it magically
9395
@Parameters private final List<Path> blockchainTestFiles = new ArrayList<>();
9496

97+
/** Helper class to track test execution results for summary reporting. */
98+
private static class TestResults {
99+
private final AtomicInteger totalTests = new AtomicInteger(0);
100+
private final AtomicInteger passedTests = new AtomicInteger(0);
101+
private final AtomicInteger failedTests = new AtomicInteger(0);
102+
private final Map<String, String> failures = new LinkedHashMap<>();
103+
104+
void recordPass(final String testName) {
105+
totalTests.incrementAndGet();
106+
passedTests.incrementAndGet();
107+
}
108+
109+
void recordFailure(final String testName, final String reason) {
110+
totalTests.incrementAndGet();
111+
failedTests.incrementAndGet();
112+
failures.put(testName, reason);
113+
}
114+
115+
void printSummary(final java.io.PrintWriter out) {
116+
out.println("\n" + "=".repeat(80));
117+
out.println("TEST SUMMARY");
118+
out.println("=".repeat(80));
119+
out.printf("Total tests: %d%n", totalTests.get());
120+
out.printf("Passed: %d%n", passedTests.get());
121+
out.printf("Failed: %d%n", failedTests.get());
122+
123+
if (!failures.isEmpty()) {
124+
out.println("\nFailed tests:");
125+
failures.forEach(
126+
(testName, reason) -> out.printf(" - %s: %s%n", testName, reason));
127+
}
128+
out.println("=".repeat(80));
129+
}
130+
}
131+
95132
/**
96133
* Default constructor for the BlockchainTestSubCommand class. This constructor doesn't take any
97134
* arguments and initializes the parentCommand to null. PicoCLI requires this constructor.
@@ -111,6 +148,7 @@ public void run() {
111148
// presume ethereum mainnet for reference and state tests
112149
SignatureAlgorithmFactory.setDefaultInstance();
113150
final ObjectMapper blockchainTestMapper = JsonUtils.createObjectMapper();
151+
final TestResults results = new TestResults();
114152

115153
final JavaType javaType =
116154
blockchainTestMapper
@@ -132,7 +170,7 @@ public void run() {
132170
if (file.isFile()) {
133171
final Map<String, BlockchainReferenceTestCaseSpec> blockchainTests =
134172
blockchainTestMapper.readValue(file, javaType);
135-
executeBlockchainTest(blockchainTests);
173+
executeBlockchainTest(blockchainTests, results);
136174
} else {
137175
parentCommand.out.println("File not found: " + fileName);
138176
}
@@ -145,23 +183,30 @@ public void run() {
145183
} else {
146184
blockchainTests = blockchainTestMapper.readValue(blockchainTestFile.toFile(), javaType);
147185
}
148-
executeBlockchainTest(blockchainTests);
186+
executeBlockchainTest(blockchainTests, results);
149187
}
150188
}
151189
} catch (final JsonProcessingException jpe) {
152190
parentCommand.out.println("File content error: " + jpe);
153191
} catch (final IOException e) {
154192
System.err.println("Unable to read state file");
155193
e.printStackTrace(System.err);
194+
} finally {
195+
// Always print summary, even if there were errors
196+
if (results.totalTests.get() > 0) {
197+
results.printSummary(parentCommand.out);
198+
}
156199
}
157200
}
158201

159202
private void executeBlockchainTest(
160-
final Map<String, BlockchainReferenceTestCaseSpec> blockchainTests) {
161-
blockchainTests.forEach(this::traceTestSpecs);
203+
final Map<String, BlockchainReferenceTestCaseSpec> blockchainTests,
204+
final TestResults results) {
205+
blockchainTests.forEach((testName, spec) -> traceTestSpecs(testName, spec, results));
162206
}
163207

164-
private void traceTestSpecs(final String test, final BlockchainReferenceTestCaseSpec spec) {
208+
private void traceTestSpecs(
209+
final String test, final BlockchainReferenceTestCaseSpec spec, final TestResults results) {
165210
if (testName != null && !testName.equals(test)) {
166211
parentCommand.out.println("Skipping test: " + test);
167212
return;
@@ -182,6 +227,9 @@ private void traceTestSpecs(final String test, final BlockchainReferenceTestCase
182227
final MutableBlockchain blockchain = spec.getBlockchain();
183228
final ProtocolContext context = spec.getProtocolContext();
184229

230+
boolean testFailed = false;
231+
String failureReason = "";
232+
185233
for (final BlockchainReferenceTestCaseSpec.CandidateBlock candidateBlock :
186234
spec.getCandidateBlocks()) {
187235
if (!candidateBlock.isExecutable()) {
@@ -205,6 +253,13 @@ private void traceTestSpecs(final String test, final BlockchainReferenceTestCase
205253
blockImporter.importBlock(context, block, validationMode, validationMode);
206254
timer.stop();
207255
if (importResult.isImported() != candidateBlock.isValid()) {
256+
testFailed = true;
257+
failureReason =
258+
String.format(
259+
"Block %d (%s) %s",
260+
block.getHeader().getNumber(),
261+
block.getHash(),
262+
importResult.isImported() ? "Failed to be rejected" : "Failed to import");
208263
parentCommand.out.printf(
209264
"Block %d (%s) %s%n",
210265
block.getHeader().getNumber(),
@@ -227,6 +282,13 @@ private void traceTestSpecs(final String test, final BlockchainReferenceTestCase
227282
}
228283
} catch (final RLPException e) {
229284
if (candidateBlock.isValid()) {
285+
testFailed = true;
286+
failureReason =
287+
String.format(
288+
"Block %d (%s) RLP exception: %s",
289+
candidateBlock.getBlock().getHeader().getNumber(),
290+
candidateBlock.getBlock().getHash(),
291+
e.getMessage());
230292
parentCommand.out.printf(
231293
"Block %d (%s) should have imported but had an RLP exception %s%n",
232294
candidateBlock.getBlock().getHeader().getNumber(),
@@ -236,12 +298,24 @@ private void traceTestSpecs(final String test, final BlockchainReferenceTestCase
236298
}
237299
}
238300
if (!blockchain.getChainHeadHash().equals(spec.getLastBlockHash())) {
301+
testFailed = true;
302+
failureReason =
303+
String.format(
304+
"Chain header mismatch, have %s want %s",
305+
blockchain.getChainHeadHash(), spec.getLastBlockHash());
239306
parentCommand.out.printf(
240307
"Chain header mismatch, have %s want %s - %s%n",
241308
blockchain.getChainHeadHash(), spec.getLastBlockHash(), test);
242309
} else {
243310
parentCommand.out.println("Chain import successful - " + test);
244311
}
312+
313+
// Record test result
314+
if (testFailed) {
315+
results.recordFailure(test, failureReason);
316+
} else {
317+
results.recordPass(test);
318+
}
245319
}
246320

247321
void verifyJournaledEVMAccountCompatability(

0 commit comments

Comments
 (0)