Skip to content

Commit a135687

Browse files
matktmatkt
authored
Add tests for parallel block processing (optimistic and BAL) (besu-eth#10010)
Signed-off-by: Karim Taam <karim.t2am@gmail.com>
1 parent 7f9ab48 commit a135687

17 files changed

+2776
-558
lines changed

‎.github/workflows/acceptance-tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
- name: List all acceptance tests
4848
run: ./gradlew acceptanceTest --test-dry-run -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
4949
- name: Extract current test list
50-
run: mkdir tmp; find . -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v '/testsuite/@name' '{}'; echo ' acceptanceTest'" | tee tmp/currentTests.list
50+
run: mkdir tmp; find . -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v '/testsuite/testcase[1]/@classname' '{}'; echo ' acceptanceTest'" | tee tmp/currentTests.list
5151
- name: Get acceptance test reports
5252
uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d
5353
continue-on-error: true

‎.github/workflows/pre-review.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ jobs:
114114
- name: List unit tests
115115
run: ./gradlew test --test-dry-run -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
116116
- name: Extract current test list
117-
run: mkdir tmp; find . -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v '/testsuite/@name' '{}'; echo '{}' | sed 's#\./\(.*\)/build/test-results/.*# \1#'" | tee tmp/currentTests.list
117+
run: mkdir tmp; find . -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v '/testsuite/testcase[1]/@classname' '{}'; echo '{}' | sed 's#\./\(.*\)/build/test-results/.*# \1#'" | tee tmp/currentTests.list
118118
- name: Get unit test reports
119119
uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d
120120
continue-on-error: true

‎.github/workflows/splitTestsByTime.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ SPLIT_COUNT=$4
2121
SPLIT_INDEX=$5
2222

2323
# extract tests time from Junit XML reports
24-
find "$REPORTS_DIR" -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v 'concat(sum(//testcase/@time), \" \", //testsuite/@name)' '{}'; echo '{}' | sed \"s#${REPORT_STRIP_PREFIX}/\(.*\)/${REPORT_STRIP_SUFFIX}.*# \1#\"" > tmp/timing.tsv
24+
find "$REPORTS_DIR" -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v 'concat(sum(//testcase/@time), \" \", //testsuite/testcase[1]/@classname)' '{}'; echo '{}' | sed \"s#${REPORT_STRIP_PREFIX}/\(.*\)/${REPORT_STRIP_SUFFIX}.*# \1#\"" > tmp/timing.tsv
2525

2626
# Sort times in descending order
2727
IFS=$'\n' sorted=($(sort -nr tmp/timing.tsv))

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/parallelization/MainnetParallelBlockProcessor.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,6 @@ public BlockProcessingResult processBlock(
151151
block,
152152
blockAccessList,
153153
new ParallelTransactionPreprocessing(transactionProcessor, executor, balConfiguration));
154-
155154
if (blockProcessingResult.isFailed()) {
156155
// Fallback to non-parallel processing if there is a block processing exception .
157156
LOG.info(

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/parallelization/ParallelizedConcurrentTransactionProcessor.java‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -216,18 +216,21 @@ public Optional<TransactionProcessingResult> getProcessingResult(
216216
miningBeneficiaryAccount.incrementBalance(reward);
217217
}
218218

219-
final Wei miningBeneficiaryPostBalance = miningBeneficiaryAccount.getBalance();
220-
transactionProcessingResult
221-
.getPartialBlockAccessView()
222-
.ifPresent(
223-
partialBlockAccessView ->
224-
partialBlockAccessView.accountChanges().stream()
225-
.filter(
226-
accountChanges -> accountChanges.getAddress().equals(miningBeneficiary))
227-
.findFirst()
228-
.ifPresent(
229-
accountChanges ->
230-
accountChanges.setPostBalance(miningBeneficiaryPostBalance)));
219+
if (!reward.isZero()) {
220+
final Wei miningBeneficiaryPostBalance = miningBeneficiaryAccount.getBalance();
221+
transactionProcessingResult
222+
.getPartialBlockAccessView()
223+
.ifPresent(
224+
partialBlockAccessView ->
225+
partialBlockAccessView.accountChanges().stream()
226+
.filter(
227+
accountChanges ->
228+
accountChanges.getAddress().equals(miningBeneficiary))
229+
.findFirst()
230+
.ifPresent(
231+
accountChanges ->
232+
accountChanges.setPostBalance(miningBeneficiaryPostBalance)));
233+
}
231234

232235
blockAccumulator.importStateChangesFromSource(transactionAccumulator);
233236

‎ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AbstractContractStorageTest.java‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* Copyright contributors to Hyperledger Besu.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
5+
* the License. You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
10+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
* specific language governing permissions and limitations under the License.
12+
*
13+
* SPDX-License-Identifier: Apache-2.0
14+
*/
15+
package org.hyperledger.besu.ethereum.mainnet.parallelization;
16+
17+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_GENESIS_1;
18+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_GENESIS_1_KEYPAIR;
19+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_GENESIS_2;
20+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_GENESIS_2_KEYPAIR;
21+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.CONTRACT_ADDRESS;
22+
23+
import org.hyperledger.besu.datatypes.Address;
24+
import org.hyperledger.besu.datatypes.Wei;
25+
import org.hyperledger.besu.ethereum.core.Transaction;
26+
27+
import java.util.Optional;
28+
29+
import org.junit.jupiter.api.DisplayName;
30+
import org.junit.jupiter.api.Test;
31+
32+
/** Integration tests for contract storage operations. */
33+
public abstract class AbstractContractStorageTest
34+
extends AbstractParallelBlockProcessorIntegrationTest {
35+
36+
private final Address contractAddr = Address.fromHexStringStrict(CONTRACT_ADDRESS);
37+
38+
@Test
39+
@DisplayName("Writing different storage slots from the same sender produces matching state")
40+
void writeMultipleSlotsSameSender() {
41+
final Transaction txSetSlot1 =
42+
createContractCallTransaction(
43+
0, contractAddr, "setSlot1", ACCOUNT_GENESIS_1_KEYPAIR, Optional.of(100));
44+
final Transaction txSetSlot2 =
45+
createContractCallTransaction(
46+
1, contractAddr, "setSlot2", ACCOUNT_GENESIS_1_KEYPAIR, Optional.of(200));
47+
final Transaction txSetSlot3 =
48+
createContractCallTransaction(
49+
2, contractAddr, "setSlot3", ACCOUNT_GENESIS_1_KEYPAIR, Optional.of(300));
50+
51+
final ComparisonResult result =
52+
executeAndCompare(Wei.of(5), txSetSlot1, txSetSlot2, txSetSlot3);
53+
54+
assertContractStorage(result.seqWorldState(), contractAddr, 0, 100);
55+
assertContractStorage(result.seqWorldState(), contractAddr, 1, 200);
56+
assertContractStorage(result.seqWorldState(), contractAddr, 2, 300);
57+
58+
assertContractStorageMatches(result.seqWorldState(), result.parWorldState(), contractAddr, 0);
59+
assertContractStorageMatches(result.seqWorldState(), result.parWorldState(), contractAddr, 1);
60+
assertContractStorageMatches(result.seqWorldState(), result.parWorldState(), contractAddr, 2);
61+
assertAccountsMatch(
62+
result.seqWorldState(),
63+
result.parWorldState(),
64+
Address.fromHexStringStrict(ACCOUNT_GENESIS_1));
65+
}
66+
67+
@Test
68+
@DisplayName("Writing then reading the same storage slot produces matching state")
69+
void writeThenReadSameSlot() {
70+
final Transaction txSetSlot1 =
71+
createContractCallTransaction(
72+
0, contractAddr, "setSlot1", ACCOUNT_GENESIS_1_KEYPAIR, Optional.of(999));
73+
final Transaction txGetSlot1 =
74+
createContractCallTransaction(
75+
0, contractAddr, "getSlot1", ACCOUNT_GENESIS_2_KEYPAIR, Optional.empty());
76+
77+
final ComparisonResult result = executeAndCompare(Wei.of(5), txSetSlot1, txGetSlot1);
78+
79+
assertContractStorage(result.seqWorldState(), contractAddr, 0, 999);
80+
assertContractStorageMatches(result.seqWorldState(), result.parWorldState(), contractAddr, 0);
81+
assertAccountsMatch(
82+
result.seqWorldState(),
83+
result.parWorldState(),
84+
Address.fromHexStringStrict(ACCOUNT_GENESIS_1));
85+
assertAccountsMatch(
86+
result.seqWorldState(),
87+
result.parWorldState(),
88+
Address.fromHexStringStrict(ACCOUNT_GENESIS_2));
89+
}
90+
91+
@Test
92+
@DisplayName("Reading then writing the same storage slot produces matching state")
93+
void readThenWriteSameSlot() {
94+
final Transaction txGetSlot1 =
95+
createContractCallTransaction(
96+
0, contractAddr, "getSlot1", ACCOUNT_GENESIS_1_KEYPAIR, Optional.empty());
97+
final Transaction txSetSlot1 =
98+
createContractCallTransaction(
99+
0, contractAddr, "setSlot1", ACCOUNT_GENESIS_2_KEYPAIR, Optional.of(777));
100+
101+
final ComparisonResult result = executeAndCompare(Wei.of(5), txGetSlot1, txSetSlot1);
102+
103+
assertContractStorage(result.seqWorldState(), contractAddr, 0, 777);
104+
assertContractStorageMatches(result.seqWorldState(), result.parWorldState(), contractAddr, 0);
105+
}
106+
107+
@Test
108+
@DisplayName("Write-read-write across two senders produces matching state")
109+
void writeReadWriteAcrossSenders() {
110+
final Transaction txSetSlot1 =
111+
createContractCallTransaction(
112+
0, contractAddr, "setSlot1", ACCOUNT_GENESIS_1_KEYPAIR, Optional.of(100));
113+
final Transaction txGetSlot1 =
114+
createContractCallTransaction(
115+
0, contractAddr, "getSlot1", ACCOUNT_GENESIS_2_KEYPAIR, Optional.empty());
116+
final Transaction txSetSlot2 =
117+
createContractCallTransaction(
118+
1, contractAddr, "setSlot2", ACCOUNT_GENESIS_1_KEYPAIR, Optional.of(200));
119+
final Transaction txSetSlot3 =
120+
createContractCallTransaction(
121+
2, contractAddr, "setSlot3", ACCOUNT_GENESIS_1_KEYPAIR, Optional.of(300));
122+
123+
final ComparisonResult result =
124+
executeAndCompare(Wei.of(5), txSetSlot1, txGetSlot1, txSetSlot2, txSetSlot3);
125+
126+
assertContractStorage(result.seqWorldState(), contractAddr, 0, 100);
127+
assertContractStorage(result.seqWorldState(), contractAddr, 1, 200);
128+
assertContractStorage(result.seqWorldState(), contractAddr, 2, 300);
129+
130+
for (int slot = 0; slot <= 2; slot++) {
131+
assertContractStorageMatches(
132+
result.seqWorldState(), result.parWorldState(), contractAddr, slot);
133+
}
134+
assertAccountsMatch(
135+
result.seqWorldState(),
136+
result.parWorldState(),
137+
Address.fromHexStringStrict(ACCOUNT_GENESIS_1));
138+
assertAccountsMatch(
139+
result.seqWorldState(),
140+
result.parWorldState(),
141+
Address.fromHexStringStrict(ACCOUNT_GENESIS_2));
142+
}
143+
144+
@Test
145+
@DisplayName("Storage reads from different senders produce matching state")
146+
void readFromDifferentSenders() {
147+
final Transaction txGetSlot1A =
148+
createContractCallTransaction(
149+
0, contractAddr, "getSlot1", ACCOUNT_GENESIS_1_KEYPAIR, Optional.empty());
150+
final Transaction txGetSlot1B =
151+
createContractCallTransaction(
152+
0, contractAddr, "getSlot1", ACCOUNT_GENESIS_2_KEYPAIR, Optional.empty());
153+
154+
final ComparisonResult result = executeAndCompare(Wei.of(5), txGetSlot1A, txGetSlot1B);
155+
156+
assertContractStorageMatches(result.seqWorldState(), result.parWorldState(), contractAddr, 0);
157+
assertAccountsMatch(
158+
result.seqWorldState(),
159+
result.parWorldState(),
160+
Address.fromHexStringStrict(ACCOUNT_GENESIS_1));
161+
assertAccountsMatch(
162+
result.seqWorldState(),
163+
result.parWorldState(),
164+
Address.fromHexStringStrict(ACCOUNT_GENESIS_2));
165+
}
166+
}

‎ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AbstractMiningBeneficiaryBalTest.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright contributors to Hyperledger Besu.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
5+
* the License. You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
10+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
* specific language governing permissions and limitations under the License.
12+
*
13+
* SPDX-License-Identifier: Apache-2.0
14+
*/
15+
package org.hyperledger.besu.ethereum.mainnet.parallelization;
16+
17+
import static org.assertj.core.api.Assertions.assertThat;
18+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_2;
19+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_3;
20+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_4;
21+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_GENESIS_1_KEYPAIR;
22+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_GENESIS_2_KEYPAIR;
23+
import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.MINING_BENEFICIARY;
24+
25+
import org.hyperledger.besu.datatypes.Wei;
26+
import org.hyperledger.besu.ethereum.core.Transaction;
27+
import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList;
28+
import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList.BalanceChange;
29+
30+
import java.util.List;
31+
import java.util.Optional;
32+
33+
import org.junit.jupiter.api.DisplayName;
34+
import org.junit.jupiter.api.Test;
35+
36+
/** Integration tests for mining beneficiary BAL balance tracking. */
37+
public abstract class AbstractMiningBeneficiaryBalTest
38+
extends AbstractParallelBlockProcessorIntegrationTest {
39+
40+
@Test
41+
@DisplayName(
42+
"Parallel BAL must record correct cumulative mining beneficiary postBalance with priority fees")
43+
void miningBeneficiaryPostBalanceWithPriorityFees() {
44+
// Two independent transfers with non-zero priority fees.
45+
// baseFee=1 so effectivePriorityFee = min(maxPriorityFeePerGas, maxFeePerGas - baseFee) > 0.
46+
// The mining beneficiary accumulates the priority fee from each transaction.
47+
final Transaction tx1 =
48+
createTransferTransaction(
49+
0, 1_000_000_000_000_000_000L, 300_000L, 2L, 10L, ACCOUNT_2, ACCOUNT_GENESIS_1_KEYPAIR);
50+
final Transaction tx2 =
51+
createTransferTransaction(
52+
0, 2_000_000_000_000_000_000L, 300_000L, 3L, 10L, ACCOUNT_3, ACCOUNT_GENESIS_2_KEYPAIR);
53+
54+
final ComparisonResult result = executeAndCompare(Wei.of(1), tx1, tx2);
55+
56+
final Optional<BlockAccessList> seqBal = getBlockAccessList(result.seqResult());
57+
final Optional<BlockAccessList> parBal = getBlockAccessList(result.parResult());
58+
assertThat(seqBal).as("Sequential BAL should be present").isPresent();
59+
assertThat(parBal).as("Parallel BAL should be present").isPresent();
60+
61+
final List<BalanceChange> seqBalChanges =
62+
getBalanceChangesFor(seqBal.get(), MINING_BENEFICIARY);
63+
final List<BalanceChange> parBalChanges =
64+
getBalanceChangesFor(parBal.get(), MINING_BENEFICIARY);
65+
66+
assertThat(seqBalChanges)
67+
.as("Sequential BAL should have balance changes for mining beneficiary")
68+
.isNotEmpty();
69+
assertThat(parBalChanges)
70+
.as("Parallel BAL balance changes for mining beneficiary must match sequential")
71+
.isEqualTo(seqBalChanges);
72+
}
73+
74+
@Test
75+
@DisplayName(
76+
"Multiple transactions with varying priority fees produce correct cumulative beneficiary balances")
77+
void multipleTxsWithVaryingPriorityFees() {
78+
// Three transactions: two from sender A (nonces 0,1) and one from sender B (nonce 0),
79+
// each with a different priority fee. baseFee=1 ensures positive effective priority fees.
80+
final Transaction tx1 =
81+
createTransferTransaction(
82+
0, 100_000_000_000_000L, 300_000L, 1L, 10L, ACCOUNT_2, ACCOUNT_GENESIS_1_KEYPAIR);
83+
final Transaction tx2 =
84+
createTransferTransaction(
85+
0, 200_000_000_000_000L, 300_000L, 4L, 10L, ACCOUNT_3, ACCOUNT_GENESIS_2_KEYPAIR);
86+
final Transaction tx3 =
87+
createTransferTransaction(
88+
1, 300_000_000_000_000L, 300_000L, 2L, 10L, ACCOUNT_4, ACCOUNT_GENESIS_1_KEYPAIR);
89+
90+
final ComparisonResult result = executeAndCompare(Wei.of(1), tx1, tx2, tx3);
91+
92+
final Optional<BlockAccessList> seqBal = getBlockAccessList(result.seqResult());
93+
final Optional<BlockAccessList> parBal = getBlockAccessList(result.parResult());
94+
assertThat(seqBal).isPresent();
95+
assertThat(parBal).isPresent();
96+
97+
final List<BalanceChange> seqBalChanges =
98+
getBalanceChangesFor(seqBal.get(), MINING_BENEFICIARY);
99+
final List<BalanceChange> parBalChanges =
100+
getBalanceChangesFor(parBal.get(), MINING_BENEFICIARY);
101+
102+
assertThat(seqBalChanges).isNotEmpty();
103+
assertThat(parBalChanges)
104+
.as("Parallel BAL mining beneficiary balance changes must match sequential")
105+
.isEqualTo(seqBalChanges);
106+
107+
// Each balance change should be strictly increasing (cumulative rewards)
108+
for (int i = 1; i < seqBalChanges.size(); i++) {
109+
assertThat(seqBalChanges.get(i).postBalance())
110+
.as("Balance change at index %d must be >= previous", i)
111+
.isGreaterThanOrEqualTo(seqBalChanges.get(i - 1).postBalance());
112+
}
113+
}
114+
}

0 commit comments

Comments
 (0)