Skip to content

Commit e5ae8d2

Browse files
ahamlatfab-10
ahamlat
and
fab-10
authored
Add --cache-last-block-headers flag to cache the last n block headers (#9223)
* Add --cache-last-block-headers flag to cache the last n block headers stored in the blockchain Signed-off-by: Ameziane H. <ameziane.hamlat@consensys.net> Co-authored-by: Fabio Di Fabio <fabio.difabio@consensys.net>
1 parent ddf82e5 commit e5ae8d2

File tree

8 files changed

+138
-44
lines changed

8 files changed

+138
-44
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
### Additions and Improvements
1818
- Implement optional sender balance checks in the layered txpool [#9176](https://github.com/hyperledger/besu/pull/9176)
19+
- Add `--cache-last-block-headers` flag to cache the last n block headers persisted to the blockchain [#9223](https://github.com/hyperledger/besu/pull/9223)
1920
- Manage unexpected exceptions during block creation [#9208](https://github.com/hyperledger/besu/pull/9208)
2021

2122
### Bug fixes

‎app/src/main/java/org/hyperledger/besu/cli/BesuCommand.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,12 @@ void setUserName(final String userName) {
610610
description = "Specifies the number of last blocks to cache (default: ${DEFAULT-VALUE})")
611611
private final Integer numberOfBlocksToCache = 0;
612612

613+
@CommandLine.Option(
614+
names = {"--cache-last-block-headers"},
615+
description =
616+
"Specifies the number of last block headers to cache (default: ${DEFAULT-VALUE})")
617+
private final Integer numberOfBlockHeadersToCache = 0;
618+
613619
@CommandLine.Option(
614620
names = {"--cache-precompiles"},
615621
description = "Specifies whether to cache precompile results (default: ${DEFAULT-VALUE})")
@@ -1821,6 +1827,7 @@ public BesuControllerBuilder setupControllerBuilder() {
18211827
.randomPeerPriority(p2PDiscoveryOptions.randomPeerPriority)
18221828
.chainPruningConfiguration(unstableChainPruningOptions.toDomainObject())
18231829
.cacheLastBlocks(numberOfBlocksToCache)
1830+
.cacheLastBlockHeaders(numberOfBlockHeadersToCache)
18241831
.genesisStateHashCacheEnabled(genesisStateHashCacheEnabled)
18251832
.apiConfiguration(apiConfigurationSupplier.get())
18261833
.besuComponent(besuComponent);

‎app/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
208208
protected Optional<BesuComponent> besuComponent = Optional.empty();
209209

210210
private int numberOfBlocksToCache = 0;
211+
private int numberOfBlockHeadersToCache = 0;
211212

212213
/** whether parallel transaction processing is enabled or not */
213214
protected boolean isParallelTxProcessingEnabled;
@@ -502,6 +503,17 @@ public BesuControllerBuilder cacheLastBlocks(final Integer numberOfBlocksToCache
502503
return this;
503504
}
504505

506+
/**
507+
* Sets the number of block headers to cache.
508+
*
509+
* @param numberOfBlockHeadersToCache the number of block headers to cache
510+
* @return the besu controller builder
511+
*/
512+
public BesuControllerBuilder cacheLastBlockHeaders(final Integer numberOfBlockHeadersToCache) {
513+
this.numberOfBlockHeadersToCache = numberOfBlockHeadersToCache;
514+
return this;
515+
}
516+
505517
/**
506518
* sets the networkConfiguration in the builder
507519
*
@@ -618,7 +630,8 @@ public BesuController build() {
618630
metricsSystem,
619631
reorgLoggingThreshold,
620632
dataDirectory.toString(),
621-
numberOfBlocksToCache);
633+
numberOfBlocksToCache,
634+
numberOfBlockHeadersToCache);
622635
final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader =
623636
besuComponent
624637
.map(BesuComponent::getCachedMerkleTrieLoader)

‎app/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2674,6 +2674,18 @@ public void cacheLastBlocksOptionShouldWork() {
26742674
assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
26752675
}
26762676

2677+
@Test
2678+
public void cacheLastBlockHeadersOptionShouldWork() {
2679+
int numberOfBlockHeaderToCache = 5000;
2680+
parseCommand("--cache-last-block-headers", String.valueOf(numberOfBlockHeaderToCache));
2681+
verify(mockControllerBuilder).cacheLastBlockHeaders(intArgumentCaptor.capture());
2682+
verify(mockControllerBuilder).build();
2683+
2684+
assertThat(intArgumentCaptor.getValue()).isEqualTo(numberOfBlockHeaderToCache);
2685+
assertThat(commandOutput.toString(UTF_8)).isEmpty();
2686+
assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
2687+
}
2688+
26772689
@Test
26782690
public void genesisStateHashCacheEnabledShouldWork() throws IOException {
26792691
final Path genesisFile = createFakeGenesisFile(GENESIS_VALID_JSON);

‎app/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,10 +303,10 @@ public void initMocks() throws Exception {
303303
.thenReturn(mockControllerBuilder);
304304
when(mockControllerBuilder.besuComponent(any())).thenReturn(mockControllerBuilder);
305305
when(mockControllerBuilder.cacheLastBlocks(any())).thenReturn(mockControllerBuilder);
306+
when(mockControllerBuilder.cacheLastBlockHeaders(any())).thenReturn(mockControllerBuilder);
306307
when(mockControllerBuilder.genesisStateHashCacheEnabled(any()))
307308
.thenReturn(mockControllerBuilder);
308309
when(mockControllerBuilder.apiConfiguration(any())).thenReturn(mockControllerBuilder);
309-
310310
when(mockControllerBuilder.build()).thenReturn(mockController);
311311
lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager);
312312
lenient().when(mockController.getProtocolSchedule()).thenReturn(mockProtocolSchedule);

‎app/src/test/resources/everything_config.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ rpc-http-max-request-content-length = 5242880
9494
rpc-max-logs-range=100
9595
json-pretty-print-enabled=false
9696
cache-last-blocks=512
97+
cache-last-block-headers=5000
9798
cache-precompiles=true
9899
rpc-gas-cap = 50000000
99100
rpc-max-trace-filter-range=100

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java‎

Lines changed: 67 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,11 @@ public class DefaultBlockchain implements MutableBlockchain {
8080

8181
private Comparator<BlockHeader> blockChoiceRule;
8282

83-
private final int numberOfBlocksToCache;
84-
private final Optional<Cache<Hash, BlockHeader>> blockHeadersCache;
85-
private final Optional<Cache<Hash, BlockBody>> blockBodiesCache;
86-
private final Optional<Cache<Hash, List<TransactionReceipt>>> transactionReceiptsCache;
87-
private final Optional<Cache<Hash, Difficulty>> totalDifficultyCache;
88-
private final Optional<Cache<Hash, BlockAccessList>> blockAccessListCache;
83+
private Optional<Cache<Hash, BlockHeader>> blockHeadersCache;
84+
private Optional<Cache<Hash, BlockBody>> blockBodiesCache;
85+
private Optional<Cache<Hash, List<TransactionReceipt>>> transactionReceiptsCache;
86+
private Optional<Cache<Hash, Difficulty>> totalDifficultyCache;
87+
private Optional<Cache<Hash, BlockAccessList>> blockAccessListCache;
8988

9089
private Counter gasUsedCounter = NoOpMetricsSystem.NO_OP_COUNTER;
9190
private Counter numberOfTransactionsCounter = NoOpMetricsSystem.NO_OP_COUNTER;
@@ -97,7 +96,7 @@ private DefaultBlockchain(
9796
final BlockchainStorage blockchainStorage,
9897
final MetricsSystem metricsSystem,
9998
final long reorgLoggingThreshold) {
100-
this(genesisBlock, blockchainStorage, metricsSystem, reorgLoggingThreshold, null, 0);
99+
this(genesisBlock, blockchainStorage, metricsSystem, reorgLoggingThreshold, null, 0, 0);
101100
}
102101

103102
private DefaultBlockchain(
@@ -106,7 +105,8 @@ private DefaultBlockchain(
106105
final MetricsSystem metricsSystem,
107106
final long reorgLoggingThreshold,
108107
final String dataDirectory,
109-
final int numberOfBlocksToCache) {
108+
final int numberOfBlocksToCache,
109+
final int numberOfBlockHeadersToCache) {
110110
checkNotNull(genesisBlock);
111111
checkNotNull(blockchainStorage);
112112
checkNotNull(metricsSystem);
@@ -127,40 +127,63 @@ private DefaultBlockchain(
127127

128128
this.reorgLoggingThreshold = reorgLoggingThreshold;
129129
this.blockChoiceRule = heaviestChainBlockChoiceRule;
130-
this.numberOfBlocksToCache = numberOfBlocksToCache;
131130

132-
if (numberOfBlocksToCache != 0) {
133-
blockHeadersCache =
134-
Optional.of(
135-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
131+
initializeCaches(metricsSystem, numberOfBlockHeadersToCache, numberOfBlocksToCache);
132+
createCounters(metricsSystem);
133+
createGauges(metricsSystem);
134+
}
135+
136+
private void initializeCaches(
137+
final MetricsSystem metricsSystem, final int headersCacheSize, final int blocksCacheSize) {
138+
if (headersCacheSize == 0 && blocksCacheSize == 0) {
139+
setAllCachesEmpty();
140+
return;
141+
}
142+
143+
final int headersSize = Math.max(headersCacheSize, blocksCacheSize);
144+
blockHeadersCache =
145+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(headersSize).build());
146+
147+
if (blocksCacheSize != 0) {
136148
blockBodiesCache =
137-
Optional.of(
138-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
149+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(blocksCacheSize).build());
139150
transactionReceiptsCache =
140-
Optional.of(
141-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
151+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(blocksCacheSize).build());
142152
totalDifficultyCache =
143-
Optional.of(
144-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
153+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(blocksCacheSize).build());
145154
blockAccessListCache =
146-
Optional.of(
147-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
148-
metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockHeaders", blockHeadersCache.get());
149-
metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockBodies", blockBodiesCache.get());
150-
metricsSystem.createGuavaCacheCollector(
151-
BLOCKCHAIN, "transactionReceipts", transactionReceiptsCache.get());
152-
metricsSystem.createGuavaCacheCollector(
153-
BLOCKCHAIN, "totalDifficulty", totalDifficultyCache.get());
155+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(blocksCacheSize).build());
156+
registerCacheMetrics(metricsSystem);
154157
} else {
155-
blockHeadersCache = Optional.empty();
156-
blockBodiesCache = Optional.empty();
157-
transactionReceiptsCache = Optional.empty();
158-
totalDifficultyCache = Optional.empty();
159-
blockAccessListCache = Optional.empty();
158+
// Only headers cache is created, rest are empty
159+
setBlockCachesEmpty();
160+
registerHeadersCacheMetrics(metricsSystem);
160161
}
162+
}
161163

162-
createCounters(metricsSystem);
163-
createGauges(metricsSystem);
164+
private void setAllCachesEmpty() {
165+
blockHeadersCache = Optional.empty();
166+
setBlockCachesEmpty();
167+
}
168+
169+
private void setBlockCachesEmpty() {
170+
blockBodiesCache = Optional.empty();
171+
transactionReceiptsCache = Optional.empty();
172+
totalDifficultyCache = Optional.empty();
173+
blockAccessListCache = Optional.empty();
174+
}
175+
176+
private void registerCacheMetrics(final MetricsSystem metricsSystem) {
177+
registerHeadersCacheMetrics(metricsSystem);
178+
metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockBodies", blockBodiesCache.get());
179+
metricsSystem.createGuavaCacheCollector(
180+
BLOCKCHAIN, "transactionReceipts", transactionReceiptsCache.get());
181+
metricsSystem.createGuavaCacheCollector(
182+
BLOCKCHAIN, "totalDifficulty", totalDifficultyCache.get());
183+
}
184+
185+
private void registerHeadersCacheMetrics(final MetricsSystem metricsSystem) {
186+
metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockHeaders", blockHeadersCache.get());
164187
}
165188

166189
private void createCounters(final MetricsSystem metricsSystem) {
@@ -237,6 +260,7 @@ public static MutableBlockchain createMutable(
237260
metricsSystem,
238261
reorgLoggingThreshold,
239262
null,
263+
0,
240264
0);
241265
}
242266

@@ -253,6 +277,7 @@ public static MutableBlockchain createMutable(
253277
metricsSystem,
254278
reorgLoggingThreshold,
255279
dataDirectory,
280+
0,
256281
0);
257282
}
258283

@@ -262,15 +287,17 @@ public static MutableBlockchain createMutable(
262287
final MetricsSystem metricsSystem,
263288
final long reorgLoggingThreshold,
264289
final String dataDirectory,
265-
final int numberOfBlocksToCache) {
290+
final int numberOfBlocksToCache,
291+
final int numberOgBlockHeadersToCache) {
266292
checkNotNull(genesisBlock);
267293
return new DefaultBlockchain(
268294
Optional.of(genesisBlock),
269295
blockchainStorage,
270296
metricsSystem,
271297
reorgLoggingThreshold,
272298
dataDirectory,
273-
numberOfBlocksToCache);
299+
numberOfBlocksToCache,
300+
numberOgBlockHeadersToCache);
274301
}
275302

276303
public static Blockchain create(
@@ -451,7 +478,7 @@ public synchronized void appendBlock(
451478
final Block block,
452479
final List<TransactionReceipt> receipts,
453480
final Optional<BlockAccessList> blockAccessList) {
454-
if (numberOfBlocksToCache != 0) cacheBlockData(block, receipts, blockAccessList);
481+
cacheBlockData(block, receipts, blockAccessList);
455482
appendBlockHelper(new BlockWithReceipts(block, receipts), false, true);
456483
}
457484

@@ -460,7 +487,7 @@ public synchronized void appendBlockWithoutIndexingTransactions(
460487
final Block block,
461488
final List<TransactionReceipt> receipts,
462489
final Optional<BlockAccessList> blockAccessList) {
463-
if (numberOfBlocksToCache != 0) cacheBlockData(block, receipts, blockAccessList);
490+
cacheBlockData(block, receipts, blockAccessList);
464491
appendBlockHelper(new BlockWithReceipts(block, receipts), false, false);
465492
}
466493

@@ -481,7 +508,7 @@ public synchronized void storeBlock(
481508
final Block block,
482509
final List<TransactionReceipt> receipts,
483510
final Optional<BlockAccessList> blockAccessList) {
484-
if (numberOfBlocksToCache != 0) cacheBlockData(block, receipts, blockAccessList);
511+
cacheBlockData(block, receipts, blockAccessList);
485512
appendBlockHelper(new BlockWithReceipts(block, receipts), true, true);
486513
}
487514

@@ -592,6 +619,7 @@ public synchronized void unsafeImportBlock(
592619
final Block block,
593620
final List<TransactionReceipt> transactionReceipts,
594621
final Optional<Difficulty> maybeTotalDifficulty) {
622+
cacheBlockData(block, transactionReceipts, Optional.empty());
595623
final BlockchainStorage.Updater updater = blockchainStorage.updater();
596624
final Hash blockHash = block.getHash();
597625
updater.putBlockHeader(blockHash, block.getHeader());

‎ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -977,7 +977,7 @@ public void testCacheUsedWhenNumberOfBlocksToCacheNotZero() {
977977
final KeyValueStorage kvStoreVariables = new InMemoryKeyValueStorage();
978978
final Block genesisBlock = gen.genesisBlock();
979979
final DefaultBlockchain blockchain =
980-
createMutableBlockchain(kvStore, kvStoreVariables, genesisBlock, "/data/test", 512);
980+
createMutableBlockchain(kvStore, kvStoreVariables, genesisBlock, "/data/test", 512, 0);
981981

982982
final BlockDataGenerator.BlockOptions options =
983983
new BlockDataGenerator.BlockOptions()
@@ -1015,6 +1015,36 @@ public void testCacheUsedWhenNumberOfBlocksToCacheNotZero() {
10151015
.isEqualTo(newBlock.getHeader().getDifficulty());
10161016
}
10171017

1018+
@Test
1019+
public void testCacheUsedWhenNumberOfBlockHeadersToCacheNotZero() {
1020+
final BlockDataGenerator gen = new BlockDataGenerator();
1021+
final KeyValueStorage kvStore = new InMemoryKeyValueStorage();
1022+
final KeyValueStorage kvStoreVariables = new InMemoryKeyValueStorage();
1023+
final Block genesisBlock = gen.genesisBlock();
1024+
final DefaultBlockchain blockchain =
1025+
createMutableBlockchain(kvStore, kvStoreVariables, genesisBlock, "/data/test", 0, 10000);
1026+
1027+
final BlockDataGenerator.BlockOptions options =
1028+
new BlockDataGenerator.BlockOptions()
1029+
.setBlockNumber(1L)
1030+
.setParentHash(genesisBlock.getHash());
1031+
final Block newBlock = gen.block(options);
1032+
final List<TransactionReceipt> receipts = gen.receipts(newBlock);
1033+
1034+
assertThat(blockchain.getBlockHeadersCache()).isNotEmpty();
1035+
assertThat(blockchain.getBlockBodiesCache()).isEmpty();
1036+
assertThat(blockchain.getTransactionReceiptsCache()).isEmpty();
1037+
assertThat(blockchain.getTotalDifficultyCache()).isEmpty();
1038+
1039+
assertThat(blockchain.getBlockHeadersCache().get().size()).isEqualTo(0);
1040+
1041+
blockchain.appendBlock(newBlock, receipts);
1042+
1043+
assertThat(blockchain.getBlockHeadersCache().get().size()).isEqualTo(1);
1044+
assertThat(blockchain.getBlockHeadersCache().get().getIfPresent(newBlock.getHash()))
1045+
.isEqualTo(newBlock.getHeader());
1046+
}
1047+
10181048
/*
10191049
* Check that block header, block body, block number, transaction locations, and receipts for this
10201050
* block are all stored.
@@ -1106,15 +1136,17 @@ private DefaultBlockchain createMutableBlockchain(
11061136
final KeyValueStorage kvStorageVariables,
11071137
final Block genesisBlock,
11081138
final String dataDirectory,
1109-
final int numberOfBlocksToCache) {
1139+
final int numberOfBlocksToCache,
1140+
final int numberOfBlockHeadersToCache) {
11101141
return (DefaultBlockchain)
11111142
DefaultBlockchain.createMutable(
11121143
genesisBlock,
11131144
createStorage(kvStore, kvStorageVariables),
11141145
new NoOpMetricsSystem(),
11151146
0,
11161147
dataDirectory,
1117-
numberOfBlocksToCache);
1148+
numberOfBlocksToCache,
1149+
numberOfBlockHeadersToCache);
11181150
}
11191151

11201152
@Test

0 commit comments

Comments
 (0)