Skip to content

Commit 70ffb41

Browse files
ahamlatahamlat
committed
Add --cache-last-block-headers flag to cache the last n block headers stored in the blockchain
Signed-off-by: Ameziane H. <ameziane.hamlat@consensys.net>
1 parent 04e1f09 commit 70ffb41

File tree

7 files changed

+148
-42
lines changed

7 files changed

+148
-42
lines changed

‎app/src/main/java/org/hyperledger/besu/cli/BesuCommand.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,12 @@ void setUserName(final String userName) {
610610
description = "Specifies the number of last blocks to cache (default: ${DEFAULT-VALUE})")
611611
private final Integer numberOfBlocksToCache = 0;
612612

613+
@CommandLine.Option(
614+
names = {"--cache-last-block-headers"},
615+
description =
616+
"Specifies the number of last block headers to cache (default: ${DEFAULT-VALUE})")
617+
private final Integer numberOfBlockHeadersToCache = 0;
618+
613619
@CommandLine.Option(
614620
names = {"--cache-precompiles"},
615621
description = "Specifies whether to cache precompile results (default: ${DEFAULT-VALUE})")
@@ -1821,6 +1827,7 @@ public BesuControllerBuilder setupControllerBuilder() {
18211827
.randomPeerPriority(p2PDiscoveryOptions.randomPeerPriority)
18221828
.chainPruningConfiguration(unstableChainPruningOptions.toDomainObject())
18231829
.cacheLastBlocks(numberOfBlocksToCache)
1830+
.cacheLastBlockHeaders(numberOfBlockHeadersToCache)
18241831
.genesisStateHashCacheEnabled(genesisStateHashCacheEnabled)
18251832
.apiConfiguration(apiConfigurationSupplier.get())
18261833
.besuComponent(besuComponent);

‎app/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
208208
protected Optional<BesuComponent> besuComponent = Optional.empty();
209209

210210
private int numberOfBlocksToCache = 0;
211+
private int numberOfBlockHeadersToCache = 0;
211212

212213
/** whether parallel transaction processing is enabled or not */
213214
protected boolean isParallelTxProcessingEnabled;
@@ -502,6 +503,17 @@ public BesuControllerBuilder cacheLastBlocks(final Integer numberOfBlocksToCache
502503
return this;
503504
}
504505

506+
/**
507+
* Sets the number of block headers to cache.
508+
*
509+
* @param numberOfBlockHeadersToCache the number of block headers to cache
510+
* @return the besu controller builder
511+
*/
512+
public BesuControllerBuilder cacheLastBlockHeaders(final Integer numberOfBlockHeadersToCache) {
513+
this.numberOfBlockHeadersToCache = numberOfBlockHeadersToCache;
514+
return this;
515+
}
516+
505517
/**
506518
* sets the networkConfiguration in the builder
507519
*
@@ -618,7 +630,8 @@ public BesuController build() {
618630
metricsSystem,
619631
reorgLoggingThreshold,
620632
dataDirectory.toString(),
621-
numberOfBlocksToCache);
633+
numberOfBlocksToCache,
634+
numberOfBlockHeadersToCache);
622635
final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader =
623636
besuComponent
624637
.map(BesuComponent::getCachedMerkleTrieLoader)

‎app/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2674,6 +2674,18 @@ public void cacheLastBlocksOptionShouldWork() {
26742674
assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
26752675
}
26762676

2677+
@Test
2678+
public void cacheLastBlockHeadersOptionShouldWork() {
2679+
int numberOfBlockHeaderToCache = 5000;
2680+
parseCommand("--cache-last-block-headers", String.valueOf(numberOfBlockHeaderToCache));
2681+
verify(mockControllerBuilder).cacheLastBlocks(intArgumentCaptor.capture());
2682+
verify(mockControllerBuilder).build();
2683+
2684+
assertThat(intArgumentCaptor.getValue()).isEqualTo(numberOfBlockHeaderToCache);
2685+
assertThat(commandOutput.toString(UTF_8)).isEmpty();
2686+
assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
2687+
}
2688+
26772689
@Test
26782690
public void genesisStateHashCacheEnabledShouldWork() throws IOException {
26792691
final Path genesisFile = createFakeGenesisFile(GENESIS_VALID_JSON);

‎app/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -292,10 +292,10 @@ public void initMocks() throws Exception {
292292
.thenReturn(mockControllerBuilder);
293293
when(mockControllerBuilder.besuComponent(any())).thenReturn(mockControllerBuilder);
294294
when(mockControllerBuilder.cacheLastBlocks(any())).thenReturn(mockControllerBuilder);
295+
when(mockControllerBuilder.cacheLastBlockHeaders(any())).thenReturn(mockControllerBuilder);
295296
when(mockControllerBuilder.genesisStateHashCacheEnabled(any()))
296297
.thenReturn(mockControllerBuilder);
297298
when(mockControllerBuilder.apiConfiguration(any())).thenReturn(mockControllerBuilder);
298-
299299
when(mockControllerBuilder.build()).thenReturn(mockController);
300300
lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager);
301301
lenient().when(mockController.getProtocolSchedule()).thenReturn(mockProtocolSchedule);

‎app/src/test/resources/everything_config.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ rpc-http-max-request-content-length = 5242880
9494
rpc-max-logs-range=100
9595
json-pretty-print-enabled=false
9696
cache-last-blocks=512
97+
cache-last-block-headers=5000
9798
cache-precompiles=true
9899
rpc-gas-cap = 50000000
99100
rpc-max-trace-filter-range=100

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java‎

Lines changed: 78 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -81,11 +81,12 @@ public class DefaultBlockchain implements MutableBlockchain {
8181
private Comparator<BlockHeader> blockChoiceRule;
8282

8383
private final int numberOfBlocksToCache;
84-
private final Optional<Cache<Hash, BlockHeader>> blockHeadersCache;
85-
private final Optional<Cache<Hash, BlockBody>> blockBodiesCache;
86-
private final Optional<Cache<Hash, List<TransactionReceipt>>> transactionReceiptsCache;
87-
private final Optional<Cache<Hash, Difficulty>> totalDifficultyCache;
88-
private final Optional<Cache<Hash, BlockAccessList>> blockAccessListCache;
84+
private final int numberOfBlockHeadersToCache;
85+
private Optional<Cache<Hash, BlockHeader>> blockHeadersCache;
86+
private Optional<Cache<Hash, BlockBody>> blockBodiesCache;
87+
private Optional<Cache<Hash, List<TransactionReceipt>>> transactionReceiptsCache;
88+
private Optional<Cache<Hash, Difficulty>> totalDifficultyCache;
89+
private Optional<Cache<Hash, BlockAccessList>> blockAccessListCache;
8990

9091
private Counter gasUsedCounter = NoOpMetricsSystem.NO_OP_COUNTER;
9192
private Counter numberOfTransactionsCounter = NoOpMetricsSystem.NO_OP_COUNTER;
@@ -97,7 +98,7 @@ private DefaultBlockchain(
9798
final BlockchainStorage blockchainStorage,
9899
final MetricsSystem metricsSystem,
99100
final long reorgLoggingThreshold) {
100-
this(genesisBlock, blockchainStorage, metricsSystem, reorgLoggingThreshold, null, 0);
101+
this(genesisBlock, blockchainStorage, metricsSystem, reorgLoggingThreshold, null, 0, 0);
101102
}
102103

103104
private DefaultBlockchain(
@@ -106,7 +107,8 @@ private DefaultBlockchain(
106107
final MetricsSystem metricsSystem,
107108
final long reorgLoggingThreshold,
108109
final String dataDirectory,
109-
final int numberOfBlocksToCache) {
110+
final int numberOfBlocksToCache,
111+
final int numberOfBlockHeadersToCache) {
110112
checkNotNull(genesisBlock);
111113
checkNotNull(blockchainStorage);
112114
checkNotNull(metricsSystem);
@@ -128,39 +130,70 @@ private DefaultBlockchain(
128130
this.reorgLoggingThreshold = reorgLoggingThreshold;
129131
this.blockChoiceRule = heaviestChainBlockChoiceRule;
130132
this.numberOfBlocksToCache = numberOfBlocksToCache;
133+
this.numberOfBlockHeadersToCache = numberOfBlockHeadersToCache;
131134

132-
if (numberOfBlocksToCache != 0) {
133-
blockHeadersCache =
134-
Optional.of(
135-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
135+
initializeCaches(metricsSystem, numberOfBlockHeadersToCache, numberOfBlocksToCache);
136+
createCounters(metricsSystem);
137+
createGauges(metricsSystem);
138+
}
139+
140+
private void initializeCaches(
141+
final MetricsSystem metricsSystem, final int headersCacheSize, final int blocksCacheSize) {
142+
if (headersCacheSize == 0 && blocksCacheSize == 0) {
143+
setAllCachesEmpty();
144+
return;
145+
}
146+
147+
int headersSize =
148+
(headersCacheSize != 0) ? Math.max(headersCacheSize, blocksCacheSize) : blocksCacheSize;
149+
blockHeadersCache =
150+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(headersSize).build());
151+
;
152+
153+
if (blocksCacheSize != 0) {
136154
blockBodiesCache =
137-
Optional.of(
138-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
155+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(blocksCacheSize).build());
156+
;
139157
transactionReceiptsCache =
140-
Optional.of(
141-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
158+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(blocksCacheSize).build());
159+
;
142160
totalDifficultyCache =
143-
Optional.of(
144-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
161+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(blocksCacheSize).build());
162+
;
145163
blockAccessListCache =
146-
Optional.of(
147-
CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build());
148-
metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockHeaders", blockHeadersCache.get());
149-
metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockBodies", blockBodiesCache.get());
150-
metricsSystem.createGuavaCacheCollector(
151-
BLOCKCHAIN, "transactionReceipts", transactionReceiptsCache.get());
152-
metricsSystem.createGuavaCacheCollector(
153-
BLOCKCHAIN, "totalDifficulty", totalDifficultyCache.get());
164+
Optional.of(CacheBuilder.newBuilder().recordStats().maximumSize(blocksCacheSize).build());
165+
;
166+
registerCacheMetrics(metricsSystem);
154167
} else {
155-
blockHeadersCache = Optional.empty();
156-
blockBodiesCache = Optional.empty();
157-
transactionReceiptsCache = Optional.empty();
158-
totalDifficultyCache = Optional.empty();
159-
blockAccessListCache = Optional.empty();
168+
// Only headers cache is created, rest are empty
169+
setBlockCachesEmpty();
170+
registerHeadersCacheMetrics(metricsSystem);
160171
}
172+
}
161173

162-
createCounters(metricsSystem);
163-
createGauges(metricsSystem);
174+
private void setAllCachesEmpty() {
175+
blockHeadersCache = Optional.empty();
176+
setBlockCachesEmpty();
177+
}
178+
179+
private void setBlockCachesEmpty() {
180+
blockBodiesCache = Optional.empty();
181+
transactionReceiptsCache = Optional.empty();
182+
totalDifficultyCache = Optional.empty();
183+
blockAccessListCache = Optional.empty();
184+
}
185+
186+
private void registerCacheMetrics(final MetricsSystem metricsSystem) {
187+
metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockHeaders", blockHeadersCache.get());
188+
metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockBodies", blockBodiesCache.get());
189+
metricsSystem.createGuavaCacheCollector(
190+
BLOCKCHAIN, "transactionReceipts", transactionReceiptsCache.get());
191+
metricsSystem.createGuavaCacheCollector(
192+
BLOCKCHAIN, "totalDifficulty", totalDifficultyCache.get());
193+
}
194+
195+
private void registerHeadersCacheMetrics(final MetricsSystem metricsSystem) {
196+
metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockHeaders", blockHeadersCache.get());
164197
}
165198

166199
private void createCounters(final MetricsSystem metricsSystem) {
@@ -237,6 +270,7 @@ public static MutableBlockchain createMutable(
237270
metricsSystem,
238271
reorgLoggingThreshold,
239272
null,
273+
0,
240274
0);
241275
}
242276

@@ -253,6 +287,7 @@ public static MutableBlockchain createMutable(
253287
metricsSystem,
254288
reorgLoggingThreshold,
255289
dataDirectory,
290+
0,
256291
0);
257292
}
258293

@@ -262,15 +297,17 @@ public static MutableBlockchain createMutable(
262297
final MetricsSystem metricsSystem,
263298
final long reorgLoggingThreshold,
264299
final String dataDirectory,
265-
final int numberOfBlocksToCache) {
300+
final int numberOfBlocksToCache,
301+
final int numberOgBlockHeadersToCache) {
266302
checkNotNull(genesisBlock);
267303
return new DefaultBlockchain(
268304
Optional.of(genesisBlock),
269305
blockchainStorage,
270306
metricsSystem,
271307
reorgLoggingThreshold,
272308
dataDirectory,
273-
numberOfBlocksToCache);
309+
numberOfBlocksToCache,
310+
numberOgBlockHeadersToCache);
274311
}
275312

276313
public static Blockchain create(
@@ -451,7 +488,7 @@ public synchronized void appendBlock(
451488
final Block block,
452489
final List<TransactionReceipt> receipts,
453490
final Optional<BlockAccessList> blockAccessList) {
454-
if (numberOfBlocksToCache != 0) cacheBlockData(block, receipts, blockAccessList);
491+
cacheBlockData(block, receipts, blockAccessList);
455492
appendBlockHelper(new BlockWithReceipts(block, receipts), false, true);
456493
}
457494

@@ -460,7 +497,7 @@ public synchronized void appendBlockWithoutIndexingTransactions(
460497
final Block block,
461498
final List<TransactionReceipt> receipts,
462499
final Optional<BlockAccessList> blockAccessList) {
463-
if (numberOfBlocksToCache != 0) cacheBlockData(block, receipts, blockAccessList);
500+
cacheBlockData(block, receipts, blockAccessList);
464501
appendBlockHelper(new BlockWithReceipts(block, receipts), false, false);
465502
}
466503

@@ -481,7 +518,7 @@ public synchronized void storeBlock(
481518
final Block block,
482519
final List<TransactionReceipt> receipts,
483520
final Optional<BlockAccessList> blockAccessList) {
484-
if (numberOfBlocksToCache != 0) cacheBlockData(block, receipts, blockAccessList);
521+
cacheBlockData(block, receipts, blockAccessList);
485522
appendBlockHelper(new BlockWithReceipts(block, receipts), true, true);
486523
}
487524

@@ -510,6 +547,10 @@ private void cacheBlockData(
510547
cache -> blockAccessList.ifPresent(t -> cache.put(block.getHash(), t)));
511548
}
512549

550+
private void cacheBlockHeader(final BlockHeader blockHeader) {
551+
blockHeadersCache.ifPresent(cache -> cache.put(blockHeader.getHash(), blockHeader));
552+
}
553+
513554
private boolean blockShouldBeProcessed(
514555
final Block block, final List<TransactionReceipt> receipts) {
515556
checkArgument(

‎ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -977,7 +977,7 @@ public void testCacheUsedWhenNumberOfBlocksToCacheNotZero() {
977977
final KeyValueStorage kvStoreVariables = new InMemoryKeyValueStorage();
978978
final Block genesisBlock = gen.genesisBlock();
979979
final DefaultBlockchain blockchain =
980-
createMutableBlockchain(kvStore, kvStoreVariables, genesisBlock, "/data/test", 512);
980+
createMutableBlockchain(kvStore, kvStoreVariables, genesisBlock, "/data/test", 512, 0);
981981

982982
final BlockDataGenerator.BlockOptions options =
983983
new BlockDataGenerator.BlockOptions()
@@ -1015,6 +1015,36 @@ public void testCacheUsedWhenNumberOfBlocksToCacheNotZero() {
10151015
.isEqualTo(newBlock.getHeader().getDifficulty());
10161016
}
10171017

1018+
@Test
1019+
public void testCacheUsedWhenNumberOfBlockHHeadersToCacheNotZero() {
1020+
final BlockDataGenerator gen = new BlockDataGenerator();
1021+
final KeyValueStorage kvStore = new InMemoryKeyValueStorage();
1022+
final KeyValueStorage kvStoreVariables = new InMemoryKeyValueStorage();
1023+
final Block genesisBlock = gen.genesisBlock();
1024+
final DefaultBlockchain blockchain =
1025+
createMutableBlockchain(kvStore, kvStoreVariables, genesisBlock, "/data/test", 0, 10000);
1026+
1027+
final BlockDataGenerator.BlockOptions options =
1028+
new BlockDataGenerator.BlockOptions()
1029+
.setBlockNumber(1L)
1030+
.setParentHash(genesisBlock.getHash());
1031+
final Block newBlock = gen.block(options);
1032+
final List<TransactionReceipt> receipts = gen.receipts(newBlock);
1033+
1034+
assertThat(blockchain.getBlockHeadersCache()).isNotEmpty();
1035+
assertThat(blockchain.getBlockBodiesCache()).isEmpty();
1036+
assertThat(blockchain.getTransactionReceiptsCache()).isEmpty();
1037+
assertThat(blockchain.getTotalDifficultyCache()).isEmpty();
1038+
1039+
assertThat(blockchain.getBlockHeadersCache().get().size()).isEqualTo(0);
1040+
1041+
blockchain.appendBlock(newBlock, receipts);
1042+
1043+
assertThat(blockchain.getBlockHeadersCache().get().size()).isEqualTo(1);
1044+
assertThat(blockchain.getBlockHeadersCache().get().getIfPresent(newBlock.getHash()))
1045+
.isEqualTo(newBlock.getHeader());
1046+
}
1047+
10181048
/*
10191049
* Check that block header, block body, block number, transaction locations, and receipts for this
10201050
* block are all stored.
@@ -1106,15 +1136,17 @@ private DefaultBlockchain createMutableBlockchain(
11061136
final KeyValueStorage kvStorageVariables,
11071137
final Block genesisBlock,
11081138
final String dataDirectory,
1109-
final int numberOfBlocksToCache) {
1139+
final int numberOfBlocksToCache,
1140+
final int numberOfBlockHeadersToCache) {
11101141
return (DefaultBlockchain)
11111142
DefaultBlockchain.createMutable(
11121143
genesisBlock,
11131144
createStorage(kvStore, kvStorageVariables),
11141145
new NoOpMetricsSystem(),
11151146
0,
11161147
dataDirectory,
1117-
numberOfBlocksToCache);
1148+
numberOfBlocksToCache,
1149+
numberOfBlockHeadersToCache);
11181150
}
11191151

11201152
@Test

0 commit comments

Comments
 (0)