Skip to content

Commit 0a24acc

Browse files
garyschultegaryschulte
authored
Default bonsai to fully flat db and code storage by codehash (#6894)
* change to full flat db and code stored by code hash by default * deprecate --Xsnapsync-synchronizer-flat-db-healing-enabled, add DataStorageOption for --Xbonsai-full-flat-db-enabled Signed-off-by: garyschulte <garyschulte@gmail.com>
1 parent d0a32bc commit 0a24acc

File tree

15 files changed

+135
-99
lines changed

15 files changed

+135
-99
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
- Allow users to specify which plugins are registered [#6700](https://github.com/hyperledger/besu/pull/6700)
4646
- Layered txpool tuning for blob transactions [#6940](https://github.com/hyperledger/besu/pull/6940)
4747
- Update Gradle to 7.6.4 [#7030](https://github.com/hyperledger/besu/pull/7030)
48+
- Default bonsai to use full-flat db and code-storage-by-code-hash [#6984](https://github.com/hyperledger/besu/pull/6894)
4849

4950
### Bug fixes
5051
- Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665)

‎besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1706,8 +1706,8 @@ && isOptionSet(commandLine, "--sync-min-peers")) {
17061706

17071707
CommandLineUtils.failIfOptionDoesntMeetRequirement(
17081708
commandLine,
1709-
"--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true",
1710-
unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled(),
1709+
"--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true",
1710+
dataStorageOptions.toDomainObject().getUnstable().getBonsaiFullFlatDbEnabled(),
17111711
asList(
17121712
"--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
17131713
"--Xsnapsync-synchronizer-flat-slot-healed-count-per-request"));

‎besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
1818
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED;
1919
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
20+
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
2021
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
2122
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
2223
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT;
@@ -93,6 +94,18 @@ public static class Unstable {
9394
"The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
9495
private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
9596

97+
// TODO: --Xsnapsync-synchronizer-flat-db-healing-enabled is deprecated, remove it in a future
98+
// release
99+
@CommandLine.Option(
100+
hidden = true,
101+
names = {
102+
"--Xbonsai-full-flat-db-enabled",
103+
"--Xsnapsync-synchronizer-flat-db-healing-enabled"
104+
},
105+
arity = "1",
106+
description = "Enables bonsai full flat database strategy. (default: ${DEFAULT-VALUE})")
107+
private Boolean bonsaiFullFlatDbEnabled = DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
108+
96109
@CommandLine.Option(
97110
hidden = true,
98111
names = {"--Xbonsai-code-using-code-hash-enabled"},
@@ -161,6 +174,8 @@ public static DataStorageOptions fromConfig(final DataStorageConfiguration domai
161174
domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled();
162175
dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize =
163176
domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize();
177+
dataStorageOptions.unstableOptions.bonsaiFullFlatDbEnabled =
178+
domainObject.getUnstable().getBonsaiFullFlatDbEnabled();
164179
dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled =
165180
domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled();
166181

@@ -177,6 +192,7 @@ public DataStorageConfiguration toDomainObject() {
177192
ImmutableDataStorageConfiguration.Unstable.builder()
178193
.bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled)
179194
.bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize)
195+
.bonsaiFullFlatDbEnabled(unstableOptions.bonsaiFullFlatDbEnabled)
180196
.bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled)
181197
.build())
182198
.build();

‎besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java‎

Lines changed: 9 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,6 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
8181
private static final String SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG =
8282
"--Xsnapsync-synchronizer-flat-slot-healed-count-per-request";
8383

84-
private static final String SNAP_FLAT_DB_HEALING_ENABLED_FLAG =
85-
"--Xsnapsync-synchronizer-flat-db-healing-enabled";
86-
8784
private static final String SNAP_SERVER_ENABLED_FLAG = "--Xsnapsync-server-enabled";
8885

8986
private static final String CHECKPOINT_POST_MERGE_FLAG = "--Xcheckpoint-post-merge-enabled";
@@ -292,18 +289,11 @@ public void parseBlockPropagationRange(final String arg) {
292289
private int snapsyncFlatStorageHealedCountPerRequest =
293290
SnapSyncConfiguration.DEFAULT_LOCAL_FLAT_STORAGE_COUNT_TO_HEAL_PER_REQUEST;
294291

295-
@CommandLine.Option(
296-
names = SNAP_FLAT_DB_HEALING_ENABLED_FLAG,
297-
hidden = true,
298-
paramLabel = "<Boolean>",
299-
description = "Snap sync flat db healing enabled (default: ${DEFAULT-VALUE})")
300-
private Boolean snapsyncFlatDbHealingEnabled =
301-
SnapSyncConfiguration.DEFAULT_IS_FLAT_DB_HEALING_ENABLED;
302-
303292
@CommandLine.Option(
304293
names = SNAP_SERVER_ENABLED_FLAG,
305294
hidden = true,
306295
paramLabel = "<Boolean>",
296+
arity = "0..1",
307297
description = "Snap sync server enabled (default: ${DEFAULT-VALUE})")
308298
private Boolean snapsyncServerEnabled = SnapSyncConfiguration.DEFAULT_SNAP_SERVER_ENABLED;
309299

@@ -316,15 +306,6 @@ public void parseBlockPropagationRange(final String arg) {
316306

317307
private SynchronizerOptions() {}
318308

319-
/**
320-
* Flag to know whether the flat db healing feature is enabled or disabled.
321-
*
322-
* @return true is the flat db healing is enabled
323-
*/
324-
public boolean isSnapsyncFlatDbHealingEnabled() {
325-
return snapsyncFlatDbHealingEnabled;
326-
}
327-
328309
/**
329310
* Flag to know whether the Snap sync server feature is enabled or disabled.
330311
*
@@ -382,9 +363,8 @@ public static SynchronizerOptions fromConfig(final SynchronizerConfiguration con
382363
config.getSnapSyncConfiguration().getLocalFlatAccountCountToHealPerRequest();
383364
options.snapsyncFlatStorageHealedCountPerRequest =
384365
config.getSnapSyncConfiguration().getLocalFlatStorageCountToHealPerRequest();
385-
options.snapsyncFlatDbHealingEnabled =
386-
config.getSnapSyncConfiguration().isFlatDbHealingEnabled();
387366
options.checkpointPostMergeSyncEnabled = config.isCheckpointPostMergeEnabled();
367+
options.snapsyncServerEnabled = config.getSnapSyncConfiguration().isSnapServerEnabled();
388368
return options;
389369
}
390370

@@ -416,7 +396,6 @@ public SynchronizerConfiguration.Builder toDomainObject() {
416396
.trienodeCountPerRequest(snapsyncTrieNodeCountPerRequest)
417397
.localFlatAccountCountToHealPerRequest(snapsyncFlatAccountHealedCountPerRequest)
418398
.localFlatStorageCountToHealPerRequest(snapsyncFlatStorageHealedCountPerRequest)
419-
.isFlatDbHealingEnabled(snapsyncFlatDbHealingEnabled)
420399
.isSnapServerEnabled(snapsyncServerEnabled)
421400
.build());
422401
builder.checkpointPostMergeEnabled(checkpointPostMergeSyncEnabled);
@@ -469,17 +448,13 @@ public List<String> getCLIOptions() {
469448
SNAP_BYTECODE_COUNT_PER_REQUEST_FLAG,
470449
OptionParser.format(snapsyncBytecodeCountPerRequest),
471450
SNAP_TRIENODE_COUNT_PER_REQUEST_FLAG,
472-
OptionParser.format(snapsyncTrieNodeCountPerRequest));
473-
if (isSnapsyncFlatDbHealingEnabled()) {
474-
value.addAll(
475-
Arrays.asList(
476-
SNAP_FLAT_ACCOUNT_HEALED_COUNT_PER_REQUEST_FLAG,
477-
OptionParser.format(snapsyncFlatAccountHealedCountPerRequest),
478-
SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG,
479-
OptionParser.format(snapsyncFlatStorageHealedCountPerRequest),
480-
SNAP_SERVER_ENABLED_FLAG,
481-
OptionParser.format(snapsyncServerEnabled)));
482-
}
451+
OptionParser.format(snapsyncTrieNodeCountPerRequest),
452+
SNAP_FLAT_ACCOUNT_HEALED_COUNT_PER_REQUEST_FLAG,
453+
OptionParser.format(snapsyncFlatAccountHealedCountPerRequest),
454+
SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG,
455+
OptionParser.format(snapsyncFlatStorageHealedCountPerRequest),
456+
SNAP_SERVER_ENABLED_FLAG,
457+
OptionParser.format(snapsyncServerEnabled));
483458
return value;
484459
}
485460
}

‎besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2338,29 +2338,48 @@ public void kzgTrustedSetupFileLoadedWithCustomGenesisFile()
23382338
}
23392339

23402340
@Test
2341-
public void snapsyncHealingOptionShouldBeDisabledByDefault() {
2341+
public void bonsaiFlatDbShouldBeEnabledByDefault() {
23422342
final TestBesuCommand besuCommand = parseCommand();
2343-
assertThat(besuCommand.unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled()).isFalse();
2343+
assertThat(
2344+
besuCommand
2345+
.getDataStorageOptions()
2346+
.toDomainObject()
2347+
.getUnstable()
2348+
.getBonsaiFullFlatDbEnabled())
2349+
.isTrue();
23442350
}
23452351

23462352
@Test
23472353
public void snapsyncHealingOptionShouldWork() {
2348-
final TestBesuCommand besuCommand =
2349-
parseCommand("--Xsnapsync-synchronizer-flat-db-healing-enabled", "true");
2350-
assertThat(besuCommand.unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled()).isTrue();
2354+
final TestBesuCommand besuCommand = parseCommand("--Xbonsai-full-flat-db-enabled", "false");
2355+
assertThat(
2356+
besuCommand
2357+
.dataStorageOptions
2358+
.toDomainObject()
2359+
.getUnstable()
2360+
.getBonsaiFullFlatDbEnabled())
2361+
.isFalse();
23512362
}
23522363

23532364
@Test
23542365
public void snapsyncForHealingFeaturesShouldFailWhenHealingIsNotEnabled() {
2355-
parseCommand("--Xsnapsync-synchronizer-flat-account-healed-count-per-request", "100");
2366+
parseCommand(
2367+
"--Xbonsai-full-flat-db-enabled",
2368+
"false",
2369+
"--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
2370+
"100");
23562371
assertThat(commandErrorOutput.toString(UTF_8))
23572372
.contains(
2358-
"--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
2373+
"--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true");
23592374

2360-
parseCommand("--Xsnapsync-synchronizer-flat-slot-healed-count-per-request", "100");
2375+
parseCommand(
2376+
"--Xbonsai-full-flat-db-enabled",
2377+
"false",
2378+
"--Xsnapsync-synchronizer-flat-slot-healed-count-per-request",
2379+
"100");
23612380
assertThat(commandErrorOutput.toString(UTF_8))
23622381
.contains(
2363-
"--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
2382+
"--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true");
23642383
}
23652384

23662385
@Test

‎besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ protected SynchronizerConfiguration.Builder createCustomizedDomainObject() {
7878
.storageCountPerRequest(SnapSyncConfiguration.DEFAULT_STORAGE_COUNT_PER_REQUEST + 2)
7979
.bytecodeCountPerRequest(
8080
SnapSyncConfiguration.DEFAULT_BYTECODE_COUNT_PER_REQUEST + 2)
81+
.isSnapServerEnabled(Boolean.TRUE)
8182
.build());
8283
}
8384

‎besu/src/test/resources/everything_config.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,8 @@ receipt-compaction-enabled=true
218218
# feature flags
219219
Xsecp256k1-native-enabled=false
220220
Xaltbn128-native-enabled=false
221+
Xsnapsync-server-enabled=true
222+
Xbonsai-full-flat-db-enabled=true
221223

222224
# compatibility flags
223225
compatibility-eth64-forkid-enabled=false

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;
1818
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
19+
import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
1920

2021
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy;
2122
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy;
@@ -70,12 +71,36 @@ public void loadFlatDbStrategy(final SegmentedKeyValueStorage composedWorldState
7071

7172
@VisibleForTesting
7273
FlatDbMode deriveFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) {
74+
final FlatDbMode requestedFlatDbMode =
75+
dataStorageConfiguration.getUnstable().getBonsaiFullFlatDbEnabled()
76+
? FlatDbMode.FULL
77+
: FlatDbMode.PARTIAL;
78+
79+
final var existingTrieData =
80+
composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).isPresent();
81+
7382
var flatDbMode =
7483
FlatDbMode.fromVersion(
7584
composedWorldStateStorage
7685
.get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE)
7786
.map(Bytes::wrap)
78-
.orElse(FlatDbMode.PARTIAL.getVersion()));
87+
.orElseGet(
88+
() -> {
89+
// if we do not have a db-supplied config for flatdb, derive it:
90+
// default to partial if trie data exists, but the flat config does not,
91+
// and default to the storage config otherwise
92+
var flatDbModeVal =
93+
existingTrieData
94+
? FlatDbMode.PARTIAL.getVersion()
95+
: requestedFlatDbMode.getVersion();
96+
// persist this config in the db
97+
var setDbModeTx = composedWorldStateStorage.startTransaction();
98+
setDbModeTx.put(
99+
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, flatDbModeVal.toArrayUnsafe());
100+
setDbModeTx.commit();
101+
102+
return flatDbModeVal;
103+
}));
79104
LOG.info("Bonsai flat db mode found {}", flatDbMode);
80105

81106
return flatDbMode;
@@ -123,7 +148,7 @@ public FlatDbStrategy getFlatDbStrategy(
123148
public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) {
124149
final SegmentedKeyValueStorageTransaction transaction =
125150
composedWorldStateStorage.startTransaction();
126-
// TODO: consider ARCHIVE mode
151+
LOG.info("setting FlatDbStrategy to FULL");
127152
transaction.put(
128153
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe());
129154
transaction.commit();
@@ -134,6 +159,7 @@ public void downgradeToPartialFlatDbMode(
134159
final SegmentedKeyValueStorage composedWorldStateStorage) {
135160
final SegmentedKeyValueStorageTransaction transaction =
136161
composedWorldStateStorage.startTransaction();
162+
LOG.info("setting FlatDbStrategy to PARTIAL");
137163
transaction.put(
138164
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe());
139165
transaction.commit();

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ public interface DataStorageConfiguration {
3838
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
3939
.build();
4040

41+
DataStorageConfiguration DEFAULT_BONSAI_PARTIAL_DB_CONFIG =
42+
ImmutableDataStorageConfiguration.builder()
43+
.dataStorageFormat(DataStorageFormat.BONSAI)
44+
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
45+
.unstable(Unstable.DEFAULT_PARTIAL)
46+
.build();
47+
4148
DataStorageConfiguration DEFAULT_FOREST_CONFIG =
4249
ImmutableDataStorageConfiguration.builder()
4350
.dataStorageFormat(DataStorageFormat.FOREST)
@@ -65,11 +72,15 @@ interface Unstable {
6572
boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = false;
6673
long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
6774
int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 30_000;
68-
boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = false;
75+
boolean DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED = true;
76+
boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = true;
6977

7078
DataStorageConfiguration.Unstable DEFAULT =
7179
ImmutableDataStorageConfiguration.Unstable.builder().build();
7280

81+
DataStorageConfiguration.Unstable DEFAULT_PARTIAL =
82+
ImmutableDataStorageConfiguration.Unstable.builder().bonsaiFullFlatDbEnabled(false).build();
83+
7384
@Value.Default
7485
default boolean getBonsaiLimitTrieLogsEnabled() {
7586
return DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
@@ -80,6 +91,11 @@ default int getBonsaiTrieLogPruningWindowSize() {
8091
return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
8192
}
8293

94+
@Value.Default
95+
default boolean getBonsaiFullFlatDbEnabled() {
96+
return DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
97+
}
98+
8399
@Value.Default
84100
default boolean getBonsaiCodeStoredByCodeHashEnabled() {
85101
return DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;

‎ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,11 @@ public abstract class AbstractIsolationTests {
152152
@BeforeEach
153153
public void createStorage() {
154154
worldStateKeyValueStorage =
155+
// FYI: BonsaiSnapshoIsolationTests work with frozen/cached worldstates, using PARTIAL
156+
// flat db strategy allows the tests to make account assertions based on trie
157+
// (whereas a full db strategy will not, since the worldstates are frozen/cached)
155158
createKeyValueStorageProvider()
156-
.createWorldStateStorage(DataStorageConfiguration.DEFAULT_BONSAI_CONFIG);
159+
.createWorldStateStorage(DataStorageConfiguration.DEFAULT_BONSAI_PARTIAL_DB_CONFIG);
157160
archive =
158161
new BonsaiWorldStateProvider(
159162
(BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage,

0 commit comments

Comments
 (0)