Skip to content

Commit aa94e14

Browse files
garyschultematkt
authored and
NickSneo
committed
Bonsai based reference test worldstate (besu-eth#5686)
* create a bonsai based reference test worldstate -> getOrCreate in BonsaiWorldStateUpdateAccumulator - do not throw if we discover an empty account in a non-null BonsaiValue<Account> -> add curentStateRoot to t8n -> storageEntriesFrom and streamAccounts implemented in BonsaiWorldStateKeyValueStorage -> add endKey version of streamFromKey * bonsai fix for self-destruct and create2 at the same address and same block Signed-off-by: garyschulte <garyschulte@gmail.com> Signed-off-by: Karim TAAM <karim.t2am@gmail.com> Co-authored-by: Karim TAAM <karim.t2am@gmail.com>
1 parent 7a3e994 commit aa94e14

File tree

53 files changed

+746
-141
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+746
-141
lines changed

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ public UInt256 getOriginalStorageValue(final UInt256 key) {
219219
@Override
220220
public NavigableMap<Bytes32, AccountStorageEntry> storageEntriesFrom(
221221
final Bytes32 startKeyHash, final int limit) {
222-
throw new RuntimeException("Bonsai Tries does not currently support enumerating storage");
222+
return context.getWorldStateStorage().storageEntriesFrom(this.addressHash, startKeyHash, limit);
223223
}
224224

225225
public Bytes serializeAccount() {

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,8 @@ public BonsaiWorldStateProvider(
102102
pluginContext);
103103
this.blockchain = blockchain;
104104
this.worldStateStorage = worldStateStorage;
105-
this.persistedState = new BonsaiWorldState(this, worldStateStorage);
106105
this.cachedMerkleTrieLoader = cachedMerkleTrieLoader;
106+
this.persistedState = new BonsaiWorldState(this, worldStateStorage);
107107
blockchain
108108
.getBlockHeader(persistedState.getWorldStateBlockHash())
109109
.ifPresent(

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright Hyperledger Besu Contributors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
5+
* the License. You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
10+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
* specific language governing permissions and limitations under the License.
12+
*
13+
* SPDX-License-Identifier: Apache-2.0
14+
*
15+
*/
16+
package org.hyperledger.besu.ethereum.bonsai.storage;
17+
18+
import org.hyperledger.besu.datatypes.Address;
19+
import org.hyperledger.besu.datatypes.Hash;
20+
import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
21+
22+
import java.util.Optional;
23+
24+
import com.google.common.collect.BiMap;
25+
import com.google.common.collect.HashBiMap;
26+
import org.apache.tuweni.bytes.Bytes;
27+
import org.apache.tuweni.bytes.Bytes32;
28+
import org.apache.tuweni.units.bigints.UInt256;
29+
30+
/** Acts as both a Hasher and PreImageStorage for Bonsai storage format. */
31+
public interface BonsaiPreImageProxy extends WorldStatePreimageStorage {
32+
/**
33+
* If this value is not already present, save in preImage store and return the hash value.
34+
*
35+
* @param value value to hash
36+
* @return Hash of value
37+
*/
38+
Hash hashAndSavePreImage(Bytes value);
39+
40+
/**
41+
* A caching PreImageProxy suitable for ReferenceTestWorldState which saves hashes in an unbounded
42+
* BiMap.
43+
*/
44+
class BonsaiReferenceTestPreImageProxy implements BonsaiPreImageProxy {
45+
BiMap<Hash, Bytes> preImageCache = HashBiMap.create();
46+
47+
@Override
48+
public synchronized Hash hashAndSavePreImage(final Bytes value) {
49+
return preImageCache.inverse().computeIfAbsent(value, Hash::hash);
50+
}
51+
52+
@Override
53+
public Optional<UInt256> getStorageTrieKeyPreimage(final Bytes32 trieKey) {
54+
return Optional.ofNullable(preImageCache.get(trieKey)).map(UInt256::fromBytes);
55+
}
56+
57+
@Override
58+
public Optional<Address> getAccountTrieKeyPreimage(final Bytes32 trieKey) {
59+
return Optional.ofNullable(preImageCache.get(trieKey)).map(Address::wrap);
60+
}
61+
62+
@Override
63+
public Updater updater() {
64+
throw new UnsupportedOperationException(
65+
"BonsaiReferenceTestPreImageProxy does not implement an updater");
66+
}
67+
}
68+
}

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
3232
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
3333
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
34+
import org.hyperledger.besu.evm.account.AccountStorageEntry;
3435
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
3536
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
3637
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
@@ -41,6 +42,7 @@
4142
import java.nio.charset.StandardCharsets;
4243
import java.util.List;
4344
import java.util.Map;
45+
import java.util.NavigableMap;
4446
import java.util.Optional;
4547
import java.util.concurrent.atomic.AtomicBoolean;
4648
import java.util.function.Predicate;
@@ -53,7 +55,6 @@
5355

5456
@SuppressWarnings("unused")
5557
public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoCloseable {
56-
5758
private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class);
5859

5960
// 0x776f726c64526f6f74
@@ -250,6 +251,11 @@ public Map<Bytes32, Bytes> streamFlatStorages(
250251
composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max);
251252
}
252253

254+
public NavigableMap<Bytes32, AccountStorageEntry> storageEntriesFrom(
255+
final Hash addressHash, final Bytes32 startKeyHash, final int limit) {
256+
throw new RuntimeException("Bonsai Tries does not currently support enumerating storage");
257+
}
258+
253259
@Override
254260
public Optional<Bytes> getNodeData(final Bytes location, final Bytes32 hash) {
255261
return Optional.empty();

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -135,10 +135,10 @@ public Map<Bytes32, Bytes> streamAccountFlatDatabase(
135135
final long max) {
136136
final Stream<Pair<Bytes32, Bytes>> pairStream =
137137
storage
138-
.streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe())
138+
.streamFromKey(
139+
ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe(), endKeyHash.toArrayUnsafe())
139140
.limit(max)
140-
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue())))
141-
.takeWhile(pair -> pair.getFirst().compareTo(endKeyHash) <= 0);
141+
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue())));
142142

143143
final TreeMap<Bytes32, Bytes> collected =
144144
pairStream.collect(
@@ -157,15 +157,14 @@ public Map<Bytes32, Bytes> streamStorageFlatDatabase(
157157
storage
158158
.streamFromKey(
159159
ACCOUNT_STORAGE_STORAGE,
160-
Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe())
161-
.takeWhile(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash))
160+
Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe(),
161+
Bytes.concatenate(accountHash, endKeyHash).toArrayUnsafe())
162162
.limit(max)
163163
.map(
164164
pair ->
165165
new Pair<>(
166166
Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)),
167-
RLP.encodeValue(Bytes.wrap(pair.getValue()).trimLeadingZeros())))
168-
.takeWhile(pair -> pair.getFirst().compareTo(endKeyHash) <= 0);
167+
RLP.encodeValue(Bytes.wrap(pair.getValue()).trimLeadingZeros())));
169168

170169
final TreeMap<Bytes32, Bytes> collected =
171170
pairStream.collect(

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java‎

Lines changed: 39 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount;
2828
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
2929
import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider;
30+
import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
3031
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage;
3132
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
3233
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
@@ -65,51 +66,52 @@ public class BonsaiWorldState
6566

6667
private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldState.class);
6768

68-
private BonsaiWorldStateKeyValueStorage worldStateStorage;
69+
protected BonsaiWorldStateKeyValueStorage worldStateStorage;
6970

70-
private final BonsaiWorldStateProvider archive;
71-
private final BonsaiWorldStateUpdateAccumulator accumulator;
71+
protected final CachedMerkleTrieLoader cachedMerkleTrieLoader;
72+
protected final TrieLogManager trieLogManager;
73+
private BonsaiWorldStateUpdateAccumulator accumulator;
7274

73-
private Hash worldStateRootHash;
75+
protected Hash worldStateRootHash;
7476
Hash worldStateBlockHash;
75-
7677
private boolean isFrozen;
7778

7879
public BonsaiWorldState(
7980
final BonsaiWorldStateProvider archive,
8081
final BonsaiWorldStateKeyValueStorage worldStateStorage) {
81-
this.archive = archive;
82+
this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager());
83+
}
84+
85+
protected BonsaiWorldState(
86+
final BonsaiWorldStateKeyValueStorage worldStateStorage,
87+
final CachedMerkleTrieLoader cachedMerkleTrieLoader,
88+
final TrieLogManager trieLogManager) {
8289
this.worldStateStorage = worldStateStorage;
83-
worldStateRootHash =
90+
this.worldStateRootHash =
8491
Hash.wrap(
8592
Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH)));
86-
worldStateBlockHash =
93+
this.worldStateBlockHash =
8794
Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO)));
88-
accumulator =
95+
this.accumulator =
8996
new BonsaiWorldStateUpdateAccumulator(
9097
this,
9198
(addr, value) ->
92-
archive
93-
.getCachedMerkleTrieLoader()
94-
.preLoadAccount(getWorldStateStorage(), worldStateRootHash, addr),
99+
cachedMerkleTrieLoader.preLoadAccount(
100+
getWorldStateStorage(), worldStateRootHash, addr),
95101
(addr, value) ->
96-
archive
97-
.getCachedMerkleTrieLoader()
98-
.preLoadStorageSlot(getWorldStateStorage(), addr, value));
102+
cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value));
103+
this.cachedMerkleTrieLoader = cachedMerkleTrieLoader;
104+
this.trieLogManager = trieLogManager;
99105
}
100106

101-
public BonsaiWorldState(
102-
final BonsaiWorldStateProvider archive,
103-
final BonsaiWorldStateKeyValueStorage worldStateStorage,
104-
final BonsaiWorldStateUpdateAccumulator updater) {
105-
this.archive = archive;
106-
this.worldStateStorage = worldStateStorage;
107-
this.worldStateRootHash =
108-
Hash.wrap(
109-
Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH)));
110-
this.worldStateBlockHash =
111-
Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO)));
112-
this.accumulator = updater;
107+
/**
108+
* Having a protected method to override the accumulator solves the chicken-egg problem of needing
109+
* a worldstate reference (this) when construction the Accumulator.
110+
*
111+
* @param accumulator accumulator to use.
112+
*/
113+
protected void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) {
114+
this.accumulator = accumulator;
113115
}
114116

115117
/**
@@ -130,10 +132,6 @@ public Hash getWorldStateRootHash() {
130132
return worldStateRootHash;
131133
}
132134

133-
public BonsaiWorldStateProvider getArchive() {
134-
return archive;
135-
}
136-
137135
@Override
138136
public boolean isPersisted() {
139137
return isPersisted(worldStateStorage);
@@ -189,9 +187,7 @@ private Hash calculateRootHash(
189187
final StoredMerklePatriciaTrie<Bytes, Bytes> accountTrie =
190188
createTrie(
191189
(location, hash) ->
192-
archive
193-
.getCachedMerkleTrieLoader()
194-
.getAccountStateTrieNode(worldStateStorage, location, hash),
190+
cachedMerkleTrieLoader.getAccountStateTrieNode(worldStateStorage, location, hash),
195191
worldStateRootHash);
196192

197193
// for manicured tries and composting, collect branches here (not implemented)
@@ -223,7 +219,7 @@ private void updateTheAccounts(
223219
final BonsaiAccount updatedAccount = bonsaiValue.getUpdated();
224220
try {
225221
if (updatedAccount == null) {
226-
final Hash addressHash = Hash.hash(accountKey);
222+
final Hash addressHash = hashAndSavePreImage(accountKey);
227223
accountTrie.remove(addressHash);
228224
maybeStateUpdater.ifPresent(
229225
bonsaiUpdater -> bonsaiUpdater.removeAccountInfoState(addressHash));
@@ -232,7 +228,7 @@ private void updateTheAccounts(
232228
final Bytes accountValue = updatedAccount.serializeAccount();
233229
maybeStateUpdater.ifPresent(
234230
bonsaiUpdater ->
235-
bonsaiUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue));
231+
bonsaiUpdater.putAccountInfoState(hashAndSavePreImage(accountKey), accountValue));
236232
accountTrie.put(addressHash, accountValue);
237233
}
238234
} catch (MerkleTrieException e) {
@@ -277,10 +273,8 @@ private void updateAccountStorageState(
277273
final StoredMerklePatriciaTrie<Bytes, Bytes> storageTrie =
278274
createTrie(
279275
(location, key) ->
280-
archive
281-
.getCachedMerkleTrieLoader()
282-
.getAccountStorageTrieNode(
283-
worldStateStorage, updatedAddressHash, location, key),
276+
cachedMerkleTrieLoader.getAccountStorageTrieNode(
277+
worldStateStorage, updatedAddressHash, location, key),
284278
storageRoot);
285279

286280
// for manicured tries and composting, collect branches here (not implemented)
@@ -405,7 +399,6 @@ public void persist(final BlockHeader blockHeader) {
405399
}
406400
saveTrieLog =
407401
() -> {
408-
final TrieLogManager trieLogManager = archive.getTrieLogManager();
409402
trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this);
410403
// not save a frozen state in the cache
411404
if (!isFrozen) {
@@ -626,4 +619,9 @@ private void closeFrozenStorage() {
626619
// no op
627620
}
628621
}
622+
623+
protected Hash hashAndSavePreImage(final Bytes value) {
624+
// by default do not save has preImages
625+
return Hash.hash(value);
626+
}
629627
}

0 commit comments

Comments
 (0)