Skip to content

Commit 5b06b50

Browse files
matthew1001matthew1001
committed
Implement snap-serving from a Bonsai Archive node and add tests
Signed-off-by: Matthew Whitehead <matthew1001@gmail.com>
1 parent 9858ba4 commit 5b06b50

File tree

4 files changed

+421
-7
lines changed

4 files changed

+421
-7
lines changed

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/ArchiveFlatDbStrategy.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,13 @@
3131
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
3232

3333
import java.util.Optional;
34+
import java.util.function.Function;
3435
import java.util.function.Supplier;
36+
import java.util.stream.Stream;
3537

38+
import kotlin.Pair;
3639
import org.apache.tuweni.bytes.Bytes;
40+
import org.apache.tuweni.bytes.Bytes32;
3741
import org.bouncycastle.util.Arrays;
3842
import org.slf4j.Logger;
3943
import org.slf4j.LoggerFactory;
@@ -103,6 +107,104 @@ public Optional<Bytes> getFlatAccount(
103107
return accountFound;
104108
}
105109

110+
@Override
111+
protected Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
112+
final SegmentedKeyValueStorage storage, final Bytes startKeyHash, final Bytes32 endKeyHash) {
113+
final Stream<Pair<Bytes32, Bytes>> stream =
114+
storage
115+
.streamFromKey(
116+
ACCOUNT_INFO_STATE,
117+
calculateArchiveKeyNoContextMinSuffix(startKeyHash.toArrayUnsafe()),
118+
calculateArchiveKeyNoContextMaxSuffix(endKeyHash.toArrayUnsafe()))
119+
.map(e -> Bytes.of(calculateArchiveKeyNoContextMaxSuffix(trimSuffix(e.getKey()))))
120+
.distinct()
121+
.map(
122+
e ->
123+
new Pair<>(
124+
Bytes32.wrap(trimSuffix(e.toArrayUnsafe())),
125+
Bytes.of(
126+
storage.getNearestBefore(ACCOUNT_INFO_STATE, e).get().value().get())));
127+
return stream;
128+
}
129+
130+
@Override
131+
protected Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
132+
final SegmentedKeyValueStorage storage, final Bytes startKeyHash) {
133+
final Stream<Pair<Bytes32, Bytes>> stream =
134+
storage
135+
.streamFromKey(
136+
ACCOUNT_INFO_STATE,
137+
calculateArchiveKeyNoContextMinSuffix(startKeyHash.toArrayUnsafe()))
138+
.map(e -> Bytes.of(calculateArchiveKeyNoContextMaxSuffix(trimSuffix(e.getKey()))))
139+
.distinct()
140+
.map(
141+
e ->
142+
new Pair<Bytes32, Bytes>(
143+
Bytes32.wrap(trimSuffix(e.toArrayUnsafe())),
144+
Bytes.of(
145+
storage.getNearestBefore(ACCOUNT_INFO_STATE, e).get().value().get())));
146+
return stream;
147+
}
148+
149+
@Override
150+
protected Stream<Pair<Bytes32, Bytes>> storageToPairStream(
151+
final SegmentedKeyValueStorage storage,
152+
final Hash accountHash,
153+
final Bytes startKeyHash,
154+
final Function<Bytes, Bytes> valueMapper) {
155+
return storage
156+
.streamFromKey(
157+
ACCOUNT_STORAGE_STORAGE,
158+
calculateArchiveKeyNoContextMinSuffix(
159+
calculateNaturalSlotKey(accountHash, Hash.wrap(Bytes32.wrap(startKeyHash)))))
160+
.map(e -> Bytes.of(calculateArchiveKeyNoContextMaxSuffix(trimSuffix(e.getKey()))))
161+
.distinct()
162+
.map(
163+
key ->
164+
new Pair<>(
165+
Bytes32.wrap(trimSuffix(key.slice(Hash.SIZE).toArrayUnsafe())),
166+
valueMapper
167+
.apply(
168+
Bytes.of(
169+
storage
170+
.getNearestBefore(ACCOUNT_STORAGE_STORAGE, key)
171+
.get()
172+
.value()
173+
.get())
174+
.trimLeadingZeros())));
175+
}
176+
177+
@Override
178+
protected Stream<Pair<Bytes32, Bytes>> storageToPairStream(
179+
final SegmentedKeyValueStorage storage,
180+
final Hash accountHash,
181+
final Bytes startKeyHash,
182+
final Bytes32 endKeyHash,
183+
final Function<Bytes, Bytes> valueMapper) {
184+
return storage
185+
.streamFromKey(
186+
ACCOUNT_STORAGE_STORAGE,
187+
calculateArchiveKeyNoContextMinSuffix(
188+
calculateNaturalSlotKey(accountHash, Hash.wrap(Bytes32.wrap(startKeyHash)))),
189+
calculateArchiveKeyNoContextMaxSuffix(
190+
calculateNaturalSlotKey(accountHash, Hash.wrap(endKeyHash))))
191+
.map(e -> Bytes.of(calculateArchiveKeyNoContextMaxSuffix(trimSuffix(e.getKey()))))
192+
.distinct()
193+
.map(
194+
key ->
195+
new Pair<>(
196+
Bytes32.wrap(trimSuffix(key.slice(Hash.SIZE).toArrayUnsafe())),
197+
valueMapper
198+
.apply(
199+
Bytes.of(
200+
storage
201+
.getNearestBefore(ACCOUNT_STORAGE_STORAGE, key)
202+
.get()
203+
.value()
204+
.get())
205+
.trimLeadingZeros())));
206+
}
207+
106208
/*
107209
* Puts the account data for the given account hash and block context.
108210
*/
@@ -128,6 +230,10 @@ public void removeFlatAccount(
128230
transaction.put(ACCOUNT_INFO_STATE, keySuffixed, DELETED_ACCOUNT_VALUE);
129231
}
130232

233+
private byte[] trimSuffix(final byte[] suffixedAddress) {
234+
return Arrays.copyOfRange(suffixedAddress, 0, suffixedAddress.length - 8);
235+
}
236+
131237
/*
132238
* Retrieves the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
133239
*/
@@ -232,6 +338,14 @@ public static byte[] calculateArchiveKeyWithMinSuffix(
232338
return calculateArchiveKeyWithSuffix(context, naturalKey, MIN_BLOCK_SUFFIX);
233339
}
234340

341+
public static byte[] calculateArchiveKeyNoContextMinSuffix(final byte[] naturalKey) {
342+
return Arrays.concatenate(naturalKey, MIN_BLOCK_SUFFIX);
343+
}
344+
345+
public static byte[] calculateArchiveKeyNoContextMaxSuffix(final byte[] naturalKey) {
346+
return Arrays.concatenate(naturalKey, MAX_BLOCK_SUFFIX);
347+
}
348+
235349
public static Bytes calculateArchiveKeyWithMaxSuffix(
236350
final BonsaiContext context, final byte[] naturalKey) {
237351
return Bytes.of(calculateArchiveKeyWithSuffix(context, naturalKey, MAX_BLOCK_SUFFIX));

‎ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ public NavigableMap<Bytes32, Bytes> streamStorageFlatDatabase(
250250
.takeWhile(takeWhile));
251251
}
252252

253-
private static Stream<Pair<Bytes32, Bytes>> storageToPairStream(
253+
protected Stream<Pair<Bytes32, Bytes>> storageToPairStream(
254254
final SegmentedKeyValueStorage storage,
255255
final Hash accountHash,
256256
final Bytes startKeyHash,
@@ -267,7 +267,7 @@ private static Stream<Pair<Bytes32, Bytes>> storageToPairStream(
267267
valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros())));
268268
}
269269

270-
private static Stream<Pair<Bytes32, Bytes>> storageToPairStream(
270+
protected Stream<Pair<Bytes32, Bytes>> storageToPairStream(
271271
final SegmentedKeyValueStorage storage,
272272
final Hash accountHash,
273273
final Bytes startKeyHash,
@@ -286,14 +286,14 @@ private static Stream<Pair<Bytes32, Bytes>> storageToPairStream(
286286
valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros())));
287287
}
288288

289-
private static Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
289+
protected Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
290290
final SegmentedKeyValueStorage storage, final Bytes startKeyHash, final Bytes32 endKeyHash) {
291291
return storage
292292
.streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe(), endKeyHash.toArrayUnsafe())
293293
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue())));
294294
}
295295

296-
private static Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
296+
protected Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
297297
final SegmentedKeyValueStorage storage, final Bytes startKeyHash) {
298298
return storage
299299
.streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe())

0 commit comments

Comments
 (0)