Skip to content

Commit 5946bfe

Browse files
ttyusupovttyusupov
committed
[yugabyte#27233] docdb: added check_data_blocks command to SST dump
Summary: Added `check_data_blocks` to sst_dump tool. It will check all data blocks consistency using SST file index and per-block checksums. `--skip_uncompress` option specifies that `check_data_blocks` command should skip uncompressing data blocks (false by default). If any corrupt data blocks are found, the tool will generate `dd` commands which could be used to save specific blocks (corrupted block and one block before/after) for analysis. Example usage: ``` ./sst_dump --command=check_data_blocks --file=000046.sst 2>&1 | tee sst_dump.out cat sst_dump.out | grep '^dd ' ``` Jira: DB-16720 Test Plan: Tested manually on artificially corrupt SST file: ``` ~/code/yugabyte5/build/latest/bin/sst_dump --command=check_data_blocks --file=000046.sst 2>&1 | tee sst_dump.out WARNING: Logging before InitGoogleLogging() is written to STDERR I0517 07:21:12.046017 3481 sst_dump_tool.cc:433] Checking data block #0 handle: BlockHandle { offset: 0 size: 3927 } W0517 07:21:12.051733 3481 file_reader_writer.cc:101] Read attempt #1 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 996452 size: 4024 }, expected checksum: 1118200703, actual checksum: 1023486239. W0517 07:21:12.051852 3481 file_reader_writer.cc:101] Read attempt yugabyte#2 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 996452 size: 4024 }, expected checksum: 1118200703, actual checksum: 1023486239. E0517 07:21:12.051862 3481 format.cc:466] ReadBlockContents: Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 996452 size: 4024 }, expected checksum: 1118200703, actual checksum: 1023486239. @ 0x7fd68e2eafde rocksdb::SstFileReader::CheckDataBlocks() @ 0x7fd68e2ec8ca rocksdb::SSTDumpTool::Run() @ 0x555855a53f78 main @ 0x7fd688e45ca3 __libc_start_main @ 0x555855a53eae _start W0517 07:21:12.056023 3481 sst_dump_tool.cc:441] Failed to read block with handle: BlockHandle { offset: 996452 size: 4024 }. Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 996452 size: 4024 }, expected checksum: 1118200703, actual checksum: 1023486239. from [] to [] Process 000046.sst Sst file format: block-based dd if="000046.sst.sblock.0" bs=1 skip=992433 count=4014 of="000046.sst.sblock.0.offset_992433.size_4014.part" dd if="000046.sst.sblock.0" bs=1 skip=996452 count=4024 of="000046.sst.sblock.0.offset_996452.size_4024.part" dd if="000046.sst.sblock.0" bs=1 skip=1000481 count=4019 of="000046.sst.sblock.0.offset_1000481.size_4019.part" W0517 07:21:12.634256 3481 file_reader_writer.cc:101] Read attempt #1 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119754675 size: 11766 }, expected checksum: 84894407, actual checksum: 2295535311. W0517 07:21:12.634332 3481 file_reader_writer.cc:101] Read attempt yugabyte#2 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119754675 size: 11766 }, expected checksum: 84894407, actual checksum: 2295535311. E0517 07:21:12.634339 3481 format.cc:466] ReadBlockContents: Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119754675 size: 11766 }, expected checksum: 84894407, actual checksum: 2295535311. @ 0x7fd68e2eafde rocksdb::SstFileReader::CheckDataBlocks() @ 0x7fd68e2ec8ca rocksdb::SSTDumpTool::Run() @ 0x555855a53f78 main @ 0x7fd688e45ca3 __libc_start_main @ 0x555855a53eae _start W0517 07:21:12.638135 3481 sst_dump_tool.cc:441] Failed to read block with handle: BlockHandle { offset: 119754675 size: 11766 }. Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119754675 size: 11766 }, expected checksum: 84894407, actual checksum: 2295535311. dd if="000046.sst.sblock.0" bs=1 skip=119742901 count=11769 of="000046.sst.sblock.0.offset_119742901.size_11769.part" dd if="000046.sst.sblock.0" bs=1 skip=119754675 count=11766 of="000046.sst.sblock.0.offset_119754675.size_11766.part" dd if="000046.sst.sblock.0" bs=1 skip=119766446 count=1384 of="000046.sst.sblock.0.offset_119766446.size_1384.part" ``` ``` cat sst_dump.out | grep '^dd ' dd if="000046.sst.sblock.0" bs=1 skip=992433 count=4014 of="000046.sst.sblock.0.offset_992433.size_4014.part" dd if="000046.sst.sblock.0" bs=1 skip=996452 count=4024 of="000046.sst.sblock.0.offset_996452.size_4024.part" dd if="000046.sst.sblock.0" bs=1 skip=1000481 count=4019 of="000046.sst.sblock.0.offset_1000481.size_4019.part" dd if="000046.sst.sblock.0" bs=1 skip=119742901 count=11769 of="000046.sst.sblock.0.offset_119742901.size_11769.part" dd if="000046.sst.sblock.0" bs=1 skip=119754675 count=11766 of="000046.sst.sblock.0.offset_119754675.size_11766.part" dd if="000046.sst.sblock.0" bs=1 skip=119766446 count=1384 of="000046.sst.sblock.0.offset_119766446.size_1384.part" ``` Reviewers: hsunder, yyan Reviewed By: yyan Subscribers: ybase Tags: #jenkins-ready Differential Revision: https://phorge.dev.yugabyte.com/D44035
1 parent 6c31ff6 commit 5946bfe

File tree

2 files changed

+101
-20
lines changed

2 files changed

+101
-20
lines changed

‎src/yb/rocksdb/tools/sst_dump_tool.cc

Lines changed: 93 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,14 @@
5353

5454
#include "yb/docdb/docdb_debug.h"
5555

56+
#include "yb/util/format.h"
5657
#include "yb/util/status_log.h"
5758

5859
using yb::docdb::StorageDbType;
5960

6061
namespace rocksdb {
6162

6263
using std::dynamic_pointer_cast;
63-
using std::unique_ptr;
64-
using std::shared_ptr;
6564

6665
std::string DocDBKVFormatter::Format(
6766
const yb::Slice&, const yb::Slice&, yb::docdb::StorageDbType) const {
@@ -97,9 +96,7 @@ Status SstFileReader::GetTableReader(const std::string& file_path) {
9796
uint64_t magic_number;
9897

9998
// read table magic number
100-
Footer footer;
101-
102-
unique_ptr<RandomAccessFile> file;
99+
std::unique_ptr<RandomAccessFile> file;
103100
uint64_t file_size;
104101
Status s = options_.env->NewRandomAccessFile(file_path, &file, soptions_);
105102
if (s.ok()) {
@@ -109,10 +106,10 @@ Status SstFileReader::GetTableReader(const std::string& file_path) {
109106
file_.reset(new RandomAccessFileReader(std::move(file)));
110107

111108
if (s.ok()) {
112-
s = ReadFooterFromFile(file_.get(), file_size, &footer);
109+
s = ReadFooterFromFile(file_.get(), file_size, &footer_);
113110
}
114111
if (s.ok()) {
115-
magic_number = footer.table_magic_number();
112+
magic_number = footer_.table_magic_number();
116113
}
117114

118115
if (s.ok()) {
@@ -135,10 +132,10 @@ Status SstFileReader::GetTableReader(const std::string& file_path) {
135132
s = NewTableReader(ioptions_, soptions_, *internal_comparator_, file_size,
136133
&table_reader_);
137134
if (s.ok() && table_reader_->IsSplitSst()) {
138-
unique_ptr<RandomAccessFile> data_file;
135+
std::unique_ptr<RandomAccessFile> data_file;
139136
RETURN_NOT_OK(options_.env->NewRandomAccessFile(
140137
TableBaseToDataFileName(file_path), &data_file, soptions_));
141-
unique_ptr<RandomAccessFileReader> data_file_reader(
138+
std::unique_ptr<RandomAccessFileReader> data_file_reader(
142139
new RandomAccessFileReader(std::move(data_file)));
143140
table_reader_->SetDataFileReader(std::move(data_file_reader));
144141
}
@@ -149,10 +146,10 @@ Status SstFileReader::GetTableReader(const std::string& file_path) {
149146
Status SstFileReader::NewTableReader(
150147
const ImmutableCFOptions& ioptions, const EnvOptions& soptions,
151148
const InternalKeyComparator& internal_comparator, uint64_t file_size,
152-
unique_ptr<TableReader>* table_reader) {
149+
std::unique_ptr<TableReader>* table_reader) {
153150
// We need to turn off pre-fetching of index and filter nodes for
154151
// BlockBasedTable
155-
shared_ptr<BlockBasedTableFactory> block_table_factory =
152+
std::shared_ptr<BlockBasedTableFactory> block_table_factory =
156153
dynamic_pointer_cast<BlockBasedTableFactory>(options_.table_factory);
157154

158155
if (block_table_factory) {
@@ -172,7 +169,7 @@ Status SstFileReader::NewTableReader(
172169
}
173170

174171
Status SstFileReader::DumpTable(const std::string& out_filename) {
175-
unique_ptr<WritableFile> out_file;
172+
std::unique_ptr<WritableFile> out_file;
176173
Env* env = Env::Default();
177174
RETURN_NOT_OK(env->NewWritableFile(out_filename, &out_file, soptions_));
178175
Status s = table_reader_->DumpTable(out_file.get());
@@ -182,20 +179,20 @@ Status SstFileReader::DumpTable(const std::string& out_filename) {
182179

183180
uint64_t SstFileReader::CalculateCompressedTableSize(
184181
const TableBuilderOptions& tb_options, size_t block_size) {
185-
unique_ptr<WritableFile> out_file;
186-
unique_ptr<Env> env(NewMemEnv(Env::Default()));
182+
std::unique_ptr<WritableFile> out_file;
183+
std::unique_ptr<Env> env(NewMemEnv(Env::Default()));
187184
CHECK_OK(env->NewWritableFile(testFileName, &out_file, soptions_));
188-
unique_ptr<WritableFileWriter> dest_writer;
185+
std::unique_ptr<WritableFileWriter> dest_writer;
189186
dest_writer.reset(new WritableFileWriter(std::move(out_file), soptions_));
190187
BlockBasedTableOptions table_options;
191188
table_options.block_size = block_size;
192189
BlockBasedTableFactory block_based_tf(table_options);
193-
unique_ptr<TableBuilder> table_builder;
190+
std::unique_ptr<TableBuilder> table_builder;
194191
table_builder = block_based_tf.NewTableBuilder(
195192
tb_options,
196193
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
197194
dest_writer.get());
198-
unique_ptr<InternalIterator> iter(table_reader_->NewIterator(ReadOptions()));
195+
std::unique_ptr<InternalIterator> iter(table_reader_->NewIterator(ReadOptions()));
199196
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
200197
if (!iter->status().ok()) {
201198
fputs(iter->status().ToString().c_str(), stderr);
@@ -387,6 +384,77 @@ Status SstFileReader::ReadSequential(bool print_kv,
387384
return ret;
388385
}
389386

387+
namespace {
388+
389+
void PrintSaveBlockCommand(const std::string& data_file_path, const BlockHandle& block_handle) {
390+
std::cout << "dd if=\"" << data_file_path << "\" bs=1 skip=" << block_handle.offset()
391+
<< " count=" << block_handle.size() << " of=\"" << data_file_path << ".offset_"
392+
<< block_handle.offset() << ".size_" << block_handle.size() << ".part\"" << std::endl;
393+
}
394+
395+
} // namespace
396+
397+
Status SstFileReader::CheckDataBlocks(DoUncompress do_uncompress) {
398+
if (!table_reader_) {
399+
return init_result_;
400+
}
401+
ReadOptions read_options;
402+
read_options.verify_checksums = true;
403+
404+
std::unique_ptr<InternalIterator> index_iterator(table_reader_->NewIndexIterator(read_options));
405+
RETURN_NOT_OK(index_iterator->status());
406+
407+
const auto data_file_path =
408+
table_reader_->IsSplitSst() ? TableBaseToDataFileName(file_name_) : file_name_;
409+
std::unique_ptr<RandomAccessFile> data_file;
410+
RETURN_NOT_OK(options_.env->NewRandomAccessFile(data_file_path, &data_file, soptions_));
411+
std::unique_ptr<RandomAccessFileReader> data_file_reader(
412+
new RandomAccessFileReader(std::move(data_file)));
413+
414+
size_t index_entry_pos = 0;
415+
BlockHandle prev_block_handle;
416+
BlockHandle block_handle;
417+
bool save_block = false;
418+
for (index_iterator->SeekToFirst(); index_iterator->Valid();
419+
index_iterator->Next(), ++index_entry_pos) {
420+
prev_block_handle = block_handle;
421+
{
422+
auto index_value_slice = index_iterator->Entry().value;
423+
auto status = block_handle.DecodeFrom(&index_value_slice);
424+
if (!status.ok()) {
425+
LOG(WARNING) << "Failed to decode SST index entry #" << index_entry_pos << ": "
426+
<< index_iterator->Entry().value.ToDebugHexString() << ". " << status;
427+
continue;
428+
}
429+
LOG_IF(WARNING, index_value_slice.size() > 0)
430+
<< "Extra bytes (" << index_value_slice.size()
431+
<< ") in index entry: " << index_iterator->Entry().value.ToDebugHexString();
432+
}
433+
YB_LOG_EVERY_N_SECS(INFO, 30) << "Checking data block #" << index_entry_pos
434+
<< " handle: " << block_handle.ToDebugString();
435+
436+
BlockContents block_contents;
437+
auto status = ReadBlockContents(
438+
data_file_reader.get(), footer_, read_options, block_handle, &block_contents, options_.env,
439+
/* mem_tracker = */ nullptr, do_uncompress);
440+
if (!status.ok()) {
441+
LOG(WARNING) << "Failed to read block with handle: " << block_handle.ToDebugString() << ". "
442+
<< status;
443+
if (prev_block_handle.IsSet()) {
444+
PrintSaveBlockCommand(data_file_path, prev_block_handle);
445+
}
446+
PrintSaveBlockCommand(data_file_path, block_handle);
447+
// Save next block as well.
448+
save_block = true;
449+
} else if (save_block) {
450+
PrintSaveBlockCommand(data_file_path, block_handle);
451+
save_block = false;
452+
}
453+
}
454+
455+
return Status::OK();
456+
}
457+
390458
Status SstFileReader::ReadTableProperties(
391459
std::shared_ptr<const TableProperties>* table_properties) {
392460
if (!table_reader_) {
@@ -401,7 +469,7 @@ namespace {
401469

402470
void print_help() {
403471
fprintf(stderr,
404-
"sst_dump [--command=check|scan|none|raw] [--verify_checksum] "
472+
"sst_dump [--command=check|scan|check_data_blocks|none|raw] [--verify_checksum] "
405473
"--file=data_dir_OR_sst_file"
406474
" [--output_format=raw|hex|decoded_regulardb|decoded_intentsdb]"
407475
" [--formatter_tablet_metadata=<path_to_tablet_metadata>"
@@ -411,7 +479,8 @@ void print_help() {
411479
" [--read_num=NUM]"
412480
" [--show_properties]"
413481
" [--show_compression_sizes]"
414-
" [--show_compression_sizes [--set_block_size=<block_size>]]\n");
482+
" [--show_compression_sizes [--set_block_size=<block_size>]]"
483+
" [--skip_uncompress]\n");
415484
}
416485

417486
} // namespace
@@ -431,6 +500,7 @@ int SSTDumpTool::Run(int argc, char** argv) {
431500
bool show_properties = false;
432501
bool show_compression_sizes = false;
433502
bool set_block_size = false;
503+
DoUncompress do_uncompress = DoUncompress ::kTrue;
434504
std::string from_key;
435505
std::string to_key;
436506
std::string block_size_str;
@@ -470,6 +540,8 @@ int SSTDumpTool::Run(int argc, char** argv) {
470540
show_properties = true;
471541
} else if (strcmp(argv[i], "--show_compression_sizes") == 0) {
472542
show_compression_sizes = true;
543+
} else if (strcmp(argv[i], "--skip_uncompress") == 0) {
544+
do_uncompress = DoUncompress::kFalse;
473545
} else if (strncmp(argv[i], "--set_block_size=", 17) == 0) {
474546
set_block_size = true;
475547
block_size_str = argv[i] + 17;
@@ -575,6 +647,8 @@ int SSTDumpTool::Run(int argc, char** argv) {
575647
if (read_num > 0 && total_read > read_num) {
576648
break;
577649
}
650+
} else if (command == "check_data_blocks") {
651+
ERROR_NOT_OK(reader.CheckDataBlocks(do_uncompress), "Failed to scan SST file blocks:");
578652
}
579653
if (show_properties) {
580654
const rocksdb::TableProperties* table_properties;

‎src/yb/rocksdb/tools/sst_dump_tool_imp.h

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,17 @@
2525

2626
#include <memory>
2727
#include <string>
28-
#include "yb/rocksdb/rocksdb_fwd.h"
28+
2929
#include "yb/rocksdb/db/dbformat.h"
3030
#include "yb/rocksdb/immutable_options.h"
31+
#include "yb/rocksdb/rocksdb_fwd.h"
32+
#include "yb/rocksdb/table/format.h"
3133
#include "yb/rocksdb/util/file_reader_writer.h"
3234

3335
namespace rocksdb {
3436

37+
YB_STRONGLY_TYPED_BOOL(DoUncompress);
38+
3539
class SstFileReader {
3640
public:
3741
SstFileReader(
@@ -43,6 +47,8 @@ class SstFileReader {
4347
const std::string& from_key, bool has_to,
4448
const std::string& to_key);
4549

50+
Status CheckDataBlocks(DoUncompress do_uncompress);
51+
4652
Status ReadTableProperties(
4753
std::shared_ptr<const TableProperties>* table_properties);
4854
uint64_t GetReadNumber() { return read_num_; }
@@ -81,6 +87,7 @@ class SstFileReader {
8187
EnvOptions soptions_;
8288

8389
Status init_result_;
90+
Footer footer_;
8491
std::unique_ptr<TableReader> table_reader_;
8592
std::unique_ptr<RandomAccessFileReader> file_;
8693
// options_ and internal_comparator_ will also be used in

0 commit comments

Comments
 (0)