Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/next/src/cli/next-build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ export type NextBuildOptions = {
}

const nextBuild = async (options: NextBuildOptions, directory?: string) => {
process.title = `next-build (v${process.env.__NEXT_VERSION})`
process.on('SIGTERM', () => {
saveCpuProfile()
process.exit(143)
Expand Down
76 changes: 56 additions & 20 deletions turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ use turbo_persistence::{
BLOCK_HEADER_SIZE, checksum_block,
meta_file::MetaFile,
mmap_helper::advise_mmap_for_persistence,
sst_filter::SstFilter,
static_sorted_file::{
BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, BLOCK_TYPE_KEY_NO_HASH,
BLOCK_TYPE_KEY_WITH_HASH, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED,
Expand Down Expand Up @@ -205,33 +206,68 @@ fn format_bytes(bytes: u64) -> String {
}
}

/// Collect SST info from all meta files in the database directory
/// Collect SST info from all active meta files in the database directory,
/// mirroring the DB's own open logic: read CURRENT, filter by .del files,
/// and apply SstFilter to skip superseded entries.
fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> {
let mut meta_files: Vec<PathBuf> = fs::read_dir(db_path)?
.filter_map(|entry| entry.ok())
.map(|entry| entry.path())
.filter(|path| path.extension().is_some_and(|ext| ext == "meta"))
.collect();
// Read the CURRENT sequence number — only files with seq <= current are valid.
let current: u32 = File::open(db_path.join("CURRENT"))
.context("Failed to open CURRENT file")?
.read_u32::<BE>()
.context("Failed to read CURRENT file")?;

// Read .del files to find sequences that were deleted but not yet cleaned up.
let mut deleted_seqs: HashSet<u32> = HashSet::new();
for entry in fs::read_dir(db_path)? {
let path = entry?.path();
if path.extension().and_then(|s| s.to_str()) == Some("del") {
let content = fs::read(&path)?;
let mut cursor: &[u8] = &content;
while !cursor.is_empty() {
deleted_seqs.insert(cursor.read_u32::<BE>()?);
}
}
}

meta_files.sort();
// Collect valid meta sequence numbers.
let mut meta_seqs: Vec<u32> = fs::read_dir(db_path)?
.filter_map(|e| e.ok())
.filter_map(|e| {
let path = e.path();
if path.extension().and_then(|s| s.to_str()) != Some("meta") {
return None;
}
let seq: u32 = path.file_stem()?.to_str()?.parse().ok()?;
if seq > current || deleted_seqs.contains(&seq) {
return None;
}
Some(seq)
})
.collect();

if meta_files.is_empty() {
bail!("No .meta files found in {}", db_path.display());
if meta_seqs.is_empty() {
bail!("No active .meta files found in {}", db_path.display());
}

let mut family_sst_info: BTreeMap<u32, Vec<SstInfo>> = BTreeMap::new();

for meta_path in &meta_files {
// Extract sequence number from filename
let filename = meta_path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
let seq_num: u32 = filename.parse().unwrap_or(0);
meta_seqs.sort_unstable();

let meta_file = MetaFile::open(db_path, seq_num)
.with_context(|| format!("Failed to open {}", meta_path.display()))?;
let mut meta_files: Vec<MetaFile> = meta_seqs
.iter()
.map(|&seq| {
MetaFile::open(db_path, seq).with_context(|| format!("Failed to open {seq:08}.meta"))
})
.collect::<Result<_>>()?;

let family = meta_file.family();
// Apply SstFilter (newest first) to drop entries superseded by a newer meta file.
let mut sst_filter = SstFilter::new();
for meta in meta_files.iter_mut().rev() {
sst_filter.apply_filter(meta);
}

for entry in meta_file.entries() {
let mut family_sst_info: BTreeMap<u32, Vec<SstInfo>> = BTreeMap::new();
for meta in &meta_files {
let family = meta.family();
for entry in meta.entries() {
family_sst_info.entry(family).or_default().push(SstInfo {
sequence_number: entry.sequence_number(),
block_count: entry.block_count(),
Expand Down Expand Up @@ -317,7 +353,7 @@ fn read_block(
///
/// Index block format: `[1B type][2B first_block][N * (8B hash + 2B block_index)]`.
fn parse_key_block_indices(index_block: &[u8]) -> HashSet<u16> {
assert!(index_block.len() >= 4, "Index block too small");
assert!(index_block.len() >= 3, "Index block too small");
let mut data = &index_block[1..]; // skip block type byte
let first_block = data.read_u16::<BE>().unwrap();
let mut indices = HashSet::new();
Expand Down
2 changes: 1 addition & 1 deletion turbopack/crates/turbo-persistence/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ pub mod mmap_helper;
mod parallel_scheduler;
mod rc_bytes;
mod shared_bytes;
mod sst_filter;
pub mod sst_filter;
pub mod static_sorted_file;
mod static_sorted_file_builder;
mod value_block_count_tracker;
Expand Down
6 changes: 6 additions & 0 deletions turbopack/crates/turbo-persistence/src/sst_filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,9 @@ impl SstFilter {
!used && !meta.has_active_entries()
}
}

impl Default for SstFilter {
fn default() -> Self {
Self::new()
}
}
Loading