Closed
Description
Both measurements were taken for lz4_flex version 0.11.1. Only bumping the Rust compiler version caused the regression.
What I find very interesting is that essentially the same regression has occurred for the lz4 crate, and it seems both regressions have only occurred for decompression of the frame format, not the block format.
I think the next step is to investigate which specific scope of code in the frame decompression has gotten slower.
Benchmark
use std::{
fs::File,
io::Cursor,
path::{Path, PathBuf},
time::Instant,
};
fn main() {
for entry_result in std::fs::read_dir("./data").unwrap() {
let entry = entry_result.unwrap();
do_bench(
"lz4_flex",
&entry.path(),
compress_lz4_flex,
decompress_lz4_flex,
);
// do_bench("lz4", &entry.path(), compress_lz4, decompress_lz4);
}
}
fn do_bench(
algo_name: &str,
input_path: &Path,
compress_fn: impl Fn(&Path, File) -> File,
decompress_fn: impl Fn(&[u8]) -> Vec<u8>,
) {
let in_f = std::fs::File::open(input_path).unwrap();
let start_size = in_f.metadata().unwrap().len();
let out_path = PathBuf::from(format!(
"output/out-{}.lz4",
input_path.file_stem().unwrap().to_str().unwrap()
));
let compress_start = Instant::now();
let output = compress_fn(&out_path, in_f);
let compress_elapsed = compress_start.elapsed().as_millis();
let end_size = output.metadata().unwrap().len();
println!(
"[{algo_name} {}] Compression: {:.4}%, Time: {:.2}s",
input_path.display(),
100.0 * end_size as f32 / start_size as f32,
compress_elapsed as f32 / 1000.0,
);
let compressed_bytes = std::fs::read(&out_path).unwrap();
let decode_start = Instant::now();
let decompressed_buf = decompress_fn(&compressed_bytes);
println!(
"Decompress in: {:.5}s",
decode_start.elapsed().as_millis() as f32 / 1000.0
);
let out_path = format!(
"output/roundtrip-{}.lz4",
input_path.file_stem().unwrap().to_str().unwrap()
);
std::fs::write(out_path, decompressed_buf).unwrap();
}
fn compress_lz4_flex(out_path: &Path, mut in_f: std::fs::File) -> std::fs::File {
let out_f = std::fs::File::create(out_path).unwrap();
let mut encoder = lz4_flex::frame::FrameEncoder::new(out_f);
std::io::copy(&mut in_f, &mut encoder).unwrap();
encoder.finish().unwrap()
}
fn decompress_lz4_flex(compressed_bytes: &[u8]) -> Vec<u8> {
let mut decoder = lz4_flex::frame::FrameDecoder::new(Cursor::new(compressed_bytes));
let mut uncompressed_buf = Vec::new();
std::io::copy(&mut decoder, &mut uncompressed_buf).unwrap();
uncompressed_buf
}
// fn compress_lz4(out_path: &Path, mut in_f: File) -> File {
// let out_f = std::fs::File::create(out_path).unwrap();
// let mut encoder = lz4::EncoderBuilder::new().build(out_f).unwrap();
// std::io::copy(&mut in_f, &mut encoder).unwrap();
// let (out_f, maybe_err) = encoder.finish();
// maybe_err.unwrap();
// out_f
// }
// fn decompress_lz4(decompress_bytes: &[u8]) -> Vec<u8> {
// let mut decoder = lz4::Decoder::new(Cursor::new(decompress_bytes)).unwrap();
// let mut uncompressed_buf = Vec::new();
// std::io::copy(&mut decoder, &mut uncompressed_buf).unwrap();
// uncompressed_buf
// }Metadata
Metadata
Assignees
Labels
No labels