From 37fe9e7387c7189be45a021499a65fa56dc49923 Mon Sep 17 00:00:00 2001 From: Timothy Quilling Date: Sun, 28 Dec 2025 17:35:34 -0500 Subject: [PATCH] chore: rearrange parakeet-db --- .../{compression/mod.rs => compression.rs} | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) rename parakeet-db/src/{compression/mod.rs => compression.rs} (91%) diff --git a/parakeet-db/src/compression/mod.rs b/parakeet-db/src/compression.rs similarity index 91% rename from parakeet-db/src/compression/mod.rs rename to parakeet-db/src/compression.rs index 1f133d53..bc2a8866 100644 --- a/parakeet-db/src/compression/mod.rs +++ b/parakeet-db/src/compression.rs @@ -2,7 +2,7 @@ use eyre::{Context, Result}; use std::sync::Arc; use zstd::bulk::{Compressor, Decompressor}; -const DICT_BYTES: &[u8] = include_bytes!("../dicts/post_content_v1.dict"); +const DICT_BYTES: &[u8] = include_bytes!("./dicts/post_content_v1.dict"); const DICT_VERSION: u8 = 1; // Changed to u8 (1 byte instead of i16) const COMPRESSION_LEVEL: i32 = 3; const MAX_DECOMPRESSED_SIZE: usize = 300_000; // 300KB max (Bluesky's current limit) @@ -47,7 +47,7 @@ impl PostContentCodec { // Determine where to strip (magic+FHD+dict = 9 bytes) let fhd = compressed[4]; let magic_fhd_dict_size = match fhd { - 0x23 | 0x63 => 9, // Always strip magic(4) + FHD(1) + dict(4) = 9 bytes + 0x23 | 0x63 => 9, // Always strip magic(4) + FHD(1) + dict(4) = 9 bytes _ => eyre::bail!("Unexpected frame header descriptor: 0x{:02x}", fhd), }; @@ -240,7 +240,6 @@ mod tests { assert_eq!(original, decompressed); } - #[test] fn test_both_fhd_types() { let codec = PostContentCodec::new(); @@ -253,30 +252,45 @@ mod tests { println!("FHD for short post: 0x{:02x}", compressed_short[1]); // Test a post that will use FHD=0x63 (longer content, with FCS) - let long_content = "This is a much longer post with a lot of content that should trigger FHD=0x63. ".repeat(10); + let long_content = + "This is a much longer post with a lot of content that should trigger FHD=0x63. " + .repeat(10); println!("Original content length: {}", long_content.len()); // First, compress with raw zstd to see what it produces - let mut raw_compressor = Compressor::with_dictionary(COMPRESSION_LEVEL, DICT_BYTES).unwrap(); + let mut raw_compressor = + Compressor::with_dictionary(COMPRESSION_LEVEL, DICT_BYTES).unwrap(); let raw_compressed = raw_compressor.compress(long_content.as_bytes()).unwrap(); println!("Raw zstd compressed length: {}", raw_compressed.len()); println!("Raw FHD: 0x{:02x}", raw_compressed[4]); if raw_compressed[4] == 0x63 { let fcs_bytes = &raw_compressed[9..11]; let fcs_value = u16::from_le_bytes([fcs_bytes[0], fcs_bytes[1]]); - println!("Raw FCS bytes: {:02x} {:02x}, value: {}, indicated size: {}", - fcs_bytes[0], fcs_bytes[1], fcs_value, fcs_value as usize + 256); + println!( + "Raw FCS bytes: {:02x} {:02x}, value: {}, indicated size: {}", + fcs_bytes[0], + fcs_bytes[1], + fcs_value, + fcs_value as usize + 256 + ); } // Now compress with our codec let compressed_long = codec.compress(&long_content).unwrap(); println!("Codec FHD: 0x{:02x}", compressed_long[1]); - println!("Codec compressed length: {}, First 12 bytes: {:02x?}", compressed_long.len(), &compressed_long[..12.min(compressed_long.len())]); + println!( + "Codec compressed length: {}, First 12 bytes: {:02x?}", + compressed_long.len(), + &compressed_long[..12.min(compressed_long.len())] + ); // Try to decompress match codec.decompress(&compressed_long) { Ok(decompressed_long) => { - println!("Decompressed successfully! Length: {}", decompressed_long.len()); + println!( + "Decompressed successfully! Length: {}", + decompressed_long.len() + ); assert_eq!(long_content, decompressed_long); } Err(e) => { -- 2.51.2