From 4756db4d29d0c54fda9d264ca777ccf50273d83e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 15 Dec 2022 17:11:00 +0100 Subject: [PATCH] wip: dump waaaaaay too much stuff --- Cargo.lock | 25 ++++++++++++++++++ src/db/Cargo.toml | 7 +++++ src/db/lib.rs | 52 ++++++++++++++++++++++++++++++++++--- src/db/sled_adapter.rs | 9 +++++++ src/garage/repair/online.rs | 15 ++++++----- 5 files changed, 98 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8751d17..fa863225 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1172,10 +1172,15 @@ dependencies = [ "clap 3.1.18", "err-derive", "heed", + "hex", "hexdump", "mktemp", + "nettext", "pretty_env_logger", + "rmp-serde", "rusqlite", + "serde", + "serde-transcode", "sled", "tracing", ] @@ -2141,6 +2146,17 @@ dependencies = [ "tokio-util 0.7.0", ] +[[package]] +name = "nettext" +version = "0.3.2" +dependencies = [ + "base64", + "err-derive", + "hex", + "nom", + "serde", +] + [[package]] name = "nom" version = "7.1.1" @@ -3282,6 +3298,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-transcode" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590c0e25c2a5bb6e85bf5c1bce768ceb86b316e7a01bdf07d2cb4ec2271990e2" +dependencies = [ + "serde", +] + [[package]] name = "serde-value" version = "0.7.0" diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 82cf49dc..a0034169 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -18,6 +18,7 @@ required-features = ["cli"] [dependencies] err-derive = "0.3" +hex = "0.4" hexdump = "0.1" tracing = "0.1.30" @@ -25,6 +26,11 @@ heed = { version = "0.11", default-features = false, features = ["lmdb"], option rusqlite = { version = "0.27", optional = true } sled = { version = "0.34", optional = true } +rmp-serde = { version = "0.15", optional = true } +serde = { version = "1.0", default-features = false, features = ["derive", "rc"], optional = true } +serde-transcode = { version = "1.1", optional = true } +nettext = { version = "0.3.2", default-features = false, features = ["serde"], optional = true, path = "../../../nettext" } + # cli deps clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } pretty_env_logger = { version = "0.4", optional = true } @@ -38,3 +44,4 @@ bundled-libs = [ "rusqlite/bundled" ] cli = ["clap", "pretty_env_logger"] lmdb = [ "heed" ] sqlite = [ "rusqlite" ] +debuglog = [ "serde", "rmp-serde", "nettext", "serde-transcode" ] diff --git a/src/db/lib.rs b/src/db/lib.rs index 11cae4e3..cd2774fa 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -173,9 +173,22 @@ impl Tree { Db(self.0.clone()) } + #[inline] + pub fn name(&self) -> Option { + self.0.tree_name(self.1) + } + #[inline] pub fn get>(&self, key: T) -> Result> { - self.0.get(self.1, key.as_ref()) + let res = self.0.get(self.1, key.as_ref())?; + #[cfg(feature = "debuglog")] + debuglog( + self.name(), + "GET", + key.as_ref(), + res.as_deref().unwrap_or(b"-"), + ); + Ok(res) } #[inline] pub fn len(&self) -> Result { @@ -204,7 +217,10 @@ impl Tree { key: T, value: U, ) -> Result> { - self.0.insert(self.1, key.as_ref(), value.as_ref()) + let res = self.0.insert(self.1, key.as_ref(), value.as_ref())?; + #[cfg(feature = "debuglog")] + debuglog(self.name(), "PUT", key.as_ref(), value.as_ref()); + Ok(res) } /// Returns the old value if there was one #[inline] @@ -267,7 +283,10 @@ impl<'a> Transaction<'a> { key: T, value: U, ) -> TxOpResult> { - self.0.insert(tree.1, key.as_ref(), value.as_ref()) + let res = self.0.insert(tree.1, key.as_ref(), value.as_ref())?; + #[cfg(feature = "debuglog")] + debuglog(tree.name(), "txPUT", key.as_ref(), value.as_ref()); + Ok(res) } /// Returns the old value if there was one #[inline] @@ -324,6 +343,7 @@ pub(crate) trait IDb: Send + Sync { fn engine(&self) -> String; fn open_tree(&self, name: &str) -> Result; fn list_trees(&self) -> Result>; + fn tree_name(&self, tree: usize) -> Option; fn get(&self, tree: usize, key: &[u8]) -> Result>; fn len(&self, tree: usize) -> Result; @@ -421,3 +441,29 @@ fn get_bound>(b: Bound<&K>) -> Bound<&[u8]> { Bound::Unbounded => Bound::Unbounded, } } + +#[cfg(feature = "debuglog")] +fn debuglog(tree: Option, action: &str, k: &[u8], v: &[u8]) { + let key = String::from_utf8(nettext::switch64::encode(k, false)).unwrap(); + let tree = tree.as_deref().unwrap_or("(?)"); + if let Ok(vstr) = std::str::from_utf8(v) { + eprintln!("{} {} {} S:{}", tree, action, key, vstr); + } else { + let mut vread = &v[..]; + let mut vder = rmp_serde::decode::Deserializer::new(&mut vread); + let mut vser = nettext::serde::Serializer { + string_format: nettext::BytesEncoding::Switch64 { + allow_whitespace: true, + }, + bytes_format: nettext::BytesEncoding::Hex { split: true }, + }; + if let Some(venc) = serde_transcode::transcode(&mut vder, &mut vser) + .ok() + .and_then(|x| String::from_utf8(x.encode_concise()).ok()) + { + eprintln!("{} {} {} N:{}", tree, action, key, venc); + } else { + eprintln!("{} {} {} X:{}", tree, action, key, hex::encode(v)); + } + } +} diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index cf61867d..31a4889f 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -88,6 +88,15 @@ impl IDb for SledDb { Ok(trees) } + fn tree_name(&self, tree: usize) -> Option { + self.trees + .read() + .unwrap() + .0 + .get(tree) + .and_then(|x| String::from_utf8(x.name().to_vec()).ok()) + } + // ---- fn get(&self, tree: usize, key: &[u8]) -> Result> { diff --git a/src/garage/repair/online.rs b/src/garage/repair/online.rs index 6e8ec2d3..7120972c 100644 --- a/src/garage/repair/online.rs +++ b/src/garage/repair/online.rs @@ -171,13 +171,14 @@ impl Worker for RepairBlockrefsWorker { } async fn work(&mut self, _must_exit: &mut watch::Receiver) -> Result { - let (item_bytes, next_pos) = match self.garage.block_ref_table.data.store.get_gt(&self.pos)? { - Some((k, v)) => (v, k), - None => { - info!("repair_block_ref: finished, done {}", self.counter); - return Ok(WorkerState::Done); - } - }; + let (item_bytes, next_pos) = + match self.garage.block_ref_table.data.store.get_gt(&self.pos)? { + Some((k, v)) => (v, k), + None => { + info!("repair_block_ref: finished, done {}", self.counter); + return Ok(WorkerState::Done); + } + }; let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?; if !block_ref.deleted.get() {