Compare commits

...

1 commit

Author SHA1 Message Date
4756db4d29
wip: dump waaaaaay too much stuff 2022-12-15 17:11:00 +01:00
5 changed files with 98 additions and 10 deletions

25
Cargo.lock generated
View file

@ -1172,10 +1172,15 @@ dependencies = [
"clap 3.1.18", "clap 3.1.18",
"err-derive", "err-derive",
"heed", "heed",
"hex",
"hexdump", "hexdump",
"mktemp", "mktemp",
"nettext",
"pretty_env_logger", "pretty_env_logger",
"rmp-serde",
"rusqlite", "rusqlite",
"serde",
"serde-transcode",
"sled", "sled",
"tracing", "tracing",
] ]
@ -2141,6 +2146,17 @@ dependencies = [
"tokio-util 0.7.0", "tokio-util 0.7.0",
] ]
[[package]]
name = "nettext"
version = "0.3.2"
dependencies = [
"base64",
"err-derive",
"hex",
"nom",
"serde",
]
[[package]] [[package]]
name = "nom" name = "nom"
version = "7.1.1" version = "7.1.1"
@ -3282,6 +3298,15 @@ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]]
name = "serde-transcode"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "590c0e25c2a5bb6e85bf5c1bce768ceb86b316e7a01bdf07d2cb4ec2271990e2"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "serde-value" name = "serde-value"
version = "0.7.0" version = "0.7.0"

View file

@ -18,6 +18,7 @@ required-features = ["cli"]
[dependencies] [dependencies]
err-derive = "0.3" err-derive = "0.3"
hex = "0.4"
hexdump = "0.1" hexdump = "0.1"
tracing = "0.1.30" tracing = "0.1.30"
@ -25,6 +26,11 @@ heed = { version = "0.11", default-features = false, features = ["lmdb"], option
rusqlite = { version = "0.27", optional = true } rusqlite = { version = "0.27", optional = true }
sled = { version = "0.34", optional = true } sled = { version = "0.34", optional = true }
rmp-serde = { version = "0.15", optional = true }
serde = { version = "1.0", default-features = false, features = ["derive", "rc"], optional = true }
serde-transcode = { version = "1.1", optional = true }
nettext = { version = "0.3.2", default-features = false, features = ["serde"], optional = true, path = "../../../nettext" }
# cli deps # cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
pretty_env_logger = { version = "0.4", optional = true } pretty_env_logger = { version = "0.4", optional = true }
@ -38,3 +44,4 @@ bundled-libs = [ "rusqlite/bundled" ]
cli = ["clap", "pretty_env_logger"] cli = ["clap", "pretty_env_logger"]
lmdb = [ "heed" ] lmdb = [ "heed" ]
sqlite = [ "rusqlite" ] sqlite = [ "rusqlite" ]
debuglog = [ "serde", "rmp-serde", "nettext", "serde-transcode" ]

View file

@ -173,9 +173,22 @@ impl Tree {
Db(self.0.clone()) Db(self.0.clone())
} }
#[inline]
pub fn name(&self) -> Option<String> {
self.0.tree_name(self.1)
}
#[inline] #[inline]
pub fn get<T: AsRef<[u8]>>(&self, key: T) -> Result<Option<Value>> { pub fn get<T: AsRef<[u8]>>(&self, key: T) -> Result<Option<Value>> {
self.0.get(self.1, key.as_ref()) let res = self.0.get(self.1, key.as_ref())?;
#[cfg(feature = "debuglog")]
debuglog(
self.name(),
"GET",
key.as_ref(),
res.as_deref().unwrap_or(b"-"),
);
Ok(res)
} }
#[inline] #[inline]
pub fn len(&self) -> Result<usize> { pub fn len(&self) -> Result<usize> {
@ -204,7 +217,10 @@ impl Tree {
key: T, key: T,
value: U, value: U,
) -> Result<Option<Value>> { ) -> Result<Option<Value>> {
self.0.insert(self.1, key.as_ref(), value.as_ref()) let res = self.0.insert(self.1, key.as_ref(), value.as_ref())?;
#[cfg(feature = "debuglog")]
debuglog(self.name(), "PUT", key.as_ref(), value.as_ref());
Ok(res)
} }
/// Returns the old value if there was one /// Returns the old value if there was one
#[inline] #[inline]
@ -267,7 +283,10 @@ impl<'a> Transaction<'a> {
key: T, key: T,
value: U, value: U,
) -> TxOpResult<Option<Value>> { ) -> TxOpResult<Option<Value>> {
self.0.insert(tree.1, key.as_ref(), value.as_ref()) let res = self.0.insert(tree.1, key.as_ref(), value.as_ref())?;
#[cfg(feature = "debuglog")]
debuglog(tree.name(), "txPUT", key.as_ref(), value.as_ref());
Ok(res)
} }
/// Returns the old value if there was one /// Returns the old value if there was one
#[inline] #[inline]
@ -324,6 +343,7 @@ pub(crate) trait IDb: Send + Sync {
fn engine(&self) -> String; fn engine(&self) -> String;
fn open_tree(&self, name: &str) -> Result<usize>; fn open_tree(&self, name: &str) -> Result<usize>;
fn list_trees(&self) -> Result<Vec<String>>; fn list_trees(&self) -> Result<Vec<String>>;
fn tree_name(&self, tree: usize) -> Option<String>;
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>; fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
fn len(&self, tree: usize) -> Result<usize>; fn len(&self, tree: usize) -> Result<usize>;
@ -421,3 +441,29 @@ fn get_bound<K: AsRef<[u8]>>(b: Bound<&K>) -> Bound<&[u8]> {
Bound::Unbounded => Bound::Unbounded, Bound::Unbounded => Bound::Unbounded,
} }
} }
#[cfg(feature = "debuglog")]
fn debuglog(tree: Option<String>, action: &str, k: &[u8], v: &[u8]) {
let key = String::from_utf8(nettext::switch64::encode(k, false)).unwrap();
let tree = tree.as_deref().unwrap_or("(?)");
if let Ok(vstr) = std::str::from_utf8(v) {
eprintln!("{} {} {} S:{}", tree, action, key, vstr);
} else {
let mut vread = &v[..];
let mut vder = rmp_serde::decode::Deserializer::new(&mut vread);
let mut vser = nettext::serde::Serializer {
string_format: nettext::BytesEncoding::Switch64 {
allow_whitespace: true,
},
bytes_format: nettext::BytesEncoding::Hex { split: true },
};
if let Some(venc) = serde_transcode::transcode(&mut vder, &mut vser)
.ok()
.and_then(|x| String::from_utf8(x.encode_concise()).ok())
{
eprintln!("{} {} {} N:{}", tree, action, key, venc);
} else {
eprintln!("{} {} {} X:{}", tree, action, key, hex::encode(v));
}
}
}

View file

@ -88,6 +88,15 @@ impl IDb for SledDb {
Ok(trees) Ok(trees)
} }
fn tree_name(&self, tree: usize) -> Option<String> {
self.trees
.read()
.unwrap()
.0
.get(tree)
.and_then(|x| String::from_utf8(x.name().to_vec()).ok())
}
// ---- // ----
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> { fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {

View file

@ -171,7 +171,8 @@ impl Worker for RepairBlockrefsWorker {
} }
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> { async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
let (item_bytes, next_pos) = match self.garage.block_ref_table.data.store.get_gt(&self.pos)? { let (item_bytes, next_pos) =
match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
Some((k, v)) => (v, k), Some((k, v)) => (v, k),
None => { None => {
info!("repair_block_ref: finished, done {}", self.counter); info!("repair_block_ref: finished, done {}", self.counter);