From f25309e58f839d1fad6b094cd33d7f36b5c3e2e0 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 10:44:54 +0200 Subject: [PATCH] Change value type to be a dyn thing --- Cargo.lock | 1 + src/block/manager.rs | 2 +- src/db/Cargo.toml | 1 + src/db/lib.rs | 87 +++++++++++++++++++++++++++++++++++++++- src/db/sled_adapter.rs | 55 ++++++++++++++++--------- src/db/sqlite_adapter.rs | 4 ++ src/db/test.rs | 37 ++++++++++------- src/garage/repair.rs | 4 +- src/table/data.rs | 2 +- src/table/gc.rs | 7 +++- src/table/merkle.rs | 11 ++--- 11 files changed, 166 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5309a79..9ff2c02d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1015,6 +1015,7 @@ name = "garage_db" version = "0.8.0" dependencies = [ "err-derive 0.3.1", + "hexdump", "mktemp", "rusqlite", "sled", diff --git a/src/block/manager.rs b/src/block/manager.rs index f34d13d0..de9b08c2 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -645,7 +645,7 @@ impl BlockManager { None => Ok(None), Some(v) => { let (time_bytes, hash_bytes) = v?; - Ok(Some((time_bytes.into_owned(), hash_bytes.into_owned()))) + Ok(Some((time_bytes.into_vec(), hash_bytes.into_vec()))) } } } diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index f627208b..ca189a67 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -15,6 +15,7 @@ path = "lib.rs" [dependencies] err-derive = "0.3" +hexdump = "0.1" sled = "0.34" rusqlite = "0.27" diff --git a/src/db/lib.rs b/src/db/lib.rs index 3a2e1d13..4cda121f 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -21,13 +21,98 @@ pub struct Transaction<'a>(pub(crate) &'a dyn ITx<'a>); #[derive(Clone)] pub struct Tree(pub(crate) Arc, pub(crate) usize); -pub type Value<'a> = Cow<'a, [u8]>; pub type ValueIter<'a> = Box, Value<'a>)>> + 'a>; pub type Exporter<'a> = Box)>> + 'a>; // ---- +pub struct Value<'a>(pub(crate) Box + 'a>); + +pub trait IValue<'a>: AsRef<[u8]> { + fn into_vec(&mut self) -> Vec; +} + +impl<'a> Value<'a> { + #[inline] + pub fn into_vec(mut self) -> Vec { + self.0.into_vec() + } +} + +impl<'a> AsRef<[u8]> for Value<'a> { + #[inline] + fn as_ref(&self) -> &[u8] { + self.0.as_ref().as_ref() + } +} + +impl<'a> std::borrow::Borrow<[u8]> for Value<'a> { + #[inline] + fn borrow(&self) -> &[u8] { + self.0.as_ref().as_ref() + } +} + +impl<'a> core::ops::Deref for Value<'a> { + type Target = [u8]; + #[inline] + fn deref(&self) -> &[u8] { + self.0.as_ref().as_ref() + } +} + +impl<'a, T> PartialEq for Value<'a> +where + T: AsRef<[u8]>, +{ + fn eq(&self, other: &T) -> bool { + self.as_ref() == other.as_ref() + } +} + +impl<'a> std::fmt::Debug for Value<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for line in hexdump::hexdump_iter(self.as_ref()) { + f.write_str(&line)?; + f.write_str("\n")?; + } + Ok(()) + } +} + +impl<'a> IValue<'a> for Vec { + fn into_vec(&mut self) -> Vec { + std::mem::take(self) + } +} + +impl<'a> From> for Vec { + fn from(v: Value<'a>) -> Vec { + v.into_vec() + } +} + +impl<'a> From> for Value<'a> { + fn from(v: Vec) -> Value<'a> { + Value(Box::new(v)) + } +} + +impl<'a> From<&'a [u8]> for Value<'a> { + fn from(v: &'a [u8]) -> Value<'a> { + Value(Box::new(v)) + } +} + +impl<'a> IValue<'a> for &'a [u8] { + fn into_vec(&mut self) -> Vec { + self.to_vec() + } +} + +// ---- + #[derive(Debug, Error)] #[error(display = "{}", _0)] pub struct Error(Cow<'static, str>); diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index c296708b..acee305b 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -10,17 +10,42 @@ use sled::transaction::{ }; use crate::{ - Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, + Db, Error, Exporter, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, + ValueIter, }; pub use sled; +// -- err + impl From for Error { fn from(e: sled::Error) -> Error { Error(format!("{}", e).into()) } } +// -- val + +impl<'a> IValue<'a> for sled::IVec { + fn into_vec(&mut self) -> Vec { + self.to_vec() + } +} + +impl<'a> From> for sled::IVec { + fn from(v: Value<'a>) -> sled::IVec { + sled::IVec::from(v.into_vec()) + } +} + +impl<'a> From for Value<'a> { + fn from(v: sled::IVec) -> Value<'a> { + Value(Box::new(v)) + } +} + +// -- db + pub struct SledDb { db: sled::Db, trees: RwLock<(Vec, HashMap)>, @@ -64,7 +89,7 @@ impl IDb for SledDb { fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; - Ok(tree.get(key)?.map(|v| v.to_vec().into())) + Ok(tree.get(key)?.map(From::from)) } fn remove(&self, tree: usize, key: &[u8]) -> Result { @@ -86,16 +111,14 @@ impl IDb for SledDb { fn iter<'a>(&'a self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) + v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) }))) } fn iter_rev<'a>(&'a self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.iter().rev().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) + v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) }))) } @@ -107,8 +130,7 @@ impl IDb for SledDb { ) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) + v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) }))) } fn range_rev<'a, 'r>( @@ -119,10 +141,7 @@ impl IDb for SledDb { ) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).rev().map( - |v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - }, + |v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into), ))) } @@ -170,10 +189,10 @@ impl IDb for SledDb { trees.push((name, tree)); } let trees_exporter: Exporter<'a> = Box::new(trees.into_iter().map(|(name, tree)| { - let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - })); + let iter: ValueIter<'a> = Box::new( + tree.iter() + .map(|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)), + ); Ok((name.to_string(), iter)) })); Ok(trees_exporter) @@ -192,7 +211,7 @@ impl IDb for SledDb { let mut i = 0; for item in data { let (k, v) = item?; - tree.insert(k.as_ref(), v.as_ref())?; + tree.insert(k, v)?; i += 1; if i % 1000 == 0 { println!("{}: imported {}", name, i); @@ -236,7 +255,7 @@ impl<'a> ITx<'a> for SledTx<'a> { fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; let tmp = self.save_error(tree.get(key))?; - Ok(tmp.map(|v| v.to_vec().into())) + Ok(tmp.map(From::from)) } fn len(&self, _tree: usize) -> Result { unimplemented!(".len() in transaction not supported with Sled backend") diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 5778ecf0..701639dc 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -13,6 +13,8 @@ use crate::{ pub use rusqlite; +// --- err + impl From for Error { fn from(e: rusqlite::Error) -> Error { Error(format!("{}", e).into()) @@ -25,6 +27,8 @@ impl From for TxError { } } +// -- db + pub struct SqliteDb { db: Mutex, trees: RwLock>, diff --git a/src/db/test.rs b/src/db/test.rs index 78d755cf..2f5e4c46 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -14,59 +14,66 @@ fn test_suite(db: Db) { let vc: &[u8] = &b"plup"[..]; tree.insert(ka, va).unwrap(); - assert_eq!(tree.get(ka).unwrap(), Some(va.into())); + assert_eq!(tree.get(ka).unwrap().unwrap(), va); let res = db.transaction::<_, (), _>(|tx| { - assert_eq!(tx.get(&tree, ka).unwrap(), Some(va.into())); + assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), va); tx.insert(&tree, ka, vb).unwrap(); - assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into())); + assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); tx.commit(12) }); assert!(matches!(res, Ok(12))); - assert_eq!(tree.get(ka).unwrap(), Some(vb.into())); + assert_eq!(tree.get(ka).unwrap().unwrap(), vb); let res = db.transaction::<(), _, _>(|tx| { - assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into())); + assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); tx.insert(&tree, ka, vc).unwrap(); - assert_eq!(tx.get(&tree, ka).unwrap(), Some(vc.into())); + assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vc); tx.abort(42) }); assert!(matches!(res, Err(TxError::Abort(42)))); - assert_eq!(tree.get(ka).unwrap(), Some(vb.into())); + assert_eq!(tree.get(ka).unwrap().unwrap(), vb); let mut iter = tree.iter().unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); tree.insert(kb, vc).unwrap(); - assert_eq!(tree.get(kb).unwrap(), Some(vc.into())); + assert_eq!(tree.get(kb).unwrap().unwrap(), vc); let mut iter = tree.iter().unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); - assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); assert!(iter.next().is_none()); drop(iter); let mut iter = tree.range(kint..).unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); assert!(iter.next().is_none()); drop(iter); let mut iter = tree.range_rev(..kint).unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); let mut iter = tree.iter_rev().unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); - assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); } diff --git a/src/garage/repair.rs b/src/garage/repair.rs index 762a8300..a096aaa0 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -117,7 +117,7 @@ impl Repair { None => Ok(None), Some(item) => { let (item_key, item_bytes) = item?; - Ok(Some((item_key.into_owned(), item_bytes.into_owned()))) + Ok(Some((item_key.into_vec(), item_bytes.into_vec()))) } } } @@ -173,7 +173,7 @@ impl Repair { None => Ok(None), Some(item) => { let (item_key, item_bytes) = item?; - Ok(Some((item_key.into_owned(), item_bytes.into_owned()))) + Ok(Some((item_key.into_vec(), item_bytes.into_vec()))) } } } diff --git a/src/table/data.rs b/src/table/data.rs index ebfae551..dc1fd445 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -276,7 +276,7 @@ where if blake2sum(&cur_v[..]) == vhash { tx.remove(&self.store, k)?; tx.insert(&self.merkle_todo, k, vec![])?; - return Ok(Some(cur_v.into_owned())); + return Ok(Some(cur_v.into_vec())); } } Ok(None) diff --git a/src/table/gc.rs b/src/table/gc.rs index 04872a38..260fecfa 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -378,8 +378,11 @@ impl GcTodoEntry { let key = self.todo_table_key(); gc_todo_tree.db().transaction(|tx| { let old_val = tx.get(gc_todo_tree, &key)?; - if old_val == Some(self.value_hash.as_slice().into()) { - tx.remove(gc_todo_tree, &key)?; + match old_val { + Some(ov) if ov == self.value_hash.as_slice() => { + tx.remove(gc_todo_tree, &key)?; + } + _ => (), } tx.commit(()) })?; diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 6e0c2f7e..f7dca97b 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -142,11 +142,12 @@ where let deleted = self.data.merkle_todo.db().transaction(|tx| { let old_val = tx.get(&self.data.merkle_todo, k)?; - if old_val == Some(vhash_by.into()) { - tx.remove(&self.data.merkle_todo, k)?; - tx.commit(true) - } else { - tx.commit(false) + match old_val { + Some(ov) if ov == vhash_by => { + tx.remove(&self.data.merkle_todo, k)?; + tx.commit(true) + } + _ => tx.commit(false), } })?;