From fd8d5c37f73d93a0cdfd8ab08f7447b0f6d3ce3e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 14:59:26 +0200 Subject: [PATCH 01/64] First iteration of a generic DB layer --- Cargo.lock | 29 +++++++ Cargo.toml | 1 + src/db/Cargo.toml | 23 ++++++ src/db/lib.rs | 174 +++++++++++++++++++++++++++++++++++++++++ src/db/sled_adapter.rs | 132 +++++++++++++++++++++++++++++++ src/db/test.rs | 49 ++++++++++++ src/table/Cargo.toml | 1 + 7 files changed, 409 insertions(+) create mode 100644 src/db/Cargo.toml create mode 100644 src/db/lib.rs create mode 100644 src/db/sled_adapter.rs create mode 100644 src/db/test.rs diff --git a/Cargo.lock b/Cargo.lock index 630642ff..1f81ebec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -987,6 +987,16 @@ dependencies = [ "zstd", ] +[[package]] +name = "garage_db" +version = "0.8.0" +dependencies = [ + "arc-swap", + "err-derive 0.3.1", + "mktemp", + "sled", +] + [[package]] name = "garage_model" version = "0.5.1" @@ -1130,6 +1140,7 @@ dependencies = [ "bytes 1.1.0", "futures", "futures-util", + "garage_db", "garage_rpc 0.7.0", "garage_util 0.7.0", "hexdump", @@ -1855,6 +1866,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "mktemp" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "975de676448231fcde04b9149d2543077e166b78fc29eae5aa219e7928410da2" +dependencies = [ + "uuid", +] + [[package]] name = "multer" version = "2.0.2" @@ -3489,6 +3509,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom", +] + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index edd0e3f9..122285db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "src/db", "src/util", "src/rpc", "src/table", diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml new file mode 100644 index 00000000..025016d5 --- /dev/null +++ b/src/db/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "garage_db" +version = "0.8.0" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "Abstraction over multiple key/value storage engines that supports transactions" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +err-derive = "0.3" +arc-swap = "1.0" + +sled = "0.34" + +[dev-dependencies] +mktemp = "0.4" diff --git a/src/db/lib.rs b/src/db/lib.rs new file mode 100644 index 00000000..0f23a9b4 --- /dev/null +++ b/src/db/lib.rs @@ -0,0 +1,174 @@ +pub mod sled_adapter; + +#[cfg(test)] +pub mod test; + +use std::borrow::Cow; +use std::sync::Arc; + +use arc_swap::ArcSwapOption; + +#[derive(Clone)] +pub struct Db(pub(crate) Arc); + +#[derive(Clone)] +pub struct Transaction<'a>(pub(crate) &'a dyn ITx<'a>); + +#[derive(Clone)] +pub struct Tree(pub(crate) Arc, pub(crate) usize); + +pub type Value<'a> = Cow<'a, [u8]>; + +// ---- + +#[derive(Debug)] +pub struct Error(Cow<'static, str>); + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum TxError { + Abort(E), + Db(Error), +} +pub type TxResult = std::result::Result>; + +impl From for TxError { + fn from(e: Error) -> TxError { + TxError::Db(e) + } +} + +// ---- + +impl Db { + pub fn tree>(&self, name: S) -> Result { + let tree_id = self.0.tree(name.as_ref())?; + Ok(Tree(self.0.clone(), tree_id)) + } + + pub fn transaction(&self, fun: F) -> TxResult + where + F: Fn(Transaction<'_>) -> TxResult + Send + Sync, + R: Send + Sync, + E: Send + Sync, + { + let f = TxFn { + function: fun, + result: ArcSwapOption::new(None), + }; + match self.0.transaction(&f) { + Err(TxError::Db(e)) => Err(TxError::Db(e)), + Err(TxError::Abort(())) => { + let r_arc = f + .result + .into_inner() + .expect("Transaction did not store result"); + let r = Arc::try_unwrap(r_arc).ok().expect("Many refs"); + assert!(matches!(r, Err(TxError::Abort(_)))); + r + } + Ok(()) => { + let r_arc = f + .result + .into_inner() + .expect("Transaction did not store result"); + let r = Arc::try_unwrap(r_arc).ok().expect("Many refs"); + assert!(matches!(r, Ok(_))); + r + } + } + } +} + +impl Tree { + pub fn get<'a, T: AsRef<[u8]>>(&'a self, key: T) -> Result>> { + self.0.get(self.1, key.as_ref()) + } + + pub fn put, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { + self.0.put(self.1, key.as_ref(), value.as_ref()) + } +} + +impl<'a> Transaction<'a> { + pub fn get>(&self, tree: &Tree, key: T) -> Result>> { + self.0.get(tree.1, key.as_ref()) + } + + pub fn put, U: AsRef<[u8]>>(&self, tree: &Tree, key: T, value: U) -> Result<()> { + self.0.put(tree.1, key.as_ref(), value.as_ref()) + } + + #[must_use] + pub fn abort(self, e: E) -> TxResult + where + R: Send + Sync, + E: Send + Sync, + { + Err(TxError::Abort(e)) + } + + #[must_use] + pub fn commit(self, r: R) -> TxResult + where + R: Send + Sync, + E: Send + Sync, + { + Ok(r) + } +} + +// ---- Internal interfaces + +pub(crate) trait IDb: Send + Sync { + fn tree(&self, name: &str) -> Result; + + fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>>; + fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + + fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; +} + +pub(crate) trait ITx<'a> { + fn get(&self, tree: usize, key: &[u8]) -> Result>>; + fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; +} + +pub(crate) trait ITxFn: Send + Sync { + fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult; +} + +enum TxFnResult { + Abort, + Ok, + Err, +} + +struct TxFn +where + F: Fn(Transaction<'_>) -> TxResult + Send + Sync, + R: Send + Sync, + E: Send + Sync, +{ + function: F, + result: ArcSwapOption>, +} + +impl ITxFn for TxFn +where + F: Fn(Transaction<'_>) -> TxResult + Send + Sync, + R: Send + Sync, + E: Send + Sync, +{ + fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult { + let res = (self.function)(Transaction(tx)); + let retval = match &res { + Ok(_) => TxFnResult::Ok, + Err(TxError::Abort(_)) => TxFnResult::Abort, + Err(TxError::Db(_)) => TxFnResult::Err, + }; + self.result.store(Some(Arc::new(res))); + retval + } +} diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs new file mode 100644 index 00000000..617b4844 --- /dev/null +++ b/src/db/sled_adapter.rs @@ -0,0 +1,132 @@ +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use arc_swap::ArcSwapOption; + +use sled::transaction::{ + ConflictableTransactionError, TransactionError, Transactional, TransactionalTree, UnabortableTransactionError +}; + +use crate::{Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, Db}; + +impl From for Error { + fn from(e: sled::Error) -> Error { + Error(format!("{}", e).into()) + } +} + +pub struct SledDb { + db: sled::Db, + trees: RwLock<(Vec, HashMap)>, +} + +impl SledDb { + pub fn new(db: sled::Db) -> Db { + let s = Self { + db, + trees: RwLock::new((Vec::new(), HashMap::new())), + }; + Db(Arc::new(s)) + } + + fn get_tree(&self, i: usize) -> Result { + self.trees + .read() + .unwrap() + .0 + .get(i) + .cloned() + .ok_or(Error("invalid tree id".into())) + } +} + +impl IDb for SledDb { + fn tree(&self, name: &str) -> Result { + let mut trees = self.trees.write().unwrap(); + if let Some(i) = trees.1.get(name) { + Ok(*i) + } else { + let tree = self.db.open_tree(name)?; + let i = trees.0.len(); + trees.0.push(tree); + trees.1.insert(name.to_string(), i); + Ok(i) + } + } + + fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>> { + let tree = self.get_tree(tree)?; + Ok(tree.get(key)?.map(|v| v.to_vec().into())) + } + fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.get_tree(tree)?; + tree.insert(key, value)?; + Ok(()) + } + + fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { + let trees = self.trees.read().unwrap(); + let res = trees.0.transaction(|txtrees| { + let tx = SledTx { + trees: txtrees, + err: ArcSwapOption::new(None), + }; + match f.try_on(&tx) { + TxFnResult::Ok => { + assert!(tx.err.into_inner().is_none()); + Ok(()) + } + TxFnResult::Abort => Err(ConflictableTransactionError::Abort(())), + TxFnResult::Err => { + let err_arc = tx + .err + .into_inner() + .expect("Transaction did not store error"); + let err = Arc::try_unwrap(err_arc).ok().expect("Many refs"); + Err(err.into()) + } + } + }); + match res { + Ok(()) => Ok(()), + Err(TransactionError::Abort(())) => Err(TxError::Abort(())), + Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())), + } + } +} + +// ---- + +struct SledTx<'a> { + trees: &'a [TransactionalTree], + err: ArcSwapOption, +} + +impl<'a> SledTx<'a> { + fn save_error(&self, v: std::result::Result) -> Result { + match v { + Ok(x) => Ok(x), + Err(e) => { + let txt = format!("{}", e); + self.err.store(Some(Arc::new(e))); + Err(Error(txt.into())) + } + } + } +} + +impl<'a> ITx<'a> for SledTx<'a> { + fn get(&self, tree: usize, key: &[u8]) -> Result>> { + let tree = self.trees.get(tree) + .ok_or(Error("invalid tree id".into()))?; + let tmp = self.save_error(tree.get(key))?; + Ok(tmp.map(|v| v.to_vec().into())) + } + + fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.trees.get(tree) + .ok_or(Error("invalid tree id".into()))?; + self.save_error(tree.insert(key, value))?; + Ok(()) + } +} diff --git a/src/db/test.rs b/src/db/test.rs new file mode 100644 index 00000000..f0e6c5de --- /dev/null +++ b/src/db/test.rs @@ -0,0 +1,49 @@ +use crate::*; + +use crate::sled_adapter::SledDb; + +fn test_suite(db: Db) -> Result<()> { + let tree = db.tree("tree")?; + + let va: &[u8] = &b"plop"[..]; + let vb: &[u8] = &b"plip"[..]; + let vc: &[u8] = &b"plup"[..]; + + tree.put(b"test", va)?; + assert_eq!(tree.get(b"test")?, Some(va.into())); + + let res = db.transaction::<_, (), _>(|tx| { + assert_eq!(tx.get(&tree, b"test")?, Some(va.into())); + + tx.put(&tree, b"test", vb)?; + + assert_eq!(tx.get(&tree, b"test")?, Some(vb.into())); + + tx.commit(12) + }); + assert!(matches!(res, Ok(12))); + assert_eq!(tree.get(b"test")?, Some(vb.into())); + + let res = db.transaction::<(), _, _>(|tx| { + assert_eq!(tx.get(&tree, b"test")?, Some(vb.into())); + + tx.put(&tree, b"test", vc)?; + + assert_eq!(tx.get(&tree, b"test")?, Some(vc.into())); + + tx.abort(42) + }); + assert!(matches!(res, Err(TxError::Abort(42)))); + assert_eq!(tree.get(b"test")?, Some(vb.into())); + + Ok(()) +} + +#[test] +fn test_sled_db() -> Result<()> { + let path = mktemp::Temp::new_dir().unwrap(); + let db = SledDb::new(sled::open(path.to_path_buf()).unwrap()); + test_suite(db)?; + drop(path); + Ok(()) +} diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index ed1a213f..6ae50366 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -14,6 +14,7 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +garage_db = { version = "0.8.0", path = "../db" } garage_rpc = { version = "0.7.0", path = "../rpc" } garage_util = { version = "0.7.0", path = "../util" } -- 2.43.0 From 04901093e7315558bdc147d27adc3f56ec2c98a1 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 15:25:24 +0200 Subject: [PATCH 02/64] Implement iter() and range() on db --- src/db/lib.rs | 15 +++++++++++++ src/db/sled_adapter.rs | 51 +++++++++++++++++++++++++++++++++++++----- src/db/test.rs | 47 +++++++++++++++++++++++++++++--------- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/src/db/lib.rs b/src/db/lib.rs index 0f23a9b4..4e400a1d 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -18,6 +18,7 @@ pub struct Transaction<'a>(pub(crate) &'a dyn ITx<'a>); pub struct Tree(pub(crate) Arc, pub(crate) usize); pub type Value<'a> = Cow<'a, [u8]>; +pub type ValueIter<'a> = Box, Value<'a>)>> + 'a>; // ---- @@ -89,6 +90,14 @@ impl Tree { pub fn put, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { self.0.put(self.1, key.as_ref(), value.as_ref()) } + + pub fn iter<'a>(&'a self, reverse: bool) -> Result> { + self.0.range(self.1, None, reverse) + } + + pub fn range<'a, T: AsRef<[u8]>>(&'a self, start: T, reverse: bool) -> Result> { + self.0.range(self.1, Some(start.as_ref()), reverse) + } } impl<'a> Transaction<'a> { @@ -126,6 +135,12 @@ pub(crate) trait IDb: Send + Sync { fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>>; fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + fn range<'a>( + &'a self, + tree: usize, + start: Option<&[u8]>, + reverse: bool, + ) -> Result>; fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; } diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 617b4844..6e375f03 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -4,10 +4,11 @@ use std::sync::{Arc, RwLock}; use arc_swap::ArcSwapOption; use sled::transaction::{ - ConflictableTransactionError, TransactionError, Transactional, TransactionalTree, UnabortableTransactionError + ConflictableTransactionError, TransactionError, Transactional, TransactionalTree, + UnabortableTransactionError, }; -use crate::{Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, Db}; +use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; impl From for Error { fn from(e: sled::Error) -> Error { @@ -58,12 +59,45 @@ impl IDb for SledDb { let tree = self.get_tree(tree)?; Ok(tree.get(key)?.map(|v| v.to_vec().into())) } + fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; tree.insert(key, value)?; Ok(()) } + fn range<'a>( + &'a self, + tree: usize, + start: Option<&[u8]>, + reverse: bool, + ) -> Result> { + let tree = self.get_tree(tree)?; + if reverse { + match start { + Some(start) => Ok(Box::new(tree.range(..=start).rev().map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + }))), + None => Ok(Box::new(tree.iter().rev().map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + }))), + } + } else { + match start { + Some(start) => Ok(Box::new(tree.range(start..).map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + }))), + None => Ok(Box::new(tree.iter().map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + }))), + } + } + } + fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { let trees = self.trees.read().unwrap(); let res = trees.0.transaction(|txtrees| { @@ -76,7 +110,10 @@ impl IDb for SledDb { assert!(tx.err.into_inner().is_none()); Ok(()) } - TxFnResult::Abort => Err(ConflictableTransactionError::Abort(())), + TxFnResult::Abort => { + assert!(tx.err.into_inner().is_none()); + Err(ConflictableTransactionError::Abort(())) + } TxFnResult::Err => { let err_arc = tx .err @@ -117,14 +154,18 @@ impl<'a> SledTx<'a> { impl<'a> ITx<'a> for SledTx<'a> { fn get(&self, tree: usize, key: &[u8]) -> Result>> { - let tree = self.trees.get(tree) + let tree = self + .trees + .get(tree) .ok_or(Error("invalid tree id".into()))?; let tmp = self.save_error(tree.get(key))?; Ok(tmp.map(|v| v.to_vec().into())) } fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { - let tree = self.trees.get(tree) + let tree = self + .trees + .get(tree) .ok_or(Error("invalid tree id".into()))?; self.save_error(tree.insert(key, value))?; Ok(()) diff --git a/src/db/test.rs b/src/db/test.rs index f0e6c5de..7e389271 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -5,36 +5,63 @@ use crate::sled_adapter::SledDb; fn test_suite(db: Db) -> Result<()> { let tree = db.tree("tree")?; + let ka: &[u8] = &b"test"[..]; + let kb: &[u8] = &b"zwello"[..]; let va: &[u8] = &b"plop"[..]; let vb: &[u8] = &b"plip"[..]; let vc: &[u8] = &b"plup"[..]; - tree.put(b"test", va)?; - assert_eq!(tree.get(b"test")?, Some(va.into())); + tree.put(ka, va)?; + assert_eq!(tree.get(ka)?, Some(va.into())); let res = db.transaction::<_, (), _>(|tx| { - assert_eq!(tx.get(&tree, b"test")?, Some(va.into())); + assert_eq!(tx.get(&tree, ka)?, Some(va.into())); - tx.put(&tree, b"test", vb)?; + tx.put(&tree, ka, vb)?; - assert_eq!(tx.get(&tree, b"test")?, Some(vb.into())); + assert_eq!(tx.get(&tree, ka)?, Some(vb.into())); tx.commit(12) }); assert!(matches!(res, Ok(12))); - assert_eq!(tree.get(b"test")?, Some(vb.into())); + assert_eq!(tree.get(ka)?, Some(vb.into())); let res = db.transaction::<(), _, _>(|tx| { - assert_eq!(tx.get(&tree, b"test")?, Some(vb.into())); + assert_eq!(tx.get(&tree, ka)?, Some(vb.into())); - tx.put(&tree, b"test", vc)?; + tx.put(&tree, ka, vc)?; - assert_eq!(tx.get(&tree, b"test")?, Some(vc.into())); + assert_eq!(tx.get(&tree, ka)?, Some(vc.into())); tx.abort(42) }); assert!(matches!(res, Err(TxError::Abort(42)))); - assert_eq!(tree.get(b"test")?, Some(vb.into())); + assert_eq!(tree.get(ka)?, Some(vb.into())); + + let mut iter = tree.iter(false)?; + assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + assert!(iter.next().is_none()); + + tree.put(kb, vc)?; + assert_eq!(tree.get(kb)?, Some(vc.into())); + + let mut iter = tree.iter(false)?; + assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); + assert!(iter.next().is_none()); + + let mut iter = tree.range("tz", false)?; + assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); + assert!(iter.next().is_none()); + + let mut iter = tree.range("tz", true)?; + assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + assert!(iter.next().is_none()); + + let mut iter = tree.iter(true)?; + assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); + assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + assert!(iter.next().is_none()); Ok(()) } -- 2.43.0 From 9f0f5b2e372a720a807914747fd48ddc93928e04 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 16:58:00 +0200 Subject: [PATCH 03/64] Adapt Garage to use new DB abstraction --- Cargo.lock | 8 +-- src/block/Cargo.toml | 3 +- src/block/manager.rs | 32 +++++------ src/block/metrics.rs | 8 +-- src/block/rc.rs | 51 ++++++++++++----- src/db/lib.rs | 114 +++++++++++++++++++++++++++++-------- src/db/sled_adapter.rs | 89 +++++++++++++++++++---------- src/db/test.rs | 21 +++---- src/garage/Cargo.toml | 1 + src/garage/admin.rs | 2 +- src/garage/repair.rs | 22 +++++-- src/garage/server.rs | 1 + src/model/Cargo.toml | 3 +- src/model/garage.rs | 8 ++- src/model/index_counter.rs | 16 +++--- src/model/migrate.rs | 2 +- src/table/Cargo.toml | 2 - src/table/data.rs | 63 ++++++++++---------- src/table/gc.rs | 30 +++++----- src/table/merkle.rs | 54 ++++++++++-------- src/table/metrics.rs | 12 ++-- src/table/sync.rs | 5 +- src/table/table.rs | 4 +- src/util/Cargo.toml | 4 +- src/util/error.rs | 12 ++-- src/util/lib.rs | 2 +- 26 files changed, 355 insertions(+), 214 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f81ebec..e7b41248 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -889,6 +889,7 @@ dependencies = [ "futures", "futures-util", "garage_api", + "garage_db", "garage_model 0.7.0", "garage_rpc 0.7.0", "garage_table 0.7.0", @@ -972,6 +973,7 @@ dependencies = [ "bytes 1.1.0", "futures", "futures-util", + "garage_db", "garage_rpc 0.7.0", "garage_table 0.7.0", "garage_util 0.7.0", @@ -981,7 +983,6 @@ dependencies = [ "rmp-serde 0.15.5", "serde", "serde_bytes", - "sled", "tokio", "tracing", "zstd", @@ -1034,6 +1035,7 @@ dependencies = [ "futures", "futures-util", "garage_block", + "garage_db", "garage_model 0.5.1", "garage_rpc 0.7.0", "garage_table 0.7.0", @@ -1045,7 +1047,6 @@ dependencies = [ "rmp-serde 0.15.5", "serde", "serde_bytes", - "sled", "tokio", "tracing", "zstd", @@ -1149,7 +1150,6 @@ dependencies = [ "rmp-serde 0.15.5", "serde", "serde_bytes", - "sled", "tokio", "tracing", ] @@ -1188,6 +1188,7 @@ dependencies = [ "chrono", "err-derive 0.3.1", "futures", + "garage_db", "hex", "http", "hyper", @@ -1198,7 +1199,6 @@ dependencies = [ "serde", "serde_json", "sha2", - "sled", "tokio", "toml", "tracing", diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index 9cba69ee..80346aca 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -14,6 +14,7 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +garage_db = { version = "0.8.0", path = "../db" } garage_rpc = { version = "0.7.0", path = "../rpc" } garage_util = { version = "0.7.0", path = "../util" } garage_table = { version = "0.7.0", path = "../table" } @@ -27,8 +28,6 @@ tracing = "0.1.30" rand = "0.8" zstd = { version = "0.9", default-features = false } -sled = "0.34" - rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/block/manager.rs b/src/block/manager.rs index 9b2d9cad..50039d2b 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -17,10 +17,11 @@ use opentelemetry::{ Context, KeyValue, }; +use garage_db as db; + use garage_util::data::*; use garage_util::error::*; use garage_util::metrics::RecordDuration; -use garage_util::sled_counter::SledCountedTree; use garage_util::time::*; use garage_util::tranquilizer::Tranquilizer; @@ -91,9 +92,9 @@ pub struct BlockManager { rc: BlockRc, - resync_queue: SledCountedTree, + resync_queue: db::Tree, resync_notify: Notify, - resync_errors: SledCountedTree, + resync_errors: db::Tree, system: Arc, endpoint: Arc>, @@ -108,7 +109,7 @@ struct BlockManagerLocked(); impl BlockManager { pub fn new( - db: &sled::Db, + db: &db::Db, data_dir: PathBuf, compression_level: Option, background_tranquility: u32, @@ -123,12 +124,10 @@ impl BlockManager { let resync_queue = db .open_tree("block_local_resync_queue") .expect("Unable to open block_local_resync_queue tree"); - let resync_queue = SledCountedTree::new(resync_queue); let resync_errors = db .open_tree("block_local_resync_errors") .expect("Unable to open block_local_resync_errors tree"); - let resync_errors = SledCountedTree::new(resync_errors); let endpoint = system .netapp @@ -219,7 +218,7 @@ impl BlockManager { /// to fix any mismatch between the two. pub async fn repair_data_store(&self, must_exit: &watch::Receiver) -> Result<(), Error> { // 1. Repair blocks from RC table. - for (i, entry) in self.rc.rc.iter().enumerate() { + for (i, entry) in self.rc.rc.iter()?.enumerate() { let (hash, _) = entry?; let hash = Hash::try_from(&hash[..]).unwrap(); self.put_to_resync(&hash, Duration::from_secs(0))?; @@ -265,17 +264,17 @@ impl BlockManager { /// Get lenght of resync queue pub fn resync_queue_len(&self) -> usize { - self.resync_queue.len() + self.resync_queue.len().unwrap() // TODO fix unwrap } /// Get number of blocks that have an error pub fn resync_errors_len(&self) -> usize { - self.resync_errors.len() + self.resync_errors.len().unwrap() // TODO fix unwrap } /// Get number of items in the refcount table pub fn rc_len(&self) -> usize { - self.rc.rc.len() + self.rc.rc.len().unwrap() // TODO fix unwrap } //// ----- Managing the reference counter ---- @@ -503,12 +502,12 @@ impl BlockManager { }); } - fn put_to_resync(&self, hash: &Hash, delay: Duration) -> Result<(), sled::Error> { + fn put_to_resync(&self, hash: &Hash, delay: Duration) -> Result<(), db::Error> { let when = now_msec() + delay.as_millis() as u64; self.put_to_resync_at(hash, when) } - fn put_to_resync_at(&self, hash: &Hash, when: u64) -> Result<(), sled::Error> { + fn put_to_resync_at(&self, hash: &Hash, when: u64) -> Result<(), db::Error> { trace!("Put resync_queue: {} {:?}", when, hash); let mut key = u64::to_be_bytes(when).to_vec(); key.extend(hash.as_ref()); @@ -547,11 +546,8 @@ impl BlockManager { // - Ok(true) -> a block was processed (successfully or not) // - Ok(false) -> no block was processed, but we are ready for the next iteration // - Err(_) -> a Sled error occurred when reading/writing from resync_queue/resync_errors - async fn resync_iter( - &self, - must_exit: &mut watch::Receiver, - ) -> Result { - if let Some(first_pair_res) = self.resync_queue.iter().next() { + async fn resync_iter(&self, must_exit: &mut watch::Receiver) -> Result { + if let Some(first_pair_res) = self.resync_queue.iter()?.next() { let (time_bytes, hash_bytes) = first_pair_res?; let time_msec = u64::from_be_bytes(time_bytes[0..8].try_into().unwrap()); @@ -966,7 +962,7 @@ impl ErrorCounter { } } - fn decode(data: sled::IVec) -> Self { + fn decode<'a>(data: db::Value<'a>) -> Self { Self { errors: u64::from_be_bytes(data[0..8].try_into().unwrap()), last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()), diff --git a/src/block/metrics.rs b/src/block/metrics.rs index f0f541a3..1d4d0028 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -1,6 +1,6 @@ use opentelemetry::{global, metrics::*}; -use garage_util::sled_counter::SledCountedTree; +use garage_db as db; /// TableMetrics reference all counter used for metrics pub struct BlockManagerMetrics { @@ -23,12 +23,12 @@ pub struct BlockManagerMetrics { } impl BlockManagerMetrics { - pub fn new(resync_queue: SledCountedTree, resync_errors: SledCountedTree) -> Self { + pub fn new(resync_queue: db::Tree, resync_errors: db::Tree) -> Self { let meter = global::meter("garage_model/block"); Self { _resync_queue_len: meter .u64_value_observer("block.resync_queue_length", move |observer| { - observer.observe(resync_queue.len() as u64, &[]) + observer.observe(resync_queue.len().unwrap() as u64, &[]) // TODO fix unwrap }) .with_description( "Number of block hashes queued for local check and possible resync", @@ -36,7 +36,7 @@ impl BlockManagerMetrics { .init(), _resync_errored_blocks: meter .u64_value_observer("block.resync_errored_blocks", move |observer| { - observer.observe(resync_errors.len() as u64, &[]) + observer.observe(resync_errors.len().unwrap() as u64, &[]) // TODO fix unwrap }) .with_description("Number of block hashes whose last resync resulted in an error") .init(), diff --git a/src/block/rc.rs b/src/block/rc.rs index ec3ea44e..f6d8c2aa 100644 --- a/src/block/rc.rs +++ b/src/block/rc.rs @@ -1,5 +1,7 @@ use std::convert::TryInto; +use garage_db as db; + use garage_util::data::*; use garage_util::error::*; use garage_util::time::*; @@ -7,31 +9,47 @@ use garage_util::time::*; use crate::manager::BLOCK_GC_DELAY; pub struct BlockRc { - pub(crate) rc: sled::Tree, + pub(crate) rc: db::Tree, } impl BlockRc { - pub(crate) fn new(rc: sled::Tree) -> Self { + pub(crate) fn new(rc: db::Tree) -> Self { Self { rc } } /// Increment the reference counter associated to a hash. /// Returns true if the RC goes from zero to nonzero. pub(crate) fn block_incref(&self, hash: &Hash) -> Result { - let old_rc = self - .rc - .fetch_and_update(&hash, |old| RcEntry::parse_opt(old).increment().serialize())?; - let old_rc = RcEntry::parse_opt(old_rc); + let old_rc = self.rc.db().transaction(|tx| { + let old_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?); + match old_rc.increment().serialize() { + Some(x) => { + tx.insert(&self.rc, &hash, x)?; + } + None => { + tx.remove(&self.rc, &hash)?; + } + }; + tx.commit(old_rc) + })?; Ok(old_rc.is_zero()) } /// Decrement the reference counter associated to a hash. /// Returns true if the RC is now zero. pub(crate) fn block_decref(&self, hash: &Hash) -> Result { - let new_rc = self - .rc - .update_and_fetch(&hash, |old| RcEntry::parse_opt(old).decrement().serialize())?; - let new_rc = RcEntry::parse_opt(new_rc); + let new_rc = self.rc.db().transaction(|tx| { + let new_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?).decrement(); + match new_rc.serialize() { + Some(x) => { + tx.insert(&self.rc, &hash, x)?; + } + None => { + tx.remove(&self.rc, &hash)?; + } + }; + tx.commit(new_rc) + })?; Ok(matches!(new_rc, RcEntry::Deletable { .. })) } @@ -44,12 +62,15 @@ impl BlockRc { /// deletion time has passed pub(crate) fn clear_deleted_block_rc(&self, hash: &Hash) -> Result<(), Error> { let now = now_msec(); - self.rc.update_and_fetch(&hash, |rcval| { - let updated = match RcEntry::parse_opt(rcval) { - RcEntry::Deletable { at_time } if now > at_time => RcEntry::Absent, - v => v, + self.rc.db().transaction(|tx| { + let rcval = RcEntry::parse_opt(tx.get(&self.rc, &hash)?); + match rcval { + RcEntry::Deletable { at_time } if now > at_time => { + tx.remove(&self.rc, &hash)?; + } + _ => (), }; - updated.serialize() + tx.commit(()) })?; Ok(()) } diff --git a/src/db/lib.rs b/src/db/lib.rs index 4e400a1d..75c6ffa2 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -3,26 +3,31 @@ pub mod sled_adapter; #[cfg(test)] pub mod test; +use core::ops::{Bound, RangeBounds}; + use std::borrow::Cow; use std::sync::Arc; use arc_swap::ArcSwapOption; +use err_derive::Error; #[derive(Clone)] pub struct Db(pub(crate) Arc); -#[derive(Clone)] +#[derive(Clone, Copy)] pub struct Transaction<'a>(pub(crate) &'a dyn ITx<'a>); #[derive(Clone)] pub struct Tree(pub(crate) Arc, pub(crate) usize); pub type Value<'a> = Cow<'a, [u8]>; -pub type ValueIter<'a> = Box, Value<'a>)>> + 'a>; +pub type ValueIter<'a> = + Box, Value<'a>)>> + Send + Sync + 'a>; // ---- -#[derive(Debug)] +#[derive(Debug, Error)] +#[error(display = "{}", _0)] pub struct Error(Cow<'static, str>); pub type Result = std::result::Result; @@ -43,14 +48,14 @@ impl From for TxError { // ---- impl Db { - pub fn tree>(&self, name: S) -> Result { - let tree_id = self.0.tree(name.as_ref())?; + pub fn open_tree>(&self, name: S) -> Result { + let tree_id = self.0.open_tree(name.as_ref())?; Ok(Tree(self.0.clone(), tree_id)) } pub fn transaction(&self, fun: F) -> TxResult where - F: Fn(Transaction<'_>) -> TxResult + Send + Sync, + F: Fn(Transaction<'_>) -> TxResult, R: Send + Sync, E: Send + Sync, { @@ -83,20 +88,50 @@ impl Db { } impl Tree { + pub fn db(&self) -> Db { + Db(self.0.clone()) + } + pub fn get<'a, T: AsRef<[u8]>>(&'a self, key: T) -> Result>> { self.0.get(self.1, key.as_ref()) } - pub fn put, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { - self.0.put(self.1, key.as_ref(), value.as_ref()) + pub fn len(&self) -> Result { + self.0.len(self.1) } - pub fn iter<'a>(&'a self, reverse: bool) -> Result> { - self.0.range(self.1, None, reverse) + pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { + self.0.insert(self.1, key.as_ref(), value.as_ref()) } - pub fn range<'a, T: AsRef<[u8]>>(&'a self, start: T, reverse: bool) -> Result> { - self.0.range(self.1, Some(start.as_ref()), reverse) + pub fn remove<'a, T: AsRef<[u8]>>(&'a self, key: T) -> Result { + self.0.remove(self.1, key.as_ref()) + } + + pub fn iter<'a>(&'a self) -> Result> { + self.0.iter(self.1) + } + pub fn iter_rev<'a>(&'a self) -> Result> { + self.0.iter_rev(self.1) + } + + pub fn range<'a, K, R>(&'a self, range: R) -> Result> + where + K: AsRef<[u8]>, + R: RangeBounds, + { + let sb = range.start_bound(); + let eb = range.end_bound(); + self.0.range(self.1, get_bound(sb), get_bound(eb)) + } + pub fn range_rev<'a, K, R>(&'a self, range: R) -> Result> + where + K: AsRef<[u8]>, + R: RangeBounds, + { + let sb = range.start_bound(); + let eb = range.end_bound(); + self.0.range_rev(self.1, get_bound(sb), get_bound(eb)) } } @@ -105,8 +140,17 @@ impl<'a> Transaction<'a> { self.0.get(tree.1, key.as_ref()) } - pub fn put, U: AsRef<[u8]>>(&self, tree: &Tree, key: T, value: U) -> Result<()> { - self.0.put(tree.1, key.as_ref(), value.as_ref()) + pub fn insert, U: AsRef<[u8]>>( + &self, + tree: &Tree, + key: T, + value: U, + ) -> Result<()> { + self.0.insert(tree.1, key.as_ref(), value.as_ref()) + } + + pub fn remove>(&self, tree: &Tree, key: T) -> Result { + self.0.remove(tree.1, key.as_ref()) } #[must_use] @@ -131,15 +175,28 @@ impl<'a> Transaction<'a> { // ---- Internal interfaces pub(crate) trait IDb: Send + Sync { - fn tree(&self, name: &str) -> Result; + fn open_tree(&self, name: &str) -> Result; fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>>; - fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; - fn range<'a>( + fn len(&self, tree: usize) -> Result; + + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + fn remove(&self, tree: usize, key: &[u8]) -> Result; + + fn iter<'a>(&'a self, tree: usize) -> Result>; + fn iter_rev<'a>(&'a self, tree: usize) -> Result>; + + fn range<'a, 'r>( &'a self, tree: usize, - start: Option<&[u8]>, - reverse: bool, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result>; + fn range_rev<'a, 'r>( + &'a self, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> Result>; fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; @@ -147,10 +204,11 @@ pub(crate) trait IDb: Send + Sync { pub(crate) trait ITx<'a> { fn get(&self, tree: usize, key: &[u8]) -> Result>>; - fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + fn remove(&self, tree: usize, key: &[u8]) -> Result; } -pub(crate) trait ITxFn: Send + Sync { +pub(crate) trait ITxFn { fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult; } @@ -162,7 +220,7 @@ enum TxFnResult { struct TxFn where - F: Fn(Transaction<'_>) -> TxResult + Send + Sync, + F: Fn(Transaction<'_>) -> TxResult, R: Send + Sync, E: Send + Sync, { @@ -172,7 +230,7 @@ where impl ITxFn for TxFn where - F: Fn(Transaction<'_>) -> TxResult + Send + Sync, + F: Fn(Transaction<'_>) -> TxResult, R: Send + Sync, E: Send + Sync, { @@ -187,3 +245,13 @@ where retval } } + +// ---- + +fn get_bound>(b: Bound<&K>) -> Bound<&[u8]> { + match b { + Bound::Included(v) => Bound::Included(v.as_ref()), + Bound::Excluded(v) => Bound::Excluded(v.as_ref()), + Bound::Unbounded => Bound::Unbounded, + } +} diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 6e375f03..b1da1c2b 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -1,3 +1,5 @@ +use core::ops::Bound; + use std::collections::HashMap; use std::sync::{Arc, RwLock}; @@ -42,7 +44,7 @@ impl SledDb { } impl IDb for SledDb { - fn tree(&self, name: &str) -> Result { + fn open_tree(&self, name: &str) -> Result { let mut trees = self.trees.write().unwrap(); if let Some(i) = trees.1.get(name) { Ok(*i) @@ -60,42 +62,63 @@ impl IDb for SledDb { Ok(tree.get(key)?.map(|v| v.to_vec().into())) } - fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn remove(&self, tree: usize, key: &[u8]) -> Result { + let tree = self.get_tree(tree)?; + Ok(tree.remove(key)?.is_some()) + } + + fn len(&self, tree: usize) -> Result { + let tree = self.get_tree(tree)?; + Ok(tree.len()) + } + + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; tree.insert(key, value)?; Ok(()) } - fn range<'a>( + fn iter<'a>(&'a self, tree: usize) -> Result> { + let tree = self.get_tree(tree)?; + Ok(Box::new(tree.iter().map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + }))) + } + + fn iter_rev<'a>(&'a self, tree: usize) -> Result> { + let tree = self.get_tree(tree)?; + Ok(Box::new(tree.iter().rev().map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + }))) + } + + fn range<'a, 'r>( &'a self, tree: usize, - start: Option<&[u8]>, - reverse: bool, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, ) -> Result> { let tree = self.get_tree(tree)?; - if reverse { - match start { - Some(start) => Ok(Box::new(tree.range(..=start).rev().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - }))), - None => Ok(Box::new(tree.iter().rev().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - }))), - } - } else { - match start { - Some(start) => Ok(Box::new(tree.range(start..).map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - }))), - None => Ok(Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - }))), - } - } + Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + }))) + } + fn range_rev<'a, 'r>( + &'a self, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result> { + let tree = self.get_tree(tree)?; + Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).rev().map( + |v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + }, + ))) } fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { @@ -162,7 +185,7 @@ impl<'a> ITx<'a> for SledTx<'a> { Ok(tmp.map(|v| v.to_vec().into())) } - fn put(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self .trees .get(tree) @@ -170,4 +193,12 @@ impl<'a> ITx<'a> for SledTx<'a> { self.save_error(tree.insert(key, value))?; Ok(()) } + + fn remove(&self, tree: usize, key: &[u8]) -> Result { + let tree = self + .trees + .get(tree) + .ok_or(Error("invalid tree id".into()))?; + Ok(self.save_error(tree.remove(key))?.is_some()) + } } diff --git a/src/db/test.rs b/src/db/test.rs index 7e389271..69e1d12c 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -3,21 +3,22 @@ use crate::*; use crate::sled_adapter::SledDb; fn test_suite(db: Db) -> Result<()> { - let tree = db.tree("tree")?; + let tree = db.open_tree("tree")?; let ka: &[u8] = &b"test"[..]; let kb: &[u8] = &b"zwello"[..]; + let kint: &[u8] = &b"tz"[..]; let va: &[u8] = &b"plop"[..]; let vb: &[u8] = &b"plip"[..]; let vc: &[u8] = &b"plup"[..]; - tree.put(ka, va)?; + tree.insert(ka, va)?; assert_eq!(tree.get(ka)?, Some(va.into())); let res = db.transaction::<_, (), _>(|tx| { assert_eq!(tx.get(&tree, ka)?, Some(va.into())); - tx.put(&tree, ka, vb)?; + tx.insert(&tree, ka, vb)?; assert_eq!(tx.get(&tree, ka)?, Some(vb.into())); @@ -29,7 +30,7 @@ fn test_suite(db: Db) -> Result<()> { let res = db.transaction::<(), _, _>(|tx| { assert_eq!(tx.get(&tree, ka)?, Some(vb.into())); - tx.put(&tree, ka, vc)?; + tx.insert(&tree, ka, vc)?; assert_eq!(tx.get(&tree, ka)?, Some(vc.into())); @@ -38,27 +39,27 @@ fn test_suite(db: Db) -> Result<()> { assert!(matches!(res, Err(TxError::Abort(42)))); assert_eq!(tree.get(ka)?, Some(vb.into())); - let mut iter = tree.iter(false)?; + let mut iter = tree.iter()?; assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert!(iter.next().is_none()); - tree.put(kb, vc)?; + tree.insert(kb, vc)?; assert_eq!(tree.get(kb)?, Some(vc.into())); - let mut iter = tree.iter(false)?; + let mut iter = tree.iter()?; assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); assert!(iter.next().is_none()); - let mut iter = tree.range("tz", false)?; + let mut iter = tree.range(kint..)?; assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); assert!(iter.next().is_none()); - let mut iter = tree.range("tz", true)?; + let mut iter = tree.range_rev(..kint)?; assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert!(iter.next().is_none()); - let mut iter = tree.iter(true)?; + let mut iter = tree.iter_rev()?; assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert!(iter.next().is_none()); diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 902f67f8..d34a7fa4 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -21,6 +21,7 @@ path = "tests/lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +garage_db = { version = "0.8.0", path = "../db" } garage_api = { version = "0.7.0", path = "../api" } garage_model = { version = "0.7.0", path = "../model" } garage_rpc = { version = "0.7.0", path = "../rpc" } diff --git a/src/garage/admin.rs b/src/garage/admin.rs index bc1f494a..cce88b35 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -727,7 +727,7 @@ impl AdminRpcHandler { { writeln!(to, "\nTable stats for {}", F::TABLE_NAME).unwrap(); if opt.detailed { - writeln!(to, " number of items: {}", t.data.store.len()).unwrap(); + writeln!(to, " number of items: {}", t.data.store.len().unwrap()).unwrap(); // TODO fix len unwrap writeln!( to, " Merkle tree size: {}", diff --git a/src/garage/repair.rs b/src/garage/repair.rs index 830eac71..04d9ee72 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -1,3 +1,4 @@ +use core::ops::Bound; use std::sync::Arc; use tokio::sync::watch; @@ -65,9 +66,15 @@ impl Repair { async fn repair_versions(&self, must_exit: &watch::Receiver) -> Result<(), Error> { let mut pos = vec![]; - while let Some((item_key, item_bytes)) = - self.garage.version_table.data.store.get_gt(&pos)? + while let Some(item) = self + .garage + .version_table + .data + .store + .range((Bound::Excluded(pos), Bound::Unbounded))? + .next() { + let (item_key, item_bytes) = item?; pos = item_key.to_vec(); let version = rmp_serde::decode::from_read_ref::<_, Version>(item_bytes.as_ref())?; @@ -109,9 +116,16 @@ impl Repair { async fn repair_block_ref(&self, must_exit: &watch::Receiver) -> Result<(), Error> { let mut pos = vec![]; - while let Some((item_key, item_bytes)) = - self.garage.block_ref_table.data.store.get_gt(&pos)? + while let Some(item) = self + .garage + .block_ref_table + .data + .store + .range((Bound::Excluded(pos), Bound::Unbounded))? + .next() { + let (item_key, item_bytes) = item?; + pos = item_key.to_vec(); let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(item_bytes.as_ref())?; diff --git a/src/garage/server.rs b/src/garage/server.rs index b58ad286..69f5d60c 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -38,6 +38,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { .flush_every_ms(Some(config.sled_flush_every_ms)) .open() .expect("Unable to open sled DB"); + let db = garage_db::sled_adapter::SledDb::new(db); info!("Initializing background runner..."); let watch_cancel = netapp::util::watch_ctrl_c(); diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 133fe44e..d908dc01 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -14,6 +14,7 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +garage_db = { version = "0.8.0", path = "../db" } garage_rpc = { version = "0.7.0", path = "../rpc" } garage_table = { version = "0.7.0", path = "../table" } garage_block = { version = "0.7.0", path = "../block" } @@ -30,8 +31,6 @@ tracing = "0.1.30" rand = "0.8" zstd = { version = "0.9", default-features = false } -sled = "0.34" - rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/model/garage.rs b/src/model/garage.rs index 2f99bd68..280f3dc7 100644 --- a/src/model/garage.rs +++ b/src/model/garage.rs @@ -2,6 +2,8 @@ use std::sync::Arc; use netapp::NetworkKey; +use garage_db as db; + use garage_util::background::*; use garage_util::config::*; @@ -33,7 +35,7 @@ pub struct Garage { pub config: Config, /// The local database - pub db: sled::Db, + pub db: db::Db, /// A background job runner pub background: Arc, /// The membership manager @@ -71,7 +73,7 @@ pub struct GarageK2V { impl Garage { /// Create and run garage - pub fn new(config: Config, db: sled::Db, background: Arc) -> Arc { + pub fn new(config: Config, db: db::Db, background: Arc) -> Arc { let network_key = NetworkKey::from_slice( &hex::decode(&config.rpc_secret).expect("Invalid RPC secret key")[..], ) @@ -199,7 +201,7 @@ impl Garage { #[cfg(feature = "k2v")] impl GarageK2V { - fn new(system: Arc, db: &sled::Db, meta_rep_param: TableShardedReplication) -> Self { + fn new(system: Arc, db: &db::Db, meta_rep_param: TableShardedReplication) -> Self { info!("Initialize K2V counter table..."); let counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), db); info!("Initialize K2V subscription manager..."); diff --git a/src/model/index_counter.rs b/src/model/index_counter.rs index 123154d4..33de797d 100644 --- a/src/model/index_counter.rs +++ b/src/model/index_counter.rs @@ -6,6 +6,8 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; use tokio::sync::{mpsc, watch}; +use garage_db as db; + use garage_rpc::ring::Ring; use garage_rpc::system::System; use garage_util::data::*; @@ -135,7 +137,7 @@ impl TableSchema for CounterTable { pub struct IndexCounter { this_node: Uuid, - local_counter: sled::Tree, + local_counter: db::Tree, propagate_tx: mpsc::UnboundedSender<(T::P, T::S, LocalCounterEntry)>, pub table: Arc, TableShardedReplication>>, } @@ -144,7 +146,7 @@ impl IndexCounter { pub fn new( system: Arc, replication: TableShardedReplication, - db: &sled::Db, + db: &db::Db, ) -> Arc { let background = system.background.clone(); @@ -177,12 +179,12 @@ impl IndexCounter { pub fn count(&self, pk: &T::P, sk: &T::S, counts: &[(&str, i64)]) -> Result<(), Error> { let tree_key = self.table.data.tree_key(pk, sk); - let new_entry = self.local_counter.transaction(|tx| { - let mut entry = match tx.get(&tree_key[..])? { + let new_entry = self.local_counter.db().transaction(|tx| { + let mut entry = match tx.get(&self.local_counter, &tree_key[..])? { Some(old_bytes) => { rmp_serde::decode::from_read_ref::<_, LocalCounterEntry>(&old_bytes) .map_err(Error::RmpDecode) - .map_err(sled::transaction::ConflictableTransactionError::Abort)? + .map_err(db::TxError::Abort)? } None => LocalCounterEntry { values: BTreeMap::new(), @@ -197,8 +199,8 @@ impl IndexCounter { let new_entry_bytes = rmp_to_vec_all_named(&entry) .map_err(Error::RmpEncode) - .map_err(sled::transaction::ConflictableTransactionError::Abort)?; - tx.insert(&tree_key[..], new_entry_bytes)?; + .map_err(db::TxError::Abort)?; + tx.insert(&self.local_counter, &tree_key[..], new_entry_bytes)?; Ok(entry) })?; diff --git a/src/model/migrate.rs b/src/model/migrate.rs index 7e61957a..1f063265 100644 --- a/src/model/migrate.rs +++ b/src/model/migrate.rs @@ -25,7 +25,7 @@ impl Migrate { .open_tree("bucket:table") .map_err(GarageError::from)?; - for res in tree.iter() { + for res in tree.iter().map_err(GarageError::from)? { let (_k, v) = res.map_err(GarageError::from)?; let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..]) .map_err(GarageError::from)?; diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 6ae50366..6de37cda 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -26,8 +26,6 @@ hexdump = "0.1" tracing = "0.1.30" rand = "0.8" -sled = "0.34" - rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/table/data.rs b/src/table/data.rs index 5cb10066..ebfae551 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -3,12 +3,12 @@ use std::convert::TryInto; use std::sync::Arc; use serde_bytes::ByteBuf; -use sled::{IVec, Transactional}; use tokio::sync::Notify; +use garage_db as db; + use garage_util::data::*; use garage_util::error::*; -use garage_util::sled_counter::SledCountedTree; use garage_rpc::system::System; @@ -25,12 +25,12 @@ pub struct TableData { pub instance: F, pub replication: R, - pub store: sled::Tree, + pub store: db::Tree, - pub(crate) merkle_tree: sled::Tree, - pub(crate) merkle_todo: sled::Tree, + pub(crate) merkle_tree: db::Tree, + pub(crate) merkle_todo: db::Tree, pub(crate) merkle_todo_notify: Notify, - pub(crate) gc_todo: SledCountedTree, + pub(crate) gc_todo: db::Tree, pub(crate) metrics: TableMetrics, } @@ -40,7 +40,7 @@ where F: TableSchema, R: TableReplication, { - pub fn new(system: Arc, instance: F, replication: R, db: &sled::Db) -> Arc { + pub fn new(system: Arc, instance: F, replication: R, db: &db::Db) -> Arc { let store = db .open_tree(&format!("{}:table", F::TABLE_NAME)) .expect("Unable to open DB tree"); @@ -55,7 +55,6 @@ where let gc_todo = db .open_tree(&format!("{}:gc_todo_v2", F::TABLE_NAME)) .expect("Unable to open DB tree"); - let gc_todo = SledCountedTree::new(gc_todo); let metrics = TableMetrics::new(F::TABLE_NAME, merkle_todo.clone(), gc_todo.clone()); @@ -98,30 +97,30 @@ where None => partition_hash.to_vec(), Some(sk) => self.tree_key(partition_key, sk), }; - let range = self.store.range(first_key..); + let range = self.store.range(first_key..)?; self.read_range_aux(partition_hash, range, filter, limit) } EnumerationOrder::Reverse => match start { Some(sk) => { let last_key = self.tree_key(partition_key, sk); - let range = self.store.range(..=last_key).rev(); + let range = self.store.range_rev(..=last_key)?; self.read_range_aux(partition_hash, range, filter, limit) } None => { let mut last_key = partition_hash.to_vec(); let lower = u128::from_be_bytes(last_key[16..32].try_into().unwrap()); last_key[16..32].copy_from_slice(&u128::to_be_bytes(lower + 1)); - let range = self.store.range(..last_key).rev(); + let range = self.store.range_rev(..last_key)?; self.read_range_aux(partition_hash, range, filter, limit) } }, } } - fn read_range_aux( + fn read_range_aux<'a>( &self, partition_hash: Hash, - range: impl Iterator>, + range: db::ValueIter<'a>, filter: &Option, limit: usize, ) -> Result>, Error> { @@ -183,12 +182,10 @@ where tree_key: &[u8], f: impl Fn(Option) -> F::E, ) -> Result, Error> { - let changed = (&self.store, &self.merkle_todo).transaction(|(store, mkl_todo)| { - let (old_entry, old_bytes, new_entry) = match store.get(tree_key)? { + let changed = self.store.db().transaction(|tx| { + let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, tree_key)? { Some(old_bytes) => { - let old_entry = self - .decode_entry(&old_bytes) - .map_err(sled::transaction::ConflictableTransactionError::Abort)?; + let old_entry = self.decode_entry(&old_bytes).map_err(db::TxError::Abort)?; let new_entry = f(Some(old_entry.clone())); (Some(old_entry), Some(old_bytes), new_entry) } @@ -204,13 +201,17 @@ where // the associated Merkle tree entry. let new_bytes = rmp_to_vec_all_named(&new_entry) .map_err(Error::RmpEncode) - .map_err(sled::transaction::ConflictableTransactionError::Abort)?; + .map_err(db::TxError::Abort)?; let encoding_changed = Some(&new_bytes[..]) != old_bytes.as_ref().map(|x| &x[..]); if value_changed || encoding_changed { let new_bytes_hash = blake2sum(&new_bytes[..]); - mkl_todo.insert(tree_key.to_vec(), new_bytes_hash.as_slice())?; - store.insert(tree_key.to_vec(), new_bytes)?; + tx.insert( + &self.merkle_todo, + tree_key.to_vec(), + new_bytes_hash.as_slice(), + )?; + tx.insert(&self.store, tree_key.to_vec(), new_bytes)?; Ok(Some((old_entry, new_entry, new_bytes_hash))) } else { Ok(None) @@ -244,11 +245,11 @@ where } pub(crate) fn delete_if_equal(self: &Arc, k: &[u8], v: &[u8]) -> Result { - let removed = (&self.store, &self.merkle_todo).transaction(|(store, mkl_todo)| { - if let Some(cur_v) = store.get(k)? { + let removed = self.store.db().transaction(|tx| { + if let Some(cur_v) = tx.get(&self.store, k)? { if cur_v == v { - store.remove(k)?; - mkl_todo.insert(k, vec![])?; + tx.remove(&self.store, k)?; + tx.insert(&self.merkle_todo, k, vec![])?; return Ok(true); } } @@ -270,12 +271,12 @@ where k: &[u8], vhash: Hash, ) -> Result { - let removed = (&self.store, &self.merkle_todo).transaction(|(store, mkl_todo)| { - if let Some(cur_v) = store.get(k)? { + let removed = self.store.db().transaction(|tx| { + if let Some(cur_v) = tx.get(&self.store, k)? { if blake2sum(&cur_v[..]) == vhash { - store.remove(k)?; - mkl_todo.insert(k, vec![])?; - return Ok(Some(cur_v)); + tx.remove(&self.store, k)?; + tx.insert(&self.merkle_todo, k, vec![])?; + return Ok(Some(cur_v.into_owned())); } } Ok(None) @@ -316,6 +317,6 @@ where } pub fn gc_todo_len(&self) -> usize { - self.gc_todo.len() + self.gc_todo.len().unwrap() // TODO fix unwrap } } diff --git a/src/table/gc.rs b/src/table/gc.rs index 2a05b6ae..04872a38 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -12,9 +12,10 @@ use futures::select; use futures_util::future::*; use tokio::sync::watch; +use garage_db as db; + use garage_util::data::*; use garage_util::error::*; -use garage_util::sled_counter::SledCountedTree; use garage_util::time::*; use garage_rpc::system::System; @@ -106,7 +107,7 @@ where // List entries in the GC todo list // These entries are put there when a tombstone is inserted in the table // (see update_entry in data.rs) - for entry_kv in self.data.gc_todo.iter() { + for entry_kv in self.data.gc_todo.iter()? { let (k, vhash) = entry_kv?; let mut todo_entry = GcTodoEntry::parse(&k, &vhash); @@ -353,17 +354,17 @@ impl GcTodoEntry { } /// Parses a GcTodoEntry from a (k, v) pair stored in the gc_todo tree - pub(crate) fn parse(sled_k: &[u8], sled_v: &[u8]) -> Self { + pub(crate) fn parse(db_k: &[u8], db_v: &[u8]) -> Self { Self { - tombstone_timestamp: u64::from_be_bytes(sled_k[0..8].try_into().unwrap()), - key: sled_k[8..].to_vec(), - value_hash: Hash::try_from(sled_v).unwrap(), + tombstone_timestamp: u64::from_be_bytes(db_k[0..8].try_into().unwrap()), + key: db_k[8..].to_vec(), + value_hash: Hash::try_from(db_v).unwrap(), value: None, } } /// Saves the GcTodoEntry in the gc_todo tree - pub(crate) fn save(&self, gc_todo_tree: &SledCountedTree) -> Result<(), Error> { + pub(crate) fn save(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { gc_todo_tree.insert(self.todo_table_key(), self.value_hash.as_slice())?; Ok(()) } @@ -373,12 +374,15 @@ impl GcTodoEntry { /// This is usefull to remove a todo entry only under the condition /// that it has not changed since the time it was read, i.e. /// what we have to do is still the same - pub(crate) fn remove_if_equal(&self, gc_todo_tree: &SledCountedTree) -> Result<(), Error> { - let _ = gc_todo_tree.compare_and_swap::<_, _, Vec>( - &self.todo_table_key()[..], - Some(self.value_hash), - None, - )?; + pub(crate) fn remove_if_equal(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { + let key = self.todo_table_key(); + gc_todo_tree.db().transaction(|tx| { + let old_val = tx.get(gc_todo_tree, &key)?; + if old_val == Some(self.value_hash.as_slice().into()) { + tx.remove(gc_todo_tree, &key)?; + } + tx.commit(()) + })?; Ok(()) } diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 93bf7e47..4b0b44ce 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -4,11 +4,10 @@ use std::time::Duration; use futures::select; use futures_util::future::*; use serde::{Deserialize, Serialize}; -use sled::transaction::{ - ConflictableTransactionError, ConflictableTransactionResult, TransactionalTree, -}; use tokio::sync::watch; +use garage_db as db; + use garage_util::background::BackgroundRunner; use garage_util::data::*; use garage_util::error::Error; @@ -90,7 +89,8 @@ where async fn updater_loop(self: Arc, mut must_exit: watch::Receiver) { while !*must_exit.borrow() { - if let Some(x) = self.data.merkle_todo.iter().next() { + if let Some(x) = self.data.merkle_todo.iter().unwrap().next() { + // TODO unwrap to remove match x { Ok((key, valhash)) => { if let Err(e) = self.update_item(&key[..], &valhash[..]) { @@ -137,13 +137,18 @@ where }; self.data .merkle_tree + .db() .transaction(|tx| self.update_item_rec(tx, k, &khash, &key, new_vhash))?; - let deleted = self - .data - .merkle_todo - .compare_and_swap::<_, _, Vec>(k, Some(vhash_by), None)? - .is_ok(); + let deleted = self.data.merkle_todo.db().transaction(|tx| { + let old_val = tx.get(&self.data.merkle_todo, k)?; + if old_val == Some(vhash_by.into()) { + tx.remove(&self.data.merkle_todo, k)?; + tx.commit(true) + } else { + tx.commit(false) + } + })?; if !deleted { debug!( @@ -157,12 +162,12 @@ where fn update_item_rec( &self, - tx: &TransactionalTree, + tx: db::Transaction<'_>, k: &[u8], khash: &Hash, key: &MerkleNodeKey, new_vhash: Option, - ) -> ConflictableTransactionResult, Error> { + ) -> db::TxResult, Error> { let i = key.prefix.len(); // Read node at current position (defined by the prefix stored in key) @@ -203,7 +208,7 @@ where } MerkleNode::Intermediate(_) => Some(MerkleNode::Intermediate(children)), x @ MerkleNode::Leaf(_, _) => { - tx.remove(key_sub.encode())?; + tx.remove(&self.data.merkle_tree, key_sub.encode())?; Some(x) } } @@ -283,28 +288,27 @@ where fn read_node_txn( &self, - tx: &TransactionalTree, + tx: db::Transaction<'_>, k: &MerkleNodeKey, - ) -> ConflictableTransactionResult { - let ent = tx.get(k.encode())?; - MerkleNode::decode_opt(ent).map_err(ConflictableTransactionError::Abort) + ) -> db::TxResult { + let ent = tx.get(&self.data.merkle_tree, k.encode())?; + MerkleNode::decode_opt(ent).map_err(db::TxError::Abort) } fn put_node_txn( &self, - tx: &TransactionalTree, + tx: db::Transaction<'_>, k: &MerkleNodeKey, v: &MerkleNode, - ) -> ConflictableTransactionResult { + ) -> db::TxResult { trace!("Put Merkle node: {:?} => {:?}", k, v); if *v == MerkleNode::Empty { - tx.remove(k.encode())?; + tx.remove(&self.data.merkle_tree, k.encode())?; Ok(self.empty_node_hash) } else { - let vby = rmp_to_vec_all_named(v) - .map_err(|e| ConflictableTransactionError::Abort(e.into()))?; + let vby = rmp_to_vec_all_named(v).map_err(|e| db::TxError::Abort(e.into()))?; let rethash = blake2sum(&vby[..]); - tx.insert(k.encode(), vby)?; + tx.insert(&self.data.merkle_tree, k.encode(), vby)?; Ok(rethash) } } @@ -316,11 +320,11 @@ where } pub fn merkle_tree_len(&self) -> usize { - self.data.merkle_tree.len() + self.data.merkle_tree.len().unwrap() // TODO fix unwrap } pub fn todo_len(&self) -> usize { - self.data.merkle_todo.len() + self.data.merkle_todo.len().unwrap() // TODO fix unwrap } } @@ -347,7 +351,7 @@ impl MerkleNodeKey { } impl MerkleNode { - fn decode_opt(ent: Option) -> Result { + fn decode_opt(ent: Option>) -> Result { match ent { None => Ok(MerkleNode::Empty), Some(v) => Ok(rmp_serde::decode::from_read_ref::<_, MerkleNode>(&v[..])?), diff --git a/src/table/metrics.rs b/src/table/metrics.rs index 752a2a6d..3318de88 100644 --- a/src/table/metrics.rs +++ b/src/table/metrics.rs @@ -1,6 +1,6 @@ use opentelemetry::{global, metrics::*, KeyValue}; -use garage_util::sled_counter::SledCountedTree; +use garage_db as db; /// TableMetrics reference all counter used for metrics pub struct TableMetrics { @@ -19,11 +19,7 @@ pub struct TableMetrics { pub(crate) sync_items_received: Counter, } impl TableMetrics { - pub fn new( - table_name: &'static str, - merkle_todo: sled::Tree, - gc_todo: SledCountedTree, - ) -> Self { + pub fn new(table_name: &'static str, merkle_todo: db::Tree, gc_todo: db::Tree) -> Self { let meter = global::meter(table_name); TableMetrics { _merkle_todo_len: meter @@ -31,7 +27,7 @@ impl TableMetrics { "table.merkle_updater_todo_queue_length", move |observer| { observer.observe( - merkle_todo.len() as u64, + merkle_todo.len().unwrap() as u64, // TODO fix unwrap &[KeyValue::new("table_name", table_name)], ) }, @@ -43,7 +39,7 @@ impl TableMetrics { "table.gc_todo_queue_length", move |observer| { observer.observe( - gc_todo.len() as u64, + gc_todo.len().unwrap() as u64, // TODO fix unwrap &[KeyValue::new("table_name", table_name)], ) }, diff --git a/src/table/sync.rs b/src/table/sync.rs index 08069ad0..87dfd1d8 100644 --- a/src/table/sync.rs +++ b/src/table/sync.rs @@ -258,7 +258,7 @@ where while !*must_exit.borrow() { let mut items = Vec::new(); - for item in self.data.store.range(begin.to_vec()..end.to_vec()) { + for item in self.data.store.range(begin.to_vec()..end.to_vec())? { let (key, value) = item?; items.push((key.to_vec(), Arc::new(ByteBuf::from(value.as_ref())))); @@ -603,7 +603,8 @@ impl SyncTodo { let retain = nodes.contains(&my_id); if !retain { // Check if we have some data to send, otherwise skip - if data.store.range(begin..end).next().is_none() { + if data.store.range(begin..end).unwrap().next().is_none() { + // TODO fix unwrap continue; } } diff --git a/src/table/table.rs b/src/table/table.rs index 2a167604..3c211728 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -13,6 +13,8 @@ use opentelemetry::{ Context, }; +use garage_db as db; + use garage_util::data::*; use garage_util::error::Error; use garage_util::metrics::RecordDuration; @@ -69,7 +71,7 @@ where { // =============== PUBLIC INTERFACE FUNCTIONS (new, insert, get, etc) =============== - pub fn new(instance: F, replication: R, system: Arc, db: &sled::Db) -> Arc { + pub fn new(instance: F, replication: R, system: Arc, db: &db::Db) -> Arc { let endpoint = system .netapp .endpoint(format!("garage_table/table.rs/Rpc:{}", F::TABLE_NAME)); diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 95cde531..5d073436 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -14,6 +14,8 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +garage_db = { version = "0.8.0", path = "../db" } + blake2 = "0.9" err-derive = "0.3" xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] } @@ -22,8 +24,6 @@ tracing = "0.1.30" rand = "0.8" sha2 = "0.9" -sled = "0.34" - chrono = "0.4" rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } diff --git a/src/util/error.rs b/src/util/error.rs index 8734a0c8..9995c746 100644 --- a/src/util/error.rs +++ b/src/util/error.rs @@ -26,8 +26,8 @@ pub enum Error { #[error(display = "Netapp error: {}", _0)] Netapp(#[error(source)] netapp::error::Error), - #[error(display = "Sled error: {}", _0)] - Sled(#[error(source)] sled::Error), + #[error(display = "DB error: {}", _0)] + Db(#[error(source)] garage_db::Error), #[error(display = "Messagepack encode error: {}", _0)] RmpEncode(#[error(source)] rmp_serde::encode::Error), @@ -78,11 +78,11 @@ impl Error { } } -impl From> for Error { - fn from(e: sled::transaction::TransactionError) -> Error { +impl From> for Error { + fn from(e: garage_db::TxError) -> Error { match e { - sled::transaction::TransactionError::Abort(x) => x, - sled::transaction::TransactionError::Storage(x) => Error::Sled(x), + garage_db::TxError::Abort(x) => x, + garage_db::TxError::Db(x) => Error::Db(x), } } } diff --git a/src/util/lib.rs b/src/util/lib.rs index d8ffdd0b..8ca6e310 100644 --- a/src/util/lib.rs +++ b/src/util/lib.rs @@ -11,7 +11,7 @@ pub mod error; pub mod formater; pub mod metrics; pub mod persister; -pub mod sled_counter; +//pub mod sled_counter; pub mod time; pub mod token_bucket; pub mod tranquilizer; -- 2.43.0 From 6805e184e9d856601305045a0cb0499d2608832a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 17:01:04 +0200 Subject: [PATCH 04/64] Do not put sled in garage's cargo.toml --- Cargo.lock | 1 - src/db/sled_adapter.rs | 2 ++ src/garage/Cargo.toml | 2 -- src/garage/server.rs | 6 ++++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e7b41248..f1063c3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -912,7 +912,6 @@ dependencies = [ "serde_bytes", "serde_json", "sha2", - "sled", "static_init", "structopt", "tokio", diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index b1da1c2b..cf69caba 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -12,6 +12,8 @@ use sled::transaction::{ use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; +pub use sled; + impl From for Error { fn from(e: sled::Error) -> Error { Error(format!("{}", e).into()) diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index d34a7fa4..eb643160 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -37,8 +37,6 @@ rand = "0.8" async-trait = "0.1.7" sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } -sled = "0.34" - rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/garage/server.rs b/src/garage/server.rs index 69f5d60c..9d148ee7 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -2,6 +2,8 @@ use std::path::PathBuf; use tokio::sync::watch; +use garage_db as db; + use garage_util::background::*; use garage_util::config::*; use garage_util::error::Error; @@ -32,13 +34,13 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Opening database..."); let mut db_path = config.metadata_dir.clone(); db_path.push("db"); - let db = sled::Config::default() + let db = db::sled_adapter::sled::Config::default() .path(&db_path) .cache_capacity(config.sled_cache_capacity) .flush_every_ms(Some(config.sled_flush_every_ms)) .open() .expect("Unable to open sled DB"); - let db = garage_db::sled_adapter::SledDb::new(db); + let db = db::sled_adapter::SledDb::new(db); info!("Initializing background runner..."); let watch_cancel = netapp::util::watch_ctrl_c(); -- 2.43.0 From fbd5b64ff3c0da95a2ae96b1f830d2ed7dc4c5a8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 17:38:30 +0200 Subject: [PATCH 05/64] Complete sled abstraction --- src/db/lib.rs | 55 ++++++++++++++++++++++-- src/db/sled_adapter.rs | 95 +++++++++++++++++++++++++++++++++++------- 2 files changed, 133 insertions(+), 17 deletions(-) diff --git a/src/db/lib.rs b/src/db/lib.rs index 75c6ffa2..c3e89ba4 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -24,6 +24,9 @@ pub type Value<'a> = Cow<'a, [u8]>; pub type ValueIter<'a> = Box, Value<'a>)>> + Send + Sync + 'a>; +pub type Exporter<'a> = + Box)>> + Send + Sync + 'a>; + // ---- #[derive(Debug, Error)] @@ -95,7 +98,6 @@ impl Tree { pub fn get<'a, T: AsRef<[u8]>>(&'a self, key: T) -> Result>> { self.0.get(self.1, key.as_ref()) } - pub fn len(&self) -> Result { self.0.len(self.1) } @@ -103,7 +105,6 @@ impl Tree { pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { self.0.insert(self.1, key.as_ref(), value.as_ref()) } - pub fn remove<'a, T: AsRef<[u8]>>(&'a self, key: T) -> Result { self.0.remove(self.1, key.as_ref()) } @@ -139,6 +140,9 @@ impl<'a> Transaction<'a> { pub fn get>(&self, tree: &Tree, key: T) -> Result>> { self.0.get(tree.1, key.as_ref()) } + pub fn len(&self, tree: &Tree) -> Result { + self.0.len(tree.1) + } pub fn insert, U: AsRef<[u8]>>( &self, @@ -148,11 +152,38 @@ impl<'a> Transaction<'a> { ) -> Result<()> { self.0.insert(tree.1, key.as_ref(), value.as_ref()) } - pub fn remove>(&self, tree: &Tree, key: T) -> Result { self.0.remove(tree.1, key.as_ref()) } + pub fn iter(&self, tree: &Tree) -> Result> { + self.0.iter(tree.1) + } + pub fn iter_rev(&self, tree: &Tree) -> Result> { + self.0.iter_rev(tree.1) + } + + pub fn range(&self, tree: &Tree, range: R) -> Result> + where + K: AsRef<[u8]>, + R: RangeBounds, + { + let sb = range.start_bound(); + let eb = range.end_bound(); + self.0.range(tree.1, get_bound(sb), get_bound(eb)) + } + pub fn range_rev(&self, tree: &Tree, range: R) -> Result> + where + K: AsRef<[u8]>, + R: RangeBounds, + { + let sb = range.start_bound(); + let eb = range.end_bound(); + self.0.range_rev(tree.1, get_bound(sb), get_bound(eb)) + } + + // ---- + #[must_use] pub fn abort(self, e: E) -> TxResult where @@ -204,8 +235,26 @@ pub(crate) trait IDb: Send + Sync { pub(crate) trait ITx<'a> { fn get(&self, tree: usize, key: &[u8]) -> Result>>; + fn len(&self, tree: usize) -> Result; + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; fn remove(&self, tree: usize, key: &[u8]) -> Result; + + fn iter(&self, tree: usize) -> Result>; + fn iter_rev(&self, tree: usize) -> Result>; + + fn range<'r>( + &self, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result>; + fn range_rev<'r>( + &self, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result>; } pub(crate) trait ITxFn { diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index cf69caba..9ee9ea58 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -10,7 +10,7 @@ use sled::transaction::{ UnabortableTransactionError, }; -use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; +use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, Exporter}; pub use sled; @@ -43,6 +43,50 @@ impl SledDb { .cloned() .ok_or(Error("invalid tree id".into())) } + + pub fn export<'a>(&'a self) -> Result> { + let mut trees = vec![]; + for name in self.db.tree_names() { + let name = std::str::from_utf8(&name).map_err(|e| Error(format!("{}", e).into()))?.to_string(); + let tree = self.open_tree(&name)?; + let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); + trees.push((name, tree)); + } + let trees_exporter: Exporter<'a> = Box::new(trees + .into_iter() + .map(|(name, tree)| { + let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + })); + Ok((name.to_string(), iter)) + })); + Ok(trees_exporter) + } + + pub fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> { + for ex_tree in ex { + let (name, data) = ex_tree?; + + let tree = self.open_tree(&name)?; + let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); + if !tree.is_empty() { + return Err(Error(format!("tree {} already contains data", name).into())); + } + + let mut i = 0; + for item in data { + let (k, v) = item?; + tree.insert(k.as_ref(), v.as_ref())?; + i += 1; + if i % 1000 == 0 { + println!("{}: imported {}", name, i); + } + } + println!("{}: finished importing, {} items", name, i); + } + Ok(()) + } } impl IDb for SledDb { @@ -165,6 +209,12 @@ struct SledTx<'a> { } impl<'a> SledTx<'a> { + fn get_tree(&self, i: usize) -> Result<&TransactionalTree> { + self.trees + .get(i) + .ok_or(Error("invalid tree id (it might have been openned after the transaction started)".into())) + } + fn save_error(&self, v: std::result::Result) -> Result { match v { Ok(x) => Ok(x), @@ -179,28 +229,45 @@ impl<'a> SledTx<'a> { impl<'a> ITx<'a> for SledTx<'a> { fn get(&self, tree: usize, key: &[u8]) -> Result>> { - let tree = self - .trees - .get(tree) - .ok_or(Error("invalid tree id".into()))?; + let tree = self.get_tree(tree)?; let tmp = self.save_error(tree.get(key))?; Ok(tmp.map(|v| v.to_vec().into())) } + fn len(&self, _tree: usize) -> Result { + unimplemented!(".len() in transaction not supported with Sled backend") + } fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { - let tree = self - .trees - .get(tree) - .ok_or(Error("invalid tree id".into()))?; + let tree = self.get_tree(tree)?; self.save_error(tree.insert(key, value))?; Ok(()) } - fn remove(&self, tree: usize, key: &[u8]) -> Result { - let tree = self - .trees - .get(tree) - .ok_or(Error("invalid tree id".into()))?; + let tree = self.get_tree(tree)?; Ok(self.save_error(tree.remove(key))?.is_some()) } + + fn iter(&self, _tree: usize) -> Result> { + unimplemented!("Iterators in transactions not supported with Sled backend"); + } + fn iter_rev(&self, _tree: usize) -> Result> { + unimplemented!("Iterators in transactions not supported with Sled backend"); + } + + fn range<'r>( + &self, + _tree: usize, + _low: Bound<&'r [u8]>, + _high: Bound<&'r [u8]>, + ) -> Result> { + unimplemented!("Iterators in transactions not supported with Sled backend"); + } + fn range_rev<'r>( + &self, + _tree: usize, + _low: Bound<&'r [u8]>, + _high: Bound<&'r [u8]>, + ) -> Result> { + unimplemented!("Iterators in transactions not supported with Sled backend"); + } } -- 2.43.0 From f29b91232fbacc0c552fbbec52f5b2cf20cdcf8d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 18:11:32 +0200 Subject: [PATCH 06/64] Use Cell instead of ArcSwap --- Cargo.lock | 1 - src/db/Cargo.toml | 1 - src/db/lib.rs | 28 +++++++--------------------- src/db/sled_adapter.rs | 40 ++++++++++++++++++++-------------------- 4 files changed, 27 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1063c3b..05f48c37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -991,7 +991,6 @@ dependencies = [ name = "garage_db" version = "0.8.0" dependencies = [ - "arc-swap", "err-derive 0.3.1", "mktemp", "sled", diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 025016d5..ae6cb45f 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -15,7 +15,6 @@ path = "lib.rs" [dependencies] err-derive = "0.3" -arc-swap = "1.0" sled = "0.34" diff --git a/src/db/lib.rs b/src/db/lib.rs index c3e89ba4..2b9b6bd7 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -6,9 +6,9 @@ pub mod test; use core::ops::{Bound, RangeBounds}; use std::borrow::Cow; +use std::cell::Cell; use std::sync::Arc; -use arc_swap::ArcSwapOption; use err_derive::Error; #[derive(Clone)] @@ -25,7 +25,7 @@ pub type ValueIter<'a> = Box, Value<'a>)>> + Send + Sync + 'a>; pub type Exporter<'a> = - Box)>> + Send + Sync + 'a>; + Box)>> + Send + Sync + 'a>; // ---- @@ -59,30 +59,26 @@ impl Db { pub fn transaction(&self, fun: F) -> TxResult where F: Fn(Transaction<'_>) -> TxResult, - R: Send + Sync, - E: Send + Sync, { let f = TxFn { function: fun, - result: ArcSwapOption::new(None), + result: Cell::new(None), }; match self.0.transaction(&f) { Err(TxError::Db(e)) => Err(TxError::Db(e)), Err(TxError::Abort(())) => { - let r_arc = f + let r = f .result .into_inner() .expect("Transaction did not store result"); - let r = Arc::try_unwrap(r_arc).ok().expect("Many refs"); assert!(matches!(r, Err(TxError::Abort(_)))); r } Ok(()) => { - let r_arc = f + let r = f .result .into_inner() .expect("Transaction did not store result"); - let r = Arc::try_unwrap(r_arc).ok().expect("Many refs"); assert!(matches!(r, Ok(_))); r } @@ -186,18 +182,12 @@ impl<'a> Transaction<'a> { #[must_use] pub fn abort(self, e: E) -> TxResult - where - R: Send + Sync, - E: Send + Sync, { Err(TxError::Abort(e)) } #[must_use] pub fn commit(self, r: R) -> TxResult - where - R: Send + Sync, - E: Send + Sync, { Ok(r) } @@ -270,18 +260,14 @@ enum TxFnResult { struct TxFn where F: Fn(Transaction<'_>) -> TxResult, - R: Send + Sync, - E: Send + Sync, { function: F, - result: ArcSwapOption>, + result: Cell>>, } impl ITxFn for TxFn where F: Fn(Transaction<'_>) -> TxResult, - R: Send + Sync, - E: Send + Sync, { fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult { let res = (self.function)(Transaction(tx)); @@ -290,7 +276,7 @@ where Err(TxError::Abort(_)) => TxFnResult::Abort, Err(TxError::Db(_)) => TxFnResult::Err, }; - self.result.store(Some(Arc::new(res))); + self.result.set(Some(res)); retval } } diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 9ee9ea58..3942317c 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -1,16 +1,17 @@ use core::ops::Bound; +use std::cell::Cell; use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use arc_swap::ArcSwapOption; - use sled::transaction::{ ConflictableTransactionError, TransactionError, Transactional, TransactionalTree, UnabortableTransactionError, }; -use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, Exporter}; +use crate::{ + Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, +}; pub use sled; @@ -47,20 +48,20 @@ impl SledDb { pub fn export<'a>(&'a self) -> Result> { let mut trees = vec![]; for name in self.db.tree_names() { - let name = std::str::from_utf8(&name).map_err(|e| Error(format!("{}", e).into()))?.to_string(); + let name = std::str::from_utf8(&name) + .map_err(|e| Error(format!("{}", e).into()))? + .to_string(); let tree = self.open_tree(&name)?; let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); trees.push((name, tree)); } - let trees_exporter: Exporter<'a> = Box::new(trees - .into_iter() - .map(|(name, tree)| { - let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - })); - Ok((name.to_string(), iter)) + let trees_exporter: Exporter<'a> = Box::new(trees.into_iter().map(|(name, tree)| { + let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) })); + Ok((name.to_string(), iter)) + })); Ok(trees_exporter) } @@ -172,7 +173,7 @@ impl IDb for SledDb { let res = trees.0.transaction(|txtrees| { let tx = SledTx { trees: txtrees, - err: ArcSwapOption::new(None), + err: Cell::new(None), }; match f.try_on(&tx) { TxFnResult::Ok => { @@ -184,11 +185,10 @@ impl IDb for SledDb { Err(ConflictableTransactionError::Abort(())) } TxFnResult::Err => { - let err_arc = tx + let err = tx .err .into_inner() .expect("Transaction did not store error"); - let err = Arc::try_unwrap(err_arc).ok().expect("Many refs"); Err(err.into()) } } @@ -205,14 +205,14 @@ impl IDb for SledDb { struct SledTx<'a> { trees: &'a [TransactionalTree], - err: ArcSwapOption, + err: Cell>, } impl<'a> SledTx<'a> { fn get_tree(&self, i: usize) -> Result<&TransactionalTree> { - self.trees - .get(i) - .ok_or(Error("invalid tree id (it might have been openned after the transaction started)".into())) + self.trees.get(i).ok_or(Error( + "invalid tree id (it might have been openned after the transaction started)".into(), + )) } fn save_error(&self, v: std::result::Result) -> Result { @@ -220,7 +220,7 @@ impl<'a> SledTx<'a> { Ok(x) => Ok(x), Err(e) => { let txt = format!("{}", e); - self.err.store(Some(Arc::new(e))); + self.err.set(Some(e)); Err(Error(txt.into())) } } -- 2.43.0 From 43704afb291e955c043686dde6b801d4d1339231 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 19:58:47 +0200 Subject: [PATCH 07/64] Begin sqlite adapter --- Cargo.lock | 104 +++++++++++++++ src/db/Cargo.toml | 2 + src/db/lib.rs | 72 ++++++----- src/db/sled_adapter.rs | 105 ++++++++-------- src/db/sqlite_adapter.rs | 265 +++++++++++++++++++++++++++++++++++++++ src/db/test.rs | 52 ++++---- 6 files changed, 498 insertions(+), 102 deletions(-) create mode 100644 src/db/sqlite_adapter.rs diff --git a/Cargo.lock b/Cargo.lock index 05f48c37..7b8c84e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,23 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -11,6 +28,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "anyhow" version = "1.0.56" @@ -716,6 +739,18 @@ dependencies = [ "synstructure", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "1.7.0" @@ -993,6 +1028,8 @@ version = "0.8.0" dependencies = [ "err-derive 0.3.1", "mktemp", + "ouroboros", + "rusqlite", "sled", ] @@ -1297,6 +1334,18 @@ name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +dependencies = [ + "hashbrown", +] [[package]] name = "heck" @@ -1767,6 +1816,16 @@ dependencies = [ "walkdir", ] +[[package]] +name = "libsqlite3-sys" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -2152,6 +2211,30 @@ version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435" +[[package]] +name = "ouroboros" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f31a3b678685b150cba82b702dcdc5e155893f63610cf388d30cd988d4ca2bf" +dependencies = [ + "aliasable", + "ouroboros_macro", + "stable_deref_trait", +] + +[[package]] +name = "ouroboros_macro" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084fd65d5dd8b3772edccb5ffd1e4b7eba43897ecd0f9401e330e8c542959408" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -2697,6 +2780,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "rusqlite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "memchr", + "smallvec", +] + [[package]] name = "rustc_version" version = "0.4.0" @@ -3006,6 +3104,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_init" version = "1.0.2" diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index ae6cb45f..34d483ec 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -15,8 +15,10 @@ path = "lib.rs" [dependencies] err-derive = "0.3" +ouroboros = "0.15" sled = "0.34" +rusqlite = "0.27" [dev-dependencies] mktemp = "0.4" diff --git a/src/db/lib.rs b/src/db/lib.rs index 2b9b6bd7..6de296af 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -1,4 +1,5 @@ pub mod sled_adapter; +pub mod sqlite_adapter; #[cfg(test)] pub mod test; @@ -22,10 +23,10 @@ pub struct Tree(pub(crate) Arc, pub(crate) usize); pub type Value<'a> = Cow<'a, [u8]>; pub type ValueIter<'a> = - Box, Value<'a>)>> + Send + Sync + 'a>; + Box, Value<'a>)>> + 'a>; pub type Exporter<'a> = - Box)>> + Send + Sync + 'a>; + Box)>> + 'a>; // ---- @@ -64,26 +65,40 @@ impl Db { function: fun, result: Cell::new(None), }; - match self.0.transaction(&f) { - Err(TxError::Db(e)) => Err(TxError::Db(e)), - Err(TxError::Abort(())) => { - let r = f - .result - .into_inner() - .expect("Transaction did not store result"); - assert!(matches!(r, Err(TxError::Abort(_)))); - r - } + let tx_res = self.0.transaction(&f); + let ret = f + .result + .into_inner() + .expect("Transaction did not store result"); + + match tx_res { Ok(()) => { - let r = f - .result - .into_inner() - .expect("Transaction did not store result"); - assert!(matches!(r, Ok(_))); - r + assert!(matches!(ret, Ok(_))); + ret } + Err(TxError::Abort(())) => { + assert!(matches!(ret, Err(TxError::Abort(_)))); + ret + } + Err(TxError::Db(e2)) => match ret { + // Ok was stored -> the error occured when finalizing + // transaction + Ok(_) => Err(TxError::Db(e2)), + // An error was already stored: that's the one we want to + // return + Err(TxError::Db(e)) => Err(TxError::Db(e)), + _ => unreachable!(), + }, } } + + pub fn export<'a>(&'a self) -> Result> { + self.0.export() + } + + pub fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> { + self.0.import(ex) + } } impl Tree { @@ -181,14 +196,12 @@ impl<'a> Transaction<'a> { // ---- #[must_use] - pub fn abort(self, e: E) -> TxResult - { + pub fn abort(self, e: E) -> TxResult { Err(TxError::Abort(e)) } #[must_use] - pub fn commit(self, r: R) -> TxResult - { + pub fn commit(self, r: R) -> TxResult { Ok(r) } } @@ -221,6 +234,9 @@ pub(crate) trait IDb: Send + Sync { ) -> Result>; fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; + + fn export<'a>(&'a self) -> Result>; + fn import<'a>(&self, ex: Exporter<'a>) -> Result<()>; } pub(crate) trait ITx<'a> { @@ -251,10 +267,10 @@ pub(crate) trait ITxFn { fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult; } -enum TxFnResult { - Abort, +pub(crate) enum TxFnResult { Ok, - Err, + Abort, + DbErr, } struct TxFn @@ -271,13 +287,13 @@ where { fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult { let res = (self.function)(Transaction(tx)); - let retval = match &res { + let res2 = match &res { Ok(_) => TxFnResult::Ok, Err(TxError::Abort(_)) => TxFnResult::Abort, - Err(TxError::Db(_)) => TxFnResult::Err, + Err(TxError::Db(_)) => TxFnResult::DbErr, }; self.result.set(Some(res)); - retval + res2 } } diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 3942317c..c296708b 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -44,50 +44,6 @@ impl SledDb { .cloned() .ok_or(Error("invalid tree id".into())) } - - pub fn export<'a>(&'a self) -> Result> { - let mut trees = vec![]; - for name in self.db.tree_names() { - let name = std::str::from_utf8(&name) - .map_err(|e| Error(format!("{}", e).into()))? - .to_string(); - let tree = self.open_tree(&name)?; - let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); - trees.push((name, tree)); - } - let trees_exporter: Exporter<'a> = Box::new(trees.into_iter().map(|(name, tree)| { - let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - })); - Ok((name.to_string(), iter)) - })); - Ok(trees_exporter) - } - - pub fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> { - for ex_tree in ex { - let (name, data) = ex_tree?; - - let tree = self.open_tree(&name)?; - let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); - if !tree.is_empty() { - return Err(Error(format!("tree {} already contains data", name).into())); - } - - let mut i = 0; - for item in data { - let (k, v) = item?; - tree.insert(k.as_ref(), v.as_ref())?; - i += 1; - if i % 1000 == 0 { - println!("{}: imported {}", name, i); - } - } - println!("{}: finished importing, {} items", name, i); - } - Ok(()) - } } impl IDb for SledDb { @@ -104,6 +60,8 @@ impl IDb for SledDb { } } + // ---- + fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; Ok(tree.get(key)?.map(|v| v.to_vec().into())) @@ -168,6 +126,8 @@ impl IDb for SledDb { ))) } + // ---- + fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { let trees = self.trees.read().unwrap(); let res = trees.0.transaction(|txtrees| { @@ -184,12 +144,9 @@ impl IDb for SledDb { assert!(tx.err.into_inner().is_none()); Err(ConflictableTransactionError::Abort(())) } - TxFnResult::Err => { - let err = tx - .err - .into_inner() - .expect("Transaction did not store error"); - Err(err.into()) + TxFnResult::DbErr => { + let e = tx.err.into_inner().expect("No DB error"); + Err(e.into()) } } }); @@ -199,6 +156,54 @@ impl IDb for SledDb { Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())), } } + + // ---- + + fn export<'a>(&'a self) -> Result> { + let mut trees = vec![]; + for name in self.db.tree_names() { + let name = std::str::from_utf8(&name) + .map_err(|e| Error(format!("{}", e).into()))? + .to_string(); + let tree = self.open_tree(&name)?; + let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); + trees.push((name, tree)); + } + let trees_exporter: Exporter<'a> = Box::new(trees.into_iter().map(|(name, tree)| { + let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| { + v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) + .map_err(Into::into) + })); + Ok((name.to_string(), iter)) + })); + Ok(trees_exporter) + } + + fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> { + for ex_tree in ex { + let (name, data) = ex_tree?; + + let tree = self.open_tree(&name)?; + let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); + if !tree.is_empty() { + return Err(Error(format!("tree {} already contains data", name).into())); + } + + let mut i = 0; + for item in data { + let (k, v) = item?; + tree.insert(k.as_ref(), v.as_ref())?; + i += 1; + if i % 1000 == 0 { + println!("{}: imported {}", name, i); + } + } + println!("{}: finished importing, {} items", name, i); + } + Ok(()) + } + + // ---- } // ---- diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs new file mode 100644 index 00000000..320684df --- /dev/null +++ b/src/db/sqlite_adapter.rs @@ -0,0 +1,265 @@ +use core::ops::Bound; + +use std::cell::Cell; +use std::sync::{Arc, Mutex, RwLock, MutexGuard}; + +use ouroboros::self_referencing; + +use rusqlite::{params, Connection, Transaction}; + +use crate::{ + Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, +}; + +pub use rusqlite; + +impl From for Error { + fn from(e: rusqlite::Error) -> Error { + Error(format!("{}", e).into()) + } +} + +impl From for TxError { + fn from(e: rusqlite::Error) -> TxError { + TxError::Db(e.into()) + } +} + +pub struct SqliteDb { + db: Mutex, + trees: RwLock>, +} + +impl SqliteDb { + pub fn new(db: rusqlite::Connection) -> Db { + let s = Self { + db: Mutex::new(db), + trees: RwLock::new(Vec::new()), + }; + Db(Arc::new(s)) + } + + fn get_tree(&self, i: usize) -> Result { + self.trees + .read() + .unwrap() + .get(i) + .cloned() + .ok_or(Error("invalid tree id".into())) + } +} + +impl IDb for SqliteDb { + fn open_tree(&self, name: &str) -> Result { + let mut trees = self.trees.write().unwrap(); + if let Some(i) = trees.iter().position(|x| x == name) { + Ok(i) + } else { + self.db.lock().unwrap().execute( + &format!( + "CREATE TABLE IF NOT EXISTS {} ( + k BLOB PRIMARY KEY, + v BLOB + )", + name + ), + [], + )?; + let i = trees.len(); + trees.push(name.to_string()); + Ok(i) + } + } + + // ---- + + fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>> { + let tree = self.get_tree(tree)?; + let db = self.db.lock().unwrap(); + let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; + let mut res_iter = stmt.query([key])?; + match res_iter.next()? { + None => Ok(None), + Some(v) => Ok(Some(v.get::<_, Vec>(0)?.into())), + } + } + + fn remove(&self, tree: usize, key: &[u8]) -> Result { + let tree = self.get_tree(tree)?; + let db = self.db.lock().unwrap(); + let res = db.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; + Ok(res > 0) + } + + fn len(&self, tree: usize) -> Result { + let tree = self.get_tree(tree)?; + let db = self.db.lock().unwrap(); + let mut stmt = db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; + let mut res_iter = stmt.query([])?; + match res_iter.next()? { + None => Ok(0), + Some(v) => Ok(v.get::<_, usize>(0)?.into()), + } + } + + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.get_tree(tree)?; + let db = self.db.lock().unwrap(); + db.execute( + &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), + params![key, value], + )?; + Ok(()) + } + + fn iter<'a>(&'a self, tree: usize) -> Result> { + let tree = self.get_tree(tree)?; + let db = self.db.lock().unwrap(); + let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); + let mut stmt = db.prepare(&sql)?; + let res = stmt.query([])?; + unimplemented!(); + } + + fn iter_rev<'a>(&'a self, tree: usize) -> Result> { + let tree = self.get_tree(tree)?; + unimplemented!(); + } + + fn range<'a, 'r>( + &'a self, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result> { + let tree = self.get_tree(tree)?; + unimplemented!(); + } + fn range_rev<'a, 'r>( + &'a self, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result> { + let tree = self.get_tree(tree)?; + unimplemented!(); + } + + // ---- + + fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { + let trees = self.trees.read().unwrap(); + let mut db = self.db.lock().unwrap(); + let tx = SqliteTx { + tx: db.transaction()?, + trees: trees.as_ref(), + }; + match f.try_on(&tx) { + TxFnResult::Ok => { + tx.tx.commit()?; + Ok(()) + } + TxFnResult::Abort => { + tx.tx.rollback()?; + Err(TxError::Abort(())) + } + TxFnResult::DbErr => { + tx.tx.rollback()?; + Err(TxError::Db(Error( + "(this message will be discarded)".into(), + ))) + } + } + } + + // ---- + + fn export<'a>(&'a self) -> Result> { + unimplemented!() + } + + fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> { + unimplemented!() + } + + // ---- +} + +// ---- + +struct SqliteTx<'a> { + tx: Transaction<'a>, + trees: &'a [String], +} + +impl<'a> SqliteTx<'a> { + fn get_tree(&self, i: usize) -> Result { + self.trees.get(i).cloned().ok_or(Error( + "invalid tree id (it might have been openned after the transaction started)".into(), + )) + } +} + +impl<'a> ITx<'a> for SqliteTx<'a> { + fn get(&self, tree: usize, key: &[u8]) -> Result>> { + let tree = self.get_tree(tree)?; + let mut stmt = self + .tx + .prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; + let mut res_iter = stmt.query([key])?; + match res_iter.next()? { + None => Ok(None), + Some(v) => Ok(Some(v.get::<_, Vec>(0)?.into())), + } + } + fn len(&self, tree: usize) -> Result { + let tree = self.get_tree(tree)?; + let mut stmt = self.tx.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; + let mut res_iter = stmt.query([])?; + match res_iter.next()? { + None => Ok(0), + Some(v) => Ok(v.get::<_, usize>(0)?.into()), + } + } + + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.get_tree(tree)?; + self.tx.execute( + &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), + params![key, value], + )?; + Ok(()) + } + fn remove(&self, tree: usize, key: &[u8]) -> Result { + let tree = self.get_tree(tree)?; + let res = self + .tx + .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; + Ok(res > 0) + } + + fn iter(&self, _tree: usize) -> Result> { + unimplemented!(); + } + fn iter_rev(&self, _tree: usize) -> Result> { + unimplemented!(); + } + + fn range<'r>( + &self, + _tree: usize, + _low: Bound<&'r [u8]>, + _high: Bound<&'r [u8]>, + ) -> Result> { + unimplemented!(); + } + fn range_rev<'r>( + &self, + _tree: usize, + _low: Bound<&'r [u8]>, + _high: Bound<&'r [u8]>, + ) -> Result> { + unimplemented!(); + } +} + diff --git a/src/db/test.rs b/src/db/test.rs index 69e1d12c..20ebd54b 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -1,9 +1,10 @@ use crate::*; use crate::sled_adapter::SledDb; +use crate::sqlite_adapter::SqliteDb; -fn test_suite(db: Db) -> Result<()> { - let tree = db.open_tree("tree")?; +fn test_suite(db: Db) { + let tree = db.open_tree("tree").unwrap(); let ka: &[u8] = &b"test"[..]; let kb: &[u8] = &b"zwello"[..]; @@ -12,66 +13,69 @@ fn test_suite(db: Db) -> Result<()> { let vb: &[u8] = &b"plip"[..]; let vc: &[u8] = &b"plup"[..]; - tree.insert(ka, va)?; - assert_eq!(tree.get(ka)?, Some(va.into())); + tree.insert(ka, va).unwrap(); + assert_eq!(tree.get(ka).unwrap(), Some(va.into())); let res = db.transaction::<_, (), _>(|tx| { - assert_eq!(tx.get(&tree, ka)?, Some(va.into())); + assert_eq!(tx.get(&tree, ka).unwrap(), Some(va.into())); - tx.insert(&tree, ka, vb)?; + tx.insert(&tree, ka, vb).unwrap(); - assert_eq!(tx.get(&tree, ka)?, Some(vb.into())); + assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into())); tx.commit(12) }); assert!(matches!(res, Ok(12))); - assert_eq!(tree.get(ka)?, Some(vb.into())); + assert_eq!(tree.get(ka).unwrap(), Some(vb.into())); let res = db.transaction::<(), _, _>(|tx| { - assert_eq!(tx.get(&tree, ka)?, Some(vb.into())); + assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into())); - tx.insert(&tree, ka, vc)?; + tx.insert(&tree, ka, vc).unwrap(); - assert_eq!(tx.get(&tree, ka)?, Some(vc.into())); + assert_eq!(tx.get(&tree, ka).unwrap(), Some(vc.into())); tx.abort(42) }); assert!(matches!(res, Err(TxError::Abort(42)))); - assert_eq!(tree.get(ka)?, Some(vb.into())); + assert_eq!(tree.get(ka).unwrap(), Some(vb.into())); - let mut iter = tree.iter()?; + let mut iter = tree.iter().unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert!(iter.next().is_none()); - tree.insert(kb, vc)?; - assert_eq!(tree.get(kb)?, Some(vc.into())); + tree.insert(kb, vc).unwrap(); + assert_eq!(tree.get(kb).unwrap(), Some(vc.into())); - let mut iter = tree.iter()?; + let mut iter = tree.iter().unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); assert!(iter.next().is_none()); - let mut iter = tree.range(kint..)?; + let mut iter = tree.range(kint..).unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); assert!(iter.next().is_none()); - let mut iter = tree.range_rev(..kint)?; + let mut iter = tree.range_rev(..kint).unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert!(iter.next().is_none()); - let mut iter = tree.iter_rev()?; + let mut iter = tree.iter_rev().unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert!(iter.next().is_none()); - - Ok(()) } #[test] -fn test_sled_db() -> Result<()> { +fn test_sled_db() { let path = mktemp::Temp::new_dir().unwrap(); let db = SledDb::new(sled::open(path.to_path_buf()).unwrap()); - test_suite(db)?; + test_suite(db); drop(path); - Ok(()) +} + +#[test] +fn test_sqlite_db() { + let db = SqliteDb::new(rusqlite::Connection::open_in_memory().unwrap()); + test_suite(db); } -- 2.43.0 From 364061453cd0a5a2e4f76f4194e6b5887aae7ed8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 22:05:57 +0200 Subject: [PATCH 08/64] Add back Send --- src/db/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/lib.rs b/src/db/lib.rs index 6de296af..ce1b1c8b 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -23,7 +23,7 @@ pub struct Tree(pub(crate) Arc, pub(crate) usize); pub type Value<'a> = Cow<'a, [u8]>; pub type ValueIter<'a> = - Box, Value<'a>)>> + 'a>; + Box, Value<'a>)>> + Send + 'a>; pub type Exporter<'a> = Box)>> + 'a>; -- 2.43.0 From c439cb11a908ee5405ed0a3a721e9c8c0e299ad2 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 23:14:10 +0200 Subject: [PATCH 09/64] Sqlite iter with unsafe code --- Cargo.lock | 43 ------------------ src/block/manager.rs | 14 ++++-- src/db/Cargo.toml | 1 - src/db/lib.rs | 6 +-- src/db/sqlite_adapter.rs | 94 ++++++++++++++++++++++++++++++++++++---- src/garage/repair.rs | 63 +++++++++++++++++---------- src/model/migrate.rs | 4 ++ src/table/merkle.rs | 46 ++++++++++---------- 8 files changed, 165 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b8c84e8..d5309a79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" - [[package]] name = "ahash" version = "0.7.6" @@ -28,12 +22,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aliasable" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" - [[package]] name = "anyhow" version = "1.0.56" @@ -1028,7 +1016,6 @@ version = "0.8.0" dependencies = [ "err-derive 0.3.1", "mktemp", - "ouroboros", "rusqlite", "sled", ] @@ -2211,30 +2198,6 @@ version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435" -[[package]] -name = "ouroboros" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f31a3b678685b150cba82b702dcdc5e155893f63610cf388d30cd988d4ca2bf" -dependencies = [ - "aliasable", - "ouroboros_macro", - "stable_deref_trait", -] - -[[package]] -name = "ouroboros_macro" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084fd65d5dd8b3772edccb5ffd1e4b7eba43897ecd0f9401e330e8c542959408" -dependencies = [ - "Inflector", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -3104,12 +3067,6 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "static_init" version = "1.0.2" diff --git a/src/block/manager.rs b/src/block/manager.rs index 50039d2b..f34d13d0 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -547,9 +547,7 @@ impl BlockManager { // - Ok(false) -> no block was processed, but we are ready for the next iteration // - Err(_) -> a Sled error occurred when reading/writing from resync_queue/resync_errors async fn resync_iter(&self, must_exit: &mut watch::Receiver) -> Result { - if let Some(first_pair_res) = self.resync_queue.iter()?.next() { - let (time_bytes, hash_bytes) = first_pair_res?; - + if let Some((time_bytes, hash_bytes)) = self.resync_get_next()? { let time_msec = u64::from_be_bytes(time_bytes[0..8].try_into().unwrap()); let now = now_msec(); @@ -642,6 +640,16 @@ impl BlockManager { } } + fn resync_get_next(&self) -> Result, Vec)>, db::Error> { + match self.resync_queue.iter()?.next() { + None => Ok(None), + Some(v) => { + let (time_bytes, hash_bytes) = v?; + Ok(Some((time_bytes.into_owned(), hash_bytes.into_owned()))) + } + } + } + async fn resync_block(&self, hash: &Hash) -> Result<(), Error> { let BlockStatus { exists, needed } = self .mutation_lock diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 34d483ec..f627208b 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -15,7 +15,6 @@ path = "lib.rs" [dependencies] err-derive = "0.3" -ouroboros = "0.15" sled = "0.34" rusqlite = "0.27" diff --git a/src/db/lib.rs b/src/db/lib.rs index ce1b1c8b..3a2e1d13 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -22,11 +22,9 @@ pub struct Transaction<'a>(pub(crate) &'a dyn ITx<'a>); pub struct Tree(pub(crate) Arc, pub(crate) usize); pub type Value<'a> = Cow<'a, [u8]>; -pub type ValueIter<'a> = - Box, Value<'a>)>> + Send + 'a>; +pub type ValueIter<'a> = Box, Value<'a>)>> + 'a>; -pub type Exporter<'a> = - Box)>> + 'a>; +pub type Exporter<'a> = Box)>> + 'a>; // ---- diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 320684df..bed72e6b 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -1,11 +1,12 @@ use core::ops::Bound; use std::cell::Cell; -use std::sync::{Arc, Mutex, RwLock, MutexGuard}; +use std::marker::PhantomPinned; +use std::pin::Pin; +use std::ptr::NonNull; +use std::sync::{Arc, Mutex, MutexGuard, RwLock}; -use ouroboros::self_referencing; - -use rusqlite::{params, Connection, Transaction}; +use rusqlite::{params, Connection, Rows, Statement, Transaction}; use crate::{ Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, @@ -114,16 +115,14 @@ impl IDb for SqliteDb { fn iter<'a>(&'a self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; - let db = self.db.lock().unwrap(); let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); - let mut stmt = db.prepare(&sql)?; - let res = stmt.query([])?; - unimplemented!(); + DbValueIterator::new(self.db.lock().unwrap(), &sql, []) } fn iter_rev<'a>(&'a self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; - unimplemented!(); + let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree); + DbValueIterator::new(self.db.lock().unwrap(), &sql, []) } fn range<'a, 'r>( @@ -263,3 +262,80 @@ impl<'a> ITx<'a> for SqliteTx<'a> { } } +// ---- + +struct DbValueIterator<'a> { + db: Option>, + stmt: Option>, + iter: Option>, + _pin: PhantomPinned, +} + +impl<'a> DbValueIterator<'a> { + fn new( + db: MutexGuard<'a, Connection>, + sql: &str, + args: P, + ) -> Result> { + let res = DbValueIterator { + db: Some(db), + stmt: None, + iter: None, + _pin: PhantomPinned, + }; + let mut boxed = Box::pin(res); + + unsafe { + let db = NonNull::from(&boxed.db); + let stmt = db.as_ref().as_ref().unwrap().prepare(&sql)?; + + let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed); + Pin::get_unchecked_mut(mut_ref).stmt = Some(stmt); + + let mut stmt = NonNull::from(&boxed.stmt); + let iter = stmt.as_mut().as_mut().unwrap().query(args)?; + + let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed); + Pin::get_unchecked_mut(mut_ref).iter = Some(iter); + } + + Ok(Box::new(DbValueIteratorPin(boxed))) + } +} + +struct DbValueIteratorPin<'a>(Pin>>); + +impl<'a> Iterator for DbValueIteratorPin<'a> { + type Item = Result<(Value<'a>, Value<'a>)>; + + fn next(&mut self) -> Option { + let next = unsafe { + let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut self.0); + Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() + }; + let row = match next { + Err(e) => return Some(Err(e.into())), + Ok(None) => { + // finished, drop everything + unsafe { + let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut self.0); + let t = Pin::get_unchecked_mut(mut_ref); + drop(t.iter.take()); + drop(t.stmt.take()); + drop(t.db.take()); + } + return None; + } + Ok(Some(r)) => r, + }; + let k = match row.get::<_, Vec>(0) { + Err(e) => return Some(Err(e.into())), + Ok(x) => x, + }; + let v = match row.get::<_, Vec>(1) { + Err(e) => return Some(Err(e.into())), + Ok(y) => y, + }; + Some(Ok((k.into(), v.into()))) + } +} diff --git a/src/garage/repair.rs b/src/garage/repair.rs index 04d9ee72..762a8300 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -66,18 +66,10 @@ impl Repair { async fn repair_versions(&self, must_exit: &watch::Receiver) -> Result<(), Error> { let mut pos = vec![]; - while let Some(item) = self - .garage - .version_table - .data - .store - .range((Bound::Excluded(pos), Bound::Unbounded))? - .next() - { - let (item_key, item_bytes) = item?; - pos = item_key.to_vec(); + while let Some((item_key, item_bytes)) = self.get_next_version_after(&pos)? { + pos = item_key; - let version = rmp_serde::decode::from_read_ref::<_, Version>(item_bytes.as_ref())?; + let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?; if version.deleted.get() { continue; } @@ -113,22 +105,30 @@ impl Repair { Ok(()) } + fn get_next_version_after(&self, pos: &[u8]) -> Result, Vec)>, Error> { + match self + .garage + .version_table + .data + .store + .range::<&[u8], _>((Bound::Excluded(pos), Bound::Unbounded))? + .next() + { + None => Ok(None), + Some(item) => { + let (item_key, item_bytes) = item?; + Ok(Some((item_key.into_owned(), item_bytes.into_owned()))) + } + } + } + async fn repair_block_ref(&self, must_exit: &watch::Receiver) -> Result<(), Error> { let mut pos = vec![]; - while let Some(item) = self - .garage - .block_ref_table - .data - .store - .range((Bound::Excluded(pos), Bound::Unbounded))? - .next() - { - let (item_key, item_bytes) = item?; + while let Some((item_key, item_bytes)) = self.get_next_block_ref_after(&pos)? { + pos = item_key; - pos = item_key.to_vec(); - - let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(item_bytes.as_ref())?; + let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?; if block_ref.deleted.get() { continue; } @@ -160,4 +160,21 @@ impl Repair { } Ok(()) } + + fn get_next_block_ref_after(&self, pos: &[u8]) -> Result, Vec)>, Error> { + match self + .garage + .block_ref_table + .data + .store + .range::<&[u8], _>((Bound::Excluded(pos), Bound::Unbounded))? + .next() + { + None => Ok(None), + Some(item) => { + let (item_key, item_bytes) = item?; + Ok(Some((item_key.into_owned(), item_bytes.into_owned()))) + } + } + } } diff --git a/src/model/migrate.rs b/src/model/migrate.rs index 1f063265..25acb4b0 100644 --- a/src/model/migrate.rs +++ b/src/model/migrate.rs @@ -25,11 +25,15 @@ impl Migrate { .open_tree("bucket:table") .map_err(GarageError::from)?; + let mut old_buckets = vec![]; for res in tree.iter().map_err(GarageError::from)? { let (_k, v) = res.map_err(GarageError::from)?; let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..]) .map_err(GarageError::from)?; + old_buckets.push(bucket); + } + for bucket in old_buckets { if let old_bucket::BucketState::Present(p) = bucket.state.get() { self.migrate_buckets050_do_bucket(&bucket, p).await?; } diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 4b0b44ce..6e0c2f7e 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -89,36 +89,36 @@ where async fn updater_loop(self: Arc, mut must_exit: watch::Receiver) { while !*must_exit.borrow() { - if let Some(x) = self.data.merkle_todo.iter().unwrap().next() { - // TODO unwrap to remove - match x { - Ok((key, valhash)) => { - if let Err(e) = self.update_item(&key[..], &valhash[..]) { - warn!( - "({}) Error while updating Merkle tree item: {}", - F::TABLE_NAME, - e - ); - } - } - Err(e) => { - warn!( - "({}) Error while iterating on Merkle todo tree: {}", - F::TABLE_NAME, - e - ); - tokio::time::sleep(Duration::from_secs(10)).await; + match self.updater_loop_iter() { + Ok(true) => (), + Ok(false) => { + select! { + _ = self.data.merkle_todo_notify.notified().fuse() => {}, + _ = must_exit.changed().fuse() => {}, } } - } else { - select! { - _ = self.data.merkle_todo_notify.notified().fuse() => {}, - _ = must_exit.changed().fuse() => {}, + Err(e) => { + warn!( + "({}) Error while updating Merkle tree item: {}", + F::TABLE_NAME, + e + ); + tokio::time::sleep(Duration::from_secs(10)).await; } } } } + fn updater_loop_iter(&self) -> Result { + if let Some(x) = self.data.merkle_todo.iter()?.next() { + let (key, valhash) = x?; + self.update_item(&key[..], &valhash[..])?; + Ok(true) + } else { + Ok(false) + } + } + fn update_item(&self, k: &[u8], vhash_by: &[u8]) -> Result<(), Error> { let khash = blake2sum(k); -- 2.43.0 From 0e4f33688745b2ffe71a80fdf0de44a1163d4f2d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 23:43:32 +0200 Subject: [PATCH 10/64] Horrible implementation of range for sqlite --- src/db/sqlite_adapter.rs | 94 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index bed72e6b..0516f2b7 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -132,7 +132,52 @@ impl IDb for SqliteDb { high: Bound<&'r [u8]>, ) -> Result> { let tree = self.get_tree(tree)?; - unimplemented!(); + + let mut sql = format!("SELECT k, v FROM {}", tree); + let mut params: Vec> = vec![]; + + match low { + Bound::Included(b) => { + sql.push_str(" WHERE k >= ?1"); + params.push(b.to_vec()); + } + Bound::Excluded(b) => { + sql.push_str(" WHERE k > ?1"); + params.push(b.to_vec()); + } + Bound::Unbounded => (), + }; + + match high { + Bound::Included(b) => { + if !params.is_empty() { + sql.push_str(" AND k <= ?2"); + } else { + sql.push_str(" WHERE k <= ?1"); + } + params.push(b.to_vec()); + } + Bound::Excluded(b) => { + if !params.is_empty() { + sql.push_str(" AND k < ?2"); + } else { + sql.push_str(" WHERE k < ?1"); + } + params.push(b.to_vec()); + } + Bound::Unbounded => (), + } + sql.push_str(" ORDER BY k ASC"); + + let params = params + .iter() + .map(|x| x as &dyn rusqlite::ToSql) + .collect::>(); + DbValueIterator::new::<&[&dyn rusqlite::ToSql]>( + self.db.lock().unwrap(), + &sql, + params.as_ref(), + ) } fn range_rev<'a, 'r>( &'a self, @@ -141,7 +186,52 @@ impl IDb for SqliteDb { high: Bound<&'r [u8]>, ) -> Result> { let tree = self.get_tree(tree)?; - unimplemented!(); + + let mut sql = format!("SELECT k, v FROM {}", tree); + let mut params: Vec> = vec![]; + + match low { + Bound::Included(b) => { + sql.push_str(" WHERE k >= ?1"); + params.push(b.to_vec()); + } + Bound::Excluded(b) => { + sql.push_str(" WHERE k > ?1"); + params.push(b.to_vec()); + } + Bound::Unbounded => (), + }; + + match high { + Bound::Included(b) => { + if !params.is_empty() { + sql.push_str(" AND k <= ?2"); + } else { + sql.push_str(" WHERE k <= ?1"); + } + params.push(b.to_vec()); + } + Bound::Excluded(b) => { + if !params.is_empty() { + sql.push_str(" AND k < ?2"); + } else { + sql.push_str(" WHERE k < ?1"); + } + params.push(b.to_vec()); + } + Bound::Unbounded => (), + } + sql.push_str(" ORDER BY k DESC"); + + let params = params + .iter() + .map(|x| x as &dyn rusqlite::ToSql) + .collect::>(); + DbValueIterator::new::<&[&dyn rusqlite::ToSql]>( + self.db.lock().unwrap(), + &sql, + params.as_ref(), + ) } // ---- -- 2.43.0 From 6ebb54cddb3ac21f7384720b8336995a8889d20d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Jun 2022 23:46:28 +0200 Subject: [PATCH 11/64] Just refactor a bit --- src/db/sqlite_adapter.rs | 116 +++++++++++++++------------------------ 1 file changed, 45 insertions(+), 71 deletions(-) diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 0516f2b7..605a61d0 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -1,6 +1,5 @@ use core::ops::Bound; -use std::cell::Cell; use std::marker::PhantomPinned; use std::pin::Pin; use std::ptr::NonNull; @@ -133,41 +132,8 @@ impl IDb for SqliteDb { ) -> Result> { let tree = self.get_tree(tree)?; - let mut sql = format!("SELECT k, v FROM {}", tree); - let mut params: Vec> = vec![]; - - match low { - Bound::Included(b) => { - sql.push_str(" WHERE k >= ?1"); - params.push(b.to_vec()); - } - Bound::Excluded(b) => { - sql.push_str(" WHERE k > ?1"); - params.push(b.to_vec()); - } - Bound::Unbounded => (), - }; - - match high { - Bound::Included(b) => { - if !params.is_empty() { - sql.push_str(" AND k <= ?2"); - } else { - sql.push_str(" WHERE k <= ?1"); - } - params.push(b.to_vec()); - } - Bound::Excluded(b) => { - if !params.is_empty() { - sql.push_str(" AND k < ?2"); - } else { - sql.push_str(" WHERE k < ?1"); - } - params.push(b.to_vec()); - } - Bound::Unbounded => (), - } - sql.push_str(" ORDER BY k ASC"); + let (bounds_sql, params) = bounds_sql(low, high); + let sql = format!("SELECT k, v FROM {} {} ORDER BY k ASC", tree, bounds_sql); let params = params .iter() @@ -187,41 +153,8 @@ impl IDb for SqliteDb { ) -> Result> { let tree = self.get_tree(tree)?; - let mut sql = format!("SELECT k, v FROM {}", tree); - let mut params: Vec> = vec![]; - - match low { - Bound::Included(b) => { - sql.push_str(" WHERE k >= ?1"); - params.push(b.to_vec()); - } - Bound::Excluded(b) => { - sql.push_str(" WHERE k > ?1"); - params.push(b.to_vec()); - } - Bound::Unbounded => (), - }; - - match high { - Bound::Included(b) => { - if !params.is_empty() { - sql.push_str(" AND k <= ?2"); - } else { - sql.push_str(" WHERE k <= ?1"); - } - params.push(b.to_vec()); - } - Bound::Excluded(b) => { - if !params.is_empty() { - sql.push_str(" AND k < ?2"); - } else { - sql.push_str(" WHERE k < ?1"); - } - params.push(b.to_vec()); - } - Bound::Unbounded => (), - } - sql.push_str(" ORDER BY k DESC"); + let (bounds_sql, params) = bounds_sql(low, high); + let sql = format!("SELECT k, v FROM {} {} ORDER BY k DESC", tree, bounds_sql); let params = params .iter() @@ -429,3 +362,44 @@ impl<'a> Iterator for DbValueIteratorPin<'a> { Some(Ok((k.into(), v.into()))) } } + +// ---- + +fn bounds_sql<'r>(low: Bound<&'r [u8]>, high: Bound<&'r [u8]>) -> (String, Vec>) { + let mut sql = String::new(); + let mut params: Vec> = vec![]; + + match low { + Bound::Included(b) => { + sql.push_str(" WHERE k >= ?1"); + params.push(b.to_vec()); + } + Bound::Excluded(b) => { + sql.push_str(" WHERE k > ?1"); + params.push(b.to_vec()); + } + Bound::Unbounded => (), + }; + + match high { + Bound::Included(b) => { + if !params.is_empty() { + sql.push_str(" AND k <= ?2"); + } else { + sql.push_str(" WHERE k <= ?1"); + } + params.push(b.to_vec()); + } + Bound::Excluded(b) => { + if !params.is_empty() { + sql.push_str(" AND k < ?2"); + } else { + sql.push_str(" WHERE k < ?1"); + } + params.push(b.to_vec()); + } + Bound::Unbounded => (), + } + + (sql, params) +} -- 2.43.0 From 7b7990635029776e8878f3b04ed7ca48192899a3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 04:44:40 +0200 Subject: [PATCH 12/64] Drop later --- src/db/sqlite_adapter.rs | 25 +++++++++++-------------- src/db/test.rs | 5 +++++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 605a61d0..5778ecf0 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -288,7 +288,7 @@ impl<'a> ITx<'a> for SqliteTx<'a> { // ---- struct DbValueIterator<'a> { - db: Option>, + db: MutexGuard<'a, Connection>, stmt: Option>, iter: Option>, _pin: PhantomPinned, @@ -301,7 +301,7 @@ impl<'a> DbValueIterator<'a> { args: P, ) -> Result> { let res = DbValueIterator { - db: Some(db), + db: db, stmt: None, iter: None, _pin: PhantomPinned, @@ -310,7 +310,7 @@ impl<'a> DbValueIterator<'a> { unsafe { let db = NonNull::from(&boxed.db); - let stmt = db.as_ref().as_ref().unwrap().prepare(&sql)?; + let stmt = db.as_ref().prepare(&sql)?; let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed); Pin::get_unchecked_mut(mut_ref).stmt = Some(stmt); @@ -326,6 +326,13 @@ impl<'a> DbValueIterator<'a> { } } +impl<'a> Drop for DbValueIterator<'a> { + fn drop(&mut self) { + drop(self.iter.take()); + drop(self.stmt.take()); + } +} + struct DbValueIteratorPin<'a>(Pin>>); impl<'a> Iterator for DbValueIteratorPin<'a> { @@ -338,17 +345,7 @@ impl<'a> Iterator for DbValueIteratorPin<'a> { }; let row = match next { Err(e) => return Some(Err(e.into())), - Ok(None) => { - // finished, drop everything - unsafe { - let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut self.0); - let t = Pin::get_unchecked_mut(mut_ref); - drop(t.iter.take()); - drop(t.stmt.take()); - drop(t.db.take()); - } - return None; - } + Ok(None) => return None, Ok(Some(r)) => r, }; let k = match row.get::<_, Vec>(0) { diff --git a/src/db/test.rs b/src/db/test.rs index 20ebd54b..78d755cf 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -43,6 +43,7 @@ fn test_suite(db: Db) { let mut iter = tree.iter().unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert!(iter.next().is_none()); + drop(iter); tree.insert(kb, vc).unwrap(); assert_eq!(tree.get(kb).unwrap(), Some(vc.into())); @@ -51,19 +52,23 @@ fn test_suite(db: Db) { assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); assert!(iter.next().is_none()); + drop(iter); let mut iter = tree.range(kint..).unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); assert!(iter.next().is_none()); + drop(iter); let mut iter = tree.range_rev(..kint).unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert!(iter.next().is_none()); + drop(iter); let mut iter = tree.iter_rev().unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); assert!(iter.next().is_none()); + drop(iter); } #[test] -- 2.43.0 From f25309e58f839d1fad6b094cd33d7f36b5c3e2e0 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 10:44:54 +0200 Subject: [PATCH 13/64] Change value type to be a dyn thing --- Cargo.lock | 1 + src/block/manager.rs | 2 +- src/db/Cargo.toml | 1 + src/db/lib.rs | 87 +++++++++++++++++++++++++++++++++++++++- src/db/sled_adapter.rs | 55 ++++++++++++++++--------- src/db/sqlite_adapter.rs | 4 ++ src/db/test.rs | 37 ++++++++++------- src/garage/repair.rs | 4 +- src/table/data.rs | 2 +- src/table/gc.rs | 7 +++- src/table/merkle.rs | 11 ++--- 11 files changed, 166 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5309a79..9ff2c02d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1015,6 +1015,7 @@ name = "garage_db" version = "0.8.0" dependencies = [ "err-derive 0.3.1", + "hexdump", "mktemp", "rusqlite", "sled", diff --git a/src/block/manager.rs b/src/block/manager.rs index f34d13d0..de9b08c2 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -645,7 +645,7 @@ impl BlockManager { None => Ok(None), Some(v) => { let (time_bytes, hash_bytes) = v?; - Ok(Some((time_bytes.into_owned(), hash_bytes.into_owned()))) + Ok(Some((time_bytes.into_vec(), hash_bytes.into_vec()))) } } } diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index f627208b..ca189a67 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -15,6 +15,7 @@ path = "lib.rs" [dependencies] err-derive = "0.3" +hexdump = "0.1" sled = "0.34" rusqlite = "0.27" diff --git a/src/db/lib.rs b/src/db/lib.rs index 3a2e1d13..4cda121f 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -21,13 +21,98 @@ pub struct Transaction<'a>(pub(crate) &'a dyn ITx<'a>); #[derive(Clone)] pub struct Tree(pub(crate) Arc, pub(crate) usize); -pub type Value<'a> = Cow<'a, [u8]>; pub type ValueIter<'a> = Box, Value<'a>)>> + 'a>; pub type Exporter<'a> = Box)>> + 'a>; // ---- +pub struct Value<'a>(pub(crate) Box + 'a>); + +pub trait IValue<'a>: AsRef<[u8]> { + fn into_vec(&mut self) -> Vec; +} + +impl<'a> Value<'a> { + #[inline] + pub fn into_vec(mut self) -> Vec { + self.0.into_vec() + } +} + +impl<'a> AsRef<[u8]> for Value<'a> { + #[inline] + fn as_ref(&self) -> &[u8] { + self.0.as_ref().as_ref() + } +} + +impl<'a> std::borrow::Borrow<[u8]> for Value<'a> { + #[inline] + fn borrow(&self) -> &[u8] { + self.0.as_ref().as_ref() + } +} + +impl<'a> core::ops::Deref for Value<'a> { + type Target = [u8]; + #[inline] + fn deref(&self) -> &[u8] { + self.0.as_ref().as_ref() + } +} + +impl<'a, T> PartialEq for Value<'a> +where + T: AsRef<[u8]>, +{ + fn eq(&self, other: &T) -> bool { + self.as_ref() == other.as_ref() + } +} + +impl<'a> std::fmt::Debug for Value<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for line in hexdump::hexdump_iter(self.as_ref()) { + f.write_str(&line)?; + f.write_str("\n")?; + } + Ok(()) + } +} + +impl<'a> IValue<'a> for Vec { + fn into_vec(&mut self) -> Vec { + std::mem::take(self) + } +} + +impl<'a> From> for Vec { + fn from(v: Value<'a>) -> Vec { + v.into_vec() + } +} + +impl<'a> From> for Value<'a> { + fn from(v: Vec) -> Value<'a> { + Value(Box::new(v)) + } +} + +impl<'a> From<&'a [u8]> for Value<'a> { + fn from(v: &'a [u8]) -> Value<'a> { + Value(Box::new(v)) + } +} + +impl<'a> IValue<'a> for &'a [u8] { + fn into_vec(&mut self) -> Vec { + self.to_vec() + } +} + +// ---- + #[derive(Debug, Error)] #[error(display = "{}", _0)] pub struct Error(Cow<'static, str>); diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index c296708b..acee305b 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -10,17 +10,42 @@ use sled::transaction::{ }; use crate::{ - Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, + Db, Error, Exporter, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, + ValueIter, }; pub use sled; +// -- err + impl From for Error { fn from(e: sled::Error) -> Error { Error(format!("{}", e).into()) } } +// -- val + +impl<'a> IValue<'a> for sled::IVec { + fn into_vec(&mut self) -> Vec { + self.to_vec() + } +} + +impl<'a> From> for sled::IVec { + fn from(v: Value<'a>) -> sled::IVec { + sled::IVec::from(v.into_vec()) + } +} + +impl<'a> From for Value<'a> { + fn from(v: sled::IVec) -> Value<'a> { + Value(Box::new(v)) + } +} + +// -- db + pub struct SledDb { db: sled::Db, trees: RwLock<(Vec, HashMap)>, @@ -64,7 +89,7 @@ impl IDb for SledDb { fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; - Ok(tree.get(key)?.map(|v| v.to_vec().into())) + Ok(tree.get(key)?.map(From::from)) } fn remove(&self, tree: usize, key: &[u8]) -> Result { @@ -86,16 +111,14 @@ impl IDb for SledDb { fn iter<'a>(&'a self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) + v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) }))) } fn iter_rev<'a>(&'a self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.iter().rev().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) + v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) }))) } @@ -107,8 +130,7 @@ impl IDb for SledDb { ) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) + v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) }))) } fn range_rev<'a, 'r>( @@ -119,10 +141,7 @@ impl IDb for SledDb { ) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).rev().map( - |v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - }, + |v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into), ))) } @@ -170,10 +189,10 @@ impl IDb for SledDb { trees.push((name, tree)); } let trees_exporter: Exporter<'a> = Box::new(trees.into_iter().map(|(name, tree)| { - let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into())) - .map_err(Into::into) - })); + let iter: ValueIter<'a> = Box::new( + tree.iter() + .map(|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)), + ); Ok((name.to_string(), iter)) })); Ok(trees_exporter) @@ -192,7 +211,7 @@ impl IDb for SledDb { let mut i = 0; for item in data { let (k, v) = item?; - tree.insert(k.as_ref(), v.as_ref())?; + tree.insert(k, v)?; i += 1; if i % 1000 == 0 { println!("{}: imported {}", name, i); @@ -236,7 +255,7 @@ impl<'a> ITx<'a> for SledTx<'a> { fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; let tmp = self.save_error(tree.get(key))?; - Ok(tmp.map(|v| v.to_vec().into())) + Ok(tmp.map(From::from)) } fn len(&self, _tree: usize) -> Result { unimplemented!(".len() in transaction not supported with Sled backend") diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 5778ecf0..701639dc 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -13,6 +13,8 @@ use crate::{ pub use rusqlite; +// --- err + impl From for Error { fn from(e: rusqlite::Error) -> Error { Error(format!("{}", e).into()) @@ -25,6 +27,8 @@ impl From for TxError { } } +// -- db + pub struct SqliteDb { db: Mutex, trees: RwLock>, diff --git a/src/db/test.rs b/src/db/test.rs index 78d755cf..2f5e4c46 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -14,59 +14,66 @@ fn test_suite(db: Db) { let vc: &[u8] = &b"plup"[..]; tree.insert(ka, va).unwrap(); - assert_eq!(tree.get(ka).unwrap(), Some(va.into())); + assert_eq!(tree.get(ka).unwrap().unwrap(), va); let res = db.transaction::<_, (), _>(|tx| { - assert_eq!(tx.get(&tree, ka).unwrap(), Some(va.into())); + assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), va); tx.insert(&tree, ka, vb).unwrap(); - assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into())); + assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); tx.commit(12) }); assert!(matches!(res, Ok(12))); - assert_eq!(tree.get(ka).unwrap(), Some(vb.into())); + assert_eq!(tree.get(ka).unwrap().unwrap(), vb); let res = db.transaction::<(), _, _>(|tx| { - assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into())); + assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); tx.insert(&tree, ka, vc).unwrap(); - assert_eq!(tx.get(&tree, ka).unwrap(), Some(vc.into())); + assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vc); tx.abort(42) }); assert!(matches!(res, Err(TxError::Abort(42)))); - assert_eq!(tree.get(ka).unwrap(), Some(vb.into())); + assert_eq!(tree.get(ka).unwrap().unwrap(), vb); let mut iter = tree.iter().unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); tree.insert(kb, vc).unwrap(); - assert_eq!(tree.get(kb).unwrap(), Some(vc.into())); + assert_eq!(tree.get(kb).unwrap().unwrap(), vc); let mut iter = tree.iter().unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); - assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); assert!(iter.next().is_none()); drop(iter); let mut iter = tree.range(kint..).unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); assert!(iter.next().is_none()); drop(iter); let mut iter = tree.range_rev(..kint).unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); let mut iter = tree.iter_rev().unwrap(); - assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into())); - assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into())); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc)); + let next = iter.next().unwrap().unwrap(); + assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb)); assert!(iter.next().is_none()); drop(iter); } diff --git a/src/garage/repair.rs b/src/garage/repair.rs index 762a8300..a096aaa0 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -117,7 +117,7 @@ impl Repair { None => Ok(None), Some(item) => { let (item_key, item_bytes) = item?; - Ok(Some((item_key.into_owned(), item_bytes.into_owned()))) + Ok(Some((item_key.into_vec(), item_bytes.into_vec()))) } } } @@ -173,7 +173,7 @@ impl Repair { None => Ok(None), Some(item) => { let (item_key, item_bytes) = item?; - Ok(Some((item_key.into_owned(), item_bytes.into_owned()))) + Ok(Some((item_key.into_vec(), item_bytes.into_vec()))) } } } diff --git a/src/table/data.rs b/src/table/data.rs index ebfae551..dc1fd445 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -276,7 +276,7 @@ where if blake2sum(&cur_v[..]) == vhash { tx.remove(&self.store, k)?; tx.insert(&self.merkle_todo, k, vec![])?; - return Ok(Some(cur_v.into_owned())); + return Ok(Some(cur_v.into_vec())); } } Ok(None) diff --git a/src/table/gc.rs b/src/table/gc.rs index 04872a38..260fecfa 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -378,8 +378,11 @@ impl GcTodoEntry { let key = self.todo_table_key(); gc_todo_tree.db().transaction(|tx| { let old_val = tx.get(gc_todo_tree, &key)?; - if old_val == Some(self.value_hash.as_slice().into()) { - tx.remove(gc_todo_tree, &key)?; + match old_val { + Some(ov) if ov == self.value_hash.as_slice() => { + tx.remove(gc_todo_tree, &key)?; + } + _ => (), } tx.commit(()) })?; diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 6e0c2f7e..f7dca97b 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -142,11 +142,12 @@ where let deleted = self.data.merkle_todo.db().transaction(|tx| { let old_val = tx.get(&self.data.merkle_todo, k)?; - if old_val == Some(vhash_by.into()) { - tx.remove(&self.data.merkle_todo, k)?; - tx.commit(true) - } else { - tx.commit(false) + match old_val { + Some(ov) if ov == vhash_by => { + tx.remove(&self.data.merkle_todo, k)?; + tx.commit(true) + } + _ => tx.commit(false), } })?; -- 2.43.0 From cc0d984118e345c72341ff12d74b1c35deb20ae8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 11:14:24 +0200 Subject: [PATCH 14/64] Fix most clippy lints --- src/block/manager.rs | 6 +++-- src/db/lib.rs | 54 ++++++++++++++++++++-------------------- src/db/sled_adapter.rs | 42 ++++++++++++++++--------------- src/db/sqlite_adapter.rs | 52 +++++++++++++++++++------------------- src/db/test.rs | 4 +-- src/garage/repair.rs | 7 ++++-- src/garage/server.rs | 2 +- src/table/data.rs | 8 ++---- 8 files changed, 90 insertions(+), 85 deletions(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index de9b08c2..fbca74e2 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -56,6 +56,8 @@ const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6; // to delete the block locally. pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600); +type OptKVPair = Option<(Vec, Vec)>; + /// RPC messages used to share blocks of data between nodes #[derive(Debug, Serialize, Deserialize)] pub enum BlockRpc { @@ -640,7 +642,7 @@ impl BlockManager { } } - fn resync_get_next(&self) -> Result, Vec)>, db::Error> { + fn resync_get_next(&self) -> Result { match self.resync_queue.iter()?.next() { None => Ok(None), Some(v) => { @@ -970,7 +972,7 @@ impl ErrorCounter { } } - fn decode<'a>(data: db::Value<'a>) -> Self { + fn decode(data: db::Value<'_>) -> Self { Self { errors: u64::from_be_bytes(data[0..8].try_into().unwrap()), last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()), diff --git a/src/db/lib.rs b/src/db/lib.rs index 4cda121f..95d2c16b 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -29,14 +29,14 @@ pub type Exporter<'a> = Box(pub(crate) Box + 'a>); -pub trait IValue<'a>: AsRef<[u8]> { - fn into_vec(&mut self) -> Vec; +pub trait IValue<'a>: AsRef<[u8]> + core::borrow::Borrow<[u8]> { + fn take_maybe(&mut self) -> Vec; } impl<'a> Value<'a> { #[inline] pub fn into_vec(mut self) -> Vec { - self.0.into_vec() + self.0.take_maybe() } } @@ -50,7 +50,7 @@ impl<'a> AsRef<[u8]> for Value<'a> { impl<'a> std::borrow::Borrow<[u8]> for Value<'a> { #[inline] fn borrow(&self) -> &[u8] { - self.0.as_ref().as_ref() + self.0.as_ref().borrow() } } @@ -82,7 +82,7 @@ impl<'a> std::fmt::Debug for Value<'a> { } impl<'a> IValue<'a> for Vec { - fn into_vec(&mut self) -> Vec { + fn take_maybe(&mut self) -> Vec { std::mem::take(self) } } @@ -106,7 +106,7 @@ impl<'a> From<&'a [u8]> for Value<'a> { } impl<'a> IValue<'a> for &'a [u8] { - fn into_vec(&mut self) -> Vec { + fn take_maybe(&mut self) -> Vec { self.to_vec() } } @@ -175,21 +175,22 @@ impl Db { } } - pub fn export<'a>(&'a self) -> Result> { + pub fn export(&self) -> Result> { self.0.export() } - pub fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> { + pub fn import(&self, ex: Exporter<'_>) -> Result<()> { self.0.import(ex) } } +#[allow(clippy::len_without_is_empty)] impl Tree { pub fn db(&self) -> Db { Db(self.0.clone()) } - pub fn get<'a, T: AsRef<[u8]>>(&'a self, key: T) -> Result>> { + pub fn get>(&self, key: T) -> Result>> { self.0.get(self.1, key.as_ref()) } pub fn len(&self) -> Result { @@ -199,18 +200,18 @@ impl Tree { pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { self.0.insert(self.1, key.as_ref(), value.as_ref()) } - pub fn remove<'a, T: AsRef<[u8]>>(&'a self, key: T) -> Result { + pub fn remove>(&self, key: T) -> Result { self.0.remove(self.1, key.as_ref()) } - pub fn iter<'a>(&'a self) -> Result> { + pub fn iter(&self) -> Result> { self.0.iter(self.1) } - pub fn iter_rev<'a>(&'a self) -> Result> { + pub fn iter_rev(&self) -> Result> { self.0.iter_rev(self.1) } - pub fn range<'a, K, R>(&'a self, range: R) -> Result> + pub fn range(&self, range: R) -> Result> where K: AsRef<[u8]>, R: RangeBounds, @@ -219,7 +220,7 @@ impl Tree { let eb = range.end_bound(); self.0.range(self.1, get_bound(sb), get_bound(eb)) } - pub fn range_rev<'a, K, R>(&'a self, range: R) -> Result> + pub fn range_rev(&self, range: R) -> Result> where K: AsRef<[u8]>, R: RangeBounds, @@ -230,6 +231,7 @@ impl Tree { } } +#[allow(clippy::len_without_is_empty)] impl<'a> Transaction<'a> { pub fn get>(&self, tree: &Tree, key: T) -> Result>> { self.0.get(tree.1, key.as_ref()) @@ -278,12 +280,10 @@ impl<'a> Transaction<'a> { // ---- - #[must_use] pub fn abort(self, e: E) -> TxResult { Err(TxError::Abort(e)) } - #[must_use] pub fn commit(self, r: R) -> TxResult { Ok(r) } @@ -294,32 +294,32 @@ impl<'a> Transaction<'a> { pub(crate) trait IDb: Send + Sync { fn open_tree(&self, name: &str) -> Result; - fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>>; + fn get(&self, tree: usize, key: &[u8]) -> Result>>; fn len(&self, tree: usize) -> Result; fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; fn remove(&self, tree: usize, key: &[u8]) -> Result; - fn iter<'a>(&'a self, tree: usize) -> Result>; - fn iter_rev<'a>(&'a self, tree: usize) -> Result>; + fn iter(&self, tree: usize) -> Result>; + fn iter_rev(&self, tree: usize) -> Result>; - fn range<'a, 'r>( - &'a self, + fn range<'r>( + &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result>; - fn range_rev<'a, 'r>( - &'a self, + ) -> Result>; + fn range_rev<'r>( + &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result>; + ) -> Result>; fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; - fn export<'a>(&'a self) -> Result>; - fn import<'a>(&self, ex: Exporter<'a>) -> Result<()>; + fn export(&self) -> Result>; + fn import(&self, ex: Exporter<'_>) -> Result<()>; } pub(crate) trait ITx<'a> { diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index acee305b..7382776f 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -27,7 +27,7 @@ impl From for Error { // -- val impl<'a> IValue<'a> for sled::IVec { - fn into_vec(&mut self) -> Vec { + fn take_maybe(&mut self) -> Vec { self.to_vec() } } @@ -52,7 +52,7 @@ pub struct SledDb { } impl SledDb { - pub fn new(db: sled::Db) -> Db { + pub fn init(db: sled::Db) -> Db { let s = Self { db, trees: RwLock::new((Vec::new(), HashMap::new())), @@ -67,7 +67,7 @@ impl SledDb { .0 .get(i) .cloned() - .ok_or(Error("invalid tree id".into())) + .ok_or_else(|| Error("invalid tree id".into())) } } @@ -87,7 +87,7 @@ impl IDb for SledDb { // ---- - fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>> { + fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; Ok(tree.get(key)?.map(From::from)) } @@ -108,37 +108,37 @@ impl IDb for SledDb { Ok(()) } - fn iter<'a>(&'a self, tree: usize) -> Result> { + fn iter(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.iter().map(|v| { v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) }))) } - fn iter_rev<'a>(&'a self, tree: usize) -> Result> { + fn iter_rev(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.iter().rev().map(|v| { v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) }))) } - fn range<'a, 'r>( - &'a self, + fn range<'r>( + &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result> { + ) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).map(|v| { v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) }))) } - fn range_rev<'a, 'r>( - &'a self, + fn range_rev<'r>( + &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result> { + ) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).rev().map( |v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into), @@ -178,7 +178,7 @@ impl IDb for SledDb { // ---- - fn export<'a>(&'a self) -> Result> { + fn export(&self) -> Result> { let mut trees = vec![]; for name in self.db.tree_names() { let name = std::str::from_utf8(&name) @@ -188,17 +188,17 @@ impl IDb for SledDb { let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); trees.push((name, tree)); } - let trees_exporter: Exporter<'a> = Box::new(trees.into_iter().map(|(name, tree)| { - let iter: ValueIter<'a> = Box::new( + let trees_exporter: Exporter<'_> = Box::new(trees.into_iter().map(|(name, tree)| { + let iter: ValueIter<'_> = Box::new( tree.iter() .map(|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)), ); - Ok((name.to_string(), iter)) + Ok((name, iter)) })); Ok(trees_exporter) } - fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> { + fn import(&self, ex: Exporter<'_>) -> Result<()> { for ex_tree in ex { let (name, data) = ex_tree?; @@ -234,9 +234,11 @@ struct SledTx<'a> { impl<'a> SledTx<'a> { fn get_tree(&self, i: usize) -> Result<&TransactionalTree> { - self.trees.get(i).ok_or(Error( - "invalid tree id (it might have been openned after the transaction started)".into(), - )) + self.trees.get(i).ok_or_else(|| { + Error( + "invalid tree id (it might have been openned after the transaction started)".into(), + ) + }) } fn save_error(&self, v: std::result::Result) -> Result { diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 701639dc..31d20553 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -35,7 +35,7 @@ pub struct SqliteDb { } impl SqliteDb { - pub fn new(db: rusqlite::Connection) -> Db { + pub fn init(db: rusqlite::Connection) -> Db { let s = Self { db: Mutex::new(db), trees: RwLock::new(Vec::new()), @@ -49,7 +49,7 @@ impl SqliteDb { .unwrap() .get(i) .cloned() - .ok_or(Error("invalid tree id".into())) + .ok_or_else(|| Error("invalid tree id".into())) } } @@ -77,7 +77,7 @@ impl IDb for SqliteDb { // ---- - fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result>> { + fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; let db = self.db.lock().unwrap(); let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; @@ -102,7 +102,7 @@ impl IDb for SqliteDb { let mut res_iter = stmt.query([])?; match res_iter.next()? { None => Ok(0), - Some(v) => Ok(v.get::<_, usize>(0)?.into()), + Some(v) => Ok(v.get::<_, usize>(0)?), } } @@ -116,24 +116,24 @@ impl IDb for SqliteDb { Ok(()) } - fn iter<'a>(&'a self, tree: usize) -> Result> { + fn iter(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); - DbValueIterator::new(self.db.lock().unwrap(), &sql, []) + DbValueIterator::make(self.db.lock().unwrap(), &sql, []) } - fn iter_rev<'a>(&'a self, tree: usize) -> Result> { + fn iter_rev(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree); - DbValueIterator::new(self.db.lock().unwrap(), &sql, []) + DbValueIterator::make(self.db.lock().unwrap(), &sql, []) } - fn range<'a, 'r>( - &'a self, + fn range<'r>( + &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result> { + ) -> Result> { let tree = self.get_tree(tree)?; let (bounds_sql, params) = bounds_sql(low, high); @@ -143,18 +143,18 @@ impl IDb for SqliteDb { .iter() .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - DbValueIterator::new::<&[&dyn rusqlite::ToSql]>( + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>( self.db.lock().unwrap(), &sql, params.as_ref(), ) } - fn range_rev<'a, 'r>( - &'a self, + fn range_rev<'r>( + &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result> { + ) -> Result> { let tree = self.get_tree(tree)?; let (bounds_sql, params) = bounds_sql(low, high); @@ -164,7 +164,7 @@ impl IDb for SqliteDb { .iter() .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - DbValueIterator::new::<&[&dyn rusqlite::ToSql]>( + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>( self.db.lock().unwrap(), &sql, params.as_ref(), @@ -200,11 +200,11 @@ impl IDb for SqliteDb { // ---- - fn export<'a>(&'a self) -> Result> { + fn export(&self) -> Result> { unimplemented!() } - fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> { + fn import(&self, ex: Exporter<'_>) -> Result<()> { unimplemented!() } @@ -220,9 +220,11 @@ struct SqliteTx<'a> { impl<'a> SqliteTx<'a> { fn get_tree(&self, i: usize) -> Result { - self.trees.get(i).cloned().ok_or(Error( - "invalid tree id (it might have been openned after the transaction started)".into(), - )) + self.trees.get(i).cloned().ok_or_else(|| { + Error( + "invalid tree id (it might have been openned after the transaction started)".into(), + ) + }) } } @@ -244,7 +246,7 @@ impl<'a> ITx<'a> for SqliteTx<'a> { let mut res_iter = stmt.query([])?; match res_iter.next()? { None => Ok(0), - Some(v) => Ok(v.get::<_, usize>(0)?.into()), + Some(v) => Ok(v.get::<_, usize>(0)?), } } @@ -299,13 +301,13 @@ struct DbValueIterator<'a> { } impl<'a> DbValueIterator<'a> { - fn new( + fn make( db: MutexGuard<'a, Connection>, sql: &str, args: P, ) -> Result> { let res = DbValueIterator { - db: db, + db, stmt: None, iter: None, _pin: PhantomPinned, @@ -314,7 +316,7 @@ impl<'a> DbValueIterator<'a> { unsafe { let db = NonNull::from(&boxed.db); - let stmt = db.as_ref().prepare(&sql)?; + let stmt = db.as_ref().prepare(sql)?; let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed); Pin::get_unchecked_mut(mut_ref).stmt = Some(stmt); diff --git a/src/db/test.rs b/src/db/test.rs index 2f5e4c46..75200cf1 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -81,13 +81,13 @@ fn test_suite(db: Db) { #[test] fn test_sled_db() { let path = mktemp::Temp::new_dir().unwrap(); - let db = SledDb::new(sled::open(path.to_path_buf()).unwrap()); + let db = SledDb::init(sled::open(path.to_path_buf()).unwrap()); test_suite(db); drop(path); } #[test] fn test_sqlite_db() { - let db = SqliteDb::new(rusqlite::Connection::open_in_memory().unwrap()); + let db = SqliteDb::init(rusqlite::Connection::open_in_memory().unwrap()); test_suite(db); } diff --git a/src/garage/repair.rs b/src/garage/repair.rs index a096aaa0..5735d6d0 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -16,6 +16,8 @@ pub struct Repair { pub garage: Arc, } +type OptKVPair = Option<(Vec, Vec)>; + impl Repair { pub async fn repair_worker(&self, opt: RepairOpt, must_exit: watch::Receiver) { if let Err(e) = self.repair_worker_aux(opt, must_exit).await { @@ -105,7 +107,7 @@ impl Repair { Ok(()) } - fn get_next_version_after(&self, pos: &[u8]) -> Result, Vec)>, Error> { + fn get_next_version_after(&self, pos: &[u8]) -> Result { match self .garage .version_table @@ -161,7 +163,8 @@ impl Repair { Ok(()) } - fn get_next_block_ref_after(&self, pos: &[u8]) -> Result, Vec)>, Error> { + #[allow(clippy::type_complexity)] + fn get_next_block_ref_after(&self, pos: &[u8]) -> Result { match self .garage .block_ref_table diff --git a/src/garage/server.rs b/src/garage/server.rs index 9d148ee7..b102067e 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -40,7 +40,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { .flush_every_ms(Some(config.sled_flush_every_ms)) .open() .expect("Unable to open sled DB"); - let db = db::sled_adapter::SledDb::new(db); + let db = db::sled_adapter::SledDb::init(db); info!("Initializing background runner..."); let watch_cancel = netapp::util::watch_ctrl_c(); diff --git a/src/table/data.rs b/src/table/data.rs index dc1fd445..427ce763 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -206,12 +206,8 @@ where if value_changed || encoding_changed { let new_bytes_hash = blake2sum(&new_bytes[..]); - tx.insert( - &self.merkle_todo, - tree_key.to_vec(), - new_bytes_hash.as_slice(), - )?; - tx.insert(&self.store, tree_key.to_vec(), new_bytes)?; + tx.insert(&self.merkle_todo, tree_key, new_bytes_hash.as_slice())?; + tx.insert(&self.store, tree_key, new_bytes)?; Ok(Some((old_entry, new_entry, new_bytes_hash))) } else { Ok(None) -- 2.43.0 From df0877bbba91d210fa8a91bd095ca13e0ea2176f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 11:44:41 +0200 Subject: [PATCH 15/64] Conversion utility --- Cargo.lock | 1 + src/db/Cargo.toml | 11 ++++++- src/db/bin/convert.rs | 55 ++++++++++++++++++++++++++++++++++ src/db/lib.rs | 43 ++++++++++++++++++++------- src/db/sled_adapter.rs | 64 +++++++++------------------------------- src/db/sqlite_adapter.rs | 35 ++++++++++++---------- 6 files changed, 131 insertions(+), 78 deletions(-) create mode 100644 src/db/bin/convert.rs diff --git a/Cargo.lock b/Cargo.lock index 9ff2c02d..73879369 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1014,6 +1014,7 @@ dependencies = [ name = "garage_db" version = "0.8.0" dependencies = [ + "clap 3.1.18", "err-derive 0.3.1", "hexdump", "mktemp", diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index ca189a67..22abc0b9 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -11,7 +11,10 @@ readme = "../../README.md" [lib] path = "lib.rs" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[[bin]] +name = "convert" +path = "bin/convert.rs" +required-features = ["cli"] [dependencies] err-derive = "0.3" @@ -20,5 +23,11 @@ hexdump = "0.1" sled = "0.34" rusqlite = "0.27" +# cli deps +clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } + [dev-dependencies] mktemp = "0.4" + +[features] +cli = ["clap"] diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs new file mode 100644 index 00000000..8c4f0ddc --- /dev/null +++ b/src/db/bin/convert.rs @@ -0,0 +1,55 @@ +use std::path::PathBuf; + +use garage_db::*; + +use clap::{Parser}; + +/// K2V command line interface +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Args { + /// Input DB path + #[clap(short = 'i')] + input_path: PathBuf, + /// Input DB engine + #[clap(short = 'a')] + input_engine: String, + + /// Output DB path + #[clap(short = 'o')] + output_path: PathBuf, + /// Output DB engine + #[clap(short = 'b')] + output_engine: String, +} + +fn main() { + let args = Args::parse(); + match do_conversion(args) { + Ok(()) => println!("Success!"), + Err(e) => eprintln!("Error: {}", e), + } +} + +fn do_conversion(args: Args) -> Result<()> { + let input = open_db(args.input_path, args.input_engine)?; + let output = open_db(args.output_path, args.output_engine)?; + output.import(&input)?; + Ok(()) +} + +fn open_db(path: PathBuf, engine: String) -> Result { + match engine.as_str() { + "sled" => { + let db = sled_adapter::sled::Config::default() + .path(&path) + .open()?; + Ok(sled_adapter::SledDb::init(db)) + } + "sqlite" | "rusqlite" => { + let db = sqlite_adapter::rusqlite::Connection::open(&path)?; + Ok(sqlite_adapter::SqliteDb::init(db)) + } + e => Err(Error(format!("Invalid DB engine: {}", e).into())), + } +} diff --git a/src/db/lib.rs b/src/db/lib.rs index 95d2c16b..49ec0765 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -23,8 +23,6 @@ pub struct Tree(pub(crate) Arc, pub(crate) usize); pub type ValueIter<'a> = Box, Value<'a>)>> + 'a>; -pub type Exporter<'a> = Box)>> + 'a>; - // ---- pub struct Value<'a>(pub(crate) Box + 'a>); @@ -115,7 +113,7 @@ impl<'a> IValue<'a> for &'a [u8] { #[derive(Debug, Error)] #[error(display = "{}", _0)] -pub struct Error(Cow<'static, str>); +pub struct Error(pub Cow<'static, str>); pub type Result = std::result::Result; @@ -140,6 +138,10 @@ impl Db { Ok(Tree(self.0.clone(), tree_id)) } + pub fn list_trees(&self) -> Result> { + self.0.list_trees() + } + pub fn transaction(&self, fun: F) -> TxResult where F: Fn(Transaction<'_>) -> TxResult, @@ -175,12 +177,33 @@ impl Db { } } - pub fn export(&self) -> Result> { - self.0.export() - } + pub fn import(&self, other: &Db) -> Result<()> { + let existing_trees = self.list_trees()?; + if !existing_trees.is_empty() { + return Err(Error(format!("destination database already contains data: {:?}", existing_trees).into())); + } - pub fn import(&self, ex: Exporter<'_>) -> Result<()> { - self.0.import(ex) + let tree_names = other.list_trees()?; + for name in tree_names { + let tree = self.open_tree(&name)?; + if tree.len()? > 0 { + return Err(Error(format!("tree {} already contains data", name).into())); + } + + let ex_tree = other.open_tree(&name)?; + + let mut i = 0; + for item in ex_tree.iter()? { + let (k, v) = item?; + tree.insert(k, v)?; + i += 1; + if i % 1000 == 0 { + println!("{}: imported {}", name, i); + } + } + println!("{}: finished importing, {} items", name, i); + } + Ok(()) } } @@ -293,6 +316,7 @@ impl<'a> Transaction<'a> { pub(crate) trait IDb: Send + Sync { fn open_tree(&self, name: &str) -> Result; + fn list_trees(&self) -> Result>; fn get(&self, tree: usize, key: &[u8]) -> Result>>; fn len(&self, tree: usize) -> Result; @@ -317,9 +341,6 @@ pub(crate) trait IDb: Send + Sync { ) -> Result>; fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; - - fn export(&self) -> Result>; - fn import(&self, ex: Exporter<'_>) -> Result<()>; } pub(crate) trait ITx<'a> { diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 7382776f..3388b0ca 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -10,8 +10,7 @@ use sled::transaction::{ }; use crate::{ - Db, Error, Exporter, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, - ValueIter, + Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter, }; pub use sled; @@ -85,6 +84,19 @@ impl IDb for SledDb { } } + fn list_trees(&self) -> Result> { + let mut trees = vec![]; + for name in self.db.tree_names() { + let name = std::str::from_utf8(&name) + .map_err(|e| Error(format!("{}", e).into()))? + .to_string(); + if name != "__sled__default" { + trees.push(name); + } + } + Ok(trees) + } + // ---- fn get(&self, tree: usize, key: &[u8]) -> Result>> { @@ -175,54 +187,6 @@ impl IDb for SledDb { Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())), } } - - // ---- - - fn export(&self) -> Result> { - let mut trees = vec![]; - for name in self.db.tree_names() { - let name = std::str::from_utf8(&name) - .map_err(|e| Error(format!("{}", e).into()))? - .to_string(); - let tree = self.open_tree(&name)?; - let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); - trees.push((name, tree)); - } - let trees_exporter: Exporter<'_> = Box::new(trees.into_iter().map(|(name, tree)| { - let iter: ValueIter<'_> = Box::new( - tree.iter() - .map(|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)), - ); - Ok((name, iter)) - })); - Ok(trees_exporter) - } - - fn import(&self, ex: Exporter<'_>) -> Result<()> { - for ex_tree in ex { - let (name, data) = ex_tree?; - - let tree = self.open_tree(&name)?; - let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); - if !tree.is_empty() { - return Err(Error(format!("tree {} already contains data", name).into())); - } - - let mut i = 0; - for item in data { - let (k, v) = item?; - tree.insert(k, v)?; - i += 1; - if i % 1000 == 0 { - println!("{}: imported {}", name, i); - } - } - println!("{}: finished importing, {} items", name, i); - } - Ok(()) - } - - // ---- } // ---- diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 31d20553..386eb951 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -7,9 +7,7 @@ use std::sync::{Arc, Mutex, MutexGuard, RwLock}; use rusqlite::{params, Connection, Rows, Statement, Transaction}; -use crate::{ - Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, -}; +use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; pub use rusqlite; @@ -55,8 +53,10 @@ impl SqliteDb { impl IDb for SqliteDb { fn open_tree(&self, name: &str) -> Result { + let name = format!("tree_{}", name.replace(":", "_COLON_")); + let mut trees = self.trees.write().unwrap(); - if let Some(i) = trees.iter().position(|x| x == name) { + if let Some(i) = trees.iter().position(|x| x == &name) { Ok(i) } else { self.db.lock().unwrap().execute( @@ -75,6 +75,21 @@ impl IDb for SqliteDb { } } + fn list_trees(&self) -> Result> { + let mut trees = vec![]; + let db = self.db.lock().unwrap(); + let mut stmt = db.prepare( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let name = row.get::<_, String>(0)?; + let name = name.replace("_COLON_", ":"); + trees.push(name); + } + Ok(trees) + } + // ---- fn get(&self, tree: usize, key: &[u8]) -> Result>> { @@ -197,18 +212,6 @@ impl IDb for SqliteDb { } } } - - // ---- - - fn export(&self) -> Result> { - unimplemented!() - } - - fn import(&self, ex: Exporter<'_>) -> Result<()> { - unimplemented!() - } - - // ---- } // ---- -- 2.43.0 From bd9ff432d716020d7235ece76257b60dc3ead95a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 12:12:25 +0200 Subject: [PATCH 16/64] Garage works on sqlite, but it's a hack --- Cargo.lock | 1 + src/db/Cargo.toml | 1 + src/db/bin/convert.rs | 8 ++-- src/db/lib.rs | 8 +++- src/db/sqlite_adapter.rs | 80 ++++++++++++++++++++++++++++++++-------- src/garage/server.rs | 35 +++++++++++++----- src/table/merkle.rs | 9 ++++- src/util/config.rs | 11 +++++- 8 files changed, 119 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73879369..c43aa81f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1017,6 +1017,7 @@ dependencies = [ "clap 3.1.18", "err-derive 0.3.1", "hexdump", + "log", "mktemp", "rusqlite", "sled", diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 22abc0b9..36b96229 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -19,6 +19,7 @@ required-features = ["cli"] [dependencies] err-derive = "0.3" hexdump = "0.1" +log = "0.4" sled = "0.34" rusqlite = "0.27" diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs index 8c4f0ddc..7525bcc9 100644 --- a/src/db/bin/convert.rs +++ b/src/db/bin/convert.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use garage_db::*; -use clap::{Parser}; +use clap::Parser; /// K2V command line interface #[derive(Parser, Debug)] @@ -41,12 +41,10 @@ fn do_conversion(args: Args) -> Result<()> { fn open_db(path: PathBuf, engine: String) -> Result { match engine.as_str() { "sled" => { - let db = sled_adapter::sled::Config::default() - .path(&path) - .open()?; + let db = sled_adapter::sled::Config::default().path(&path).open()?; Ok(sled_adapter::SledDb::init(db)) } - "sqlite" | "rusqlite" => { + "sqlite" | "sqlite3" | "rusqlite" => { let db = sqlite_adapter::rusqlite::Connection::open(&path)?; Ok(sqlite_adapter::SqliteDb::init(db)) } diff --git a/src/db/lib.rs b/src/db/lib.rs index 49ec0765..1b31df43 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -180,7 +180,13 @@ impl Db { pub fn import(&self, other: &Db) -> Result<()> { let existing_trees = self.list_trees()?; if !existing_trees.is_empty() { - return Err(Error(format!("destination database already contains data: {:?}", existing_trees).into())); + return Err(Error( + format!( + "destination database already contains data: {:?}", + existing_trees + ) + .into(), + )); } let tree_names = other.list_trees()?; diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 386eb951..49c07562 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -5,6 +5,8 @@ use std::pin::Pin; use std::ptr::NonNull; use std::sync::{Arc, Mutex, MutexGuard, RwLock}; +use log::trace; + use rusqlite::{params, Connection, Rows, Statement, Transaction}; use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; @@ -53,13 +55,17 @@ impl SqliteDb { impl IDb for SqliteDb { fn open_tree(&self, name: &str) -> Result { - let name = format!("tree_{}", name.replace(":", "_COLON_")); + let name = format!("tree_{}", name.replace(':', "_COLON_")); let mut trees = self.trees.write().unwrap(); if let Some(i) = trees.iter().position(|x| x == &name) { Ok(i) } else { - self.db.lock().unwrap().execute( + trace!("open tree {}: lock db", name); + let db = self.db.lock().unwrap(); + trace!("create table {}", name); + + db.execute( &format!( "CREATE TABLE IF NOT EXISTS {} ( k BLOB PRIMARY KEY, @@ -69,6 +75,8 @@ impl IDb for SqliteDb { ), [], )?; + trace!("table created: {}", name); + let i = trees.len(); trees.push(name.to_string()); Ok(i) @@ -77,7 +85,11 @@ impl IDb for SqliteDb { fn list_trees(&self) -> Result> { let mut trees = vec![]; + + trace!("list_trees: lock db"); let db = self.db.lock().unwrap(); + trace!("list_trees: lock acquired"); + let mut stmt = db.prepare( "SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'", )?; @@ -94,7 +106,11 @@ impl IDb for SqliteDb { fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; + + trace!("get: lock db"); let db = self.db.lock().unwrap(); + trace!("get: lock acquired"); + let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; let mut res_iter = stmt.query([key])?; match res_iter.next()? { @@ -105,14 +121,22 @@ impl IDb for SqliteDb { fn remove(&self, tree: usize, key: &[u8]) -> Result { let tree = self.get_tree(tree)?; + + trace!("remove: lock db"); let db = self.db.lock().unwrap(); + trace!("remove: lock acquired"); + let res = db.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; Ok(res > 0) } fn len(&self, tree: usize) -> Result { let tree = self.get_tree(tree)?; + + trace!("len: lock db"); let db = self.db.lock().unwrap(); + trace!("len: lock acquired"); + let mut stmt = db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; let mut res_iter = stmt.query([])?; match res_iter.next()? { @@ -123,7 +147,11 @@ impl IDb for SqliteDb { fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; + + trace!("insert: lock db"); let db = self.db.lock().unwrap(); + trace!("insert: lock acquired"); + db.execute( &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), params![key, value], @@ -134,13 +162,23 @@ impl IDb for SqliteDb { fn iter(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); - DbValueIterator::make(self.db.lock().unwrap(), &sql, []) + + trace!("iter {}: lock db", tree); + let db = self.db.lock().unwrap(); + trace!("iter {}: lock acquired", tree); + + DbValueIterator::make(db, &sql, []) } fn iter_rev(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree); - DbValueIterator::make(self.db.lock().unwrap(), &sql, []) + + trace!("iter_rev {}: lock db", tree); + let db = self.db.lock().unwrap(); + trace!("iter_rev {}: lock acquired", tree); + + DbValueIterator::make(db, &sql, []) } fn range<'r>( @@ -158,11 +196,12 @@ impl IDb for SqliteDb { .iter() .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - DbValueIterator::make::<&[&dyn rusqlite::ToSql]>( - self.db.lock().unwrap(), - &sql, - params.as_ref(), - ) + + trace!("range {}: lock db", tree); + let db = self.db.lock().unwrap(); + trace!("range {}: lock acquired", tree); + + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(db, &sql, params.as_ref()) } fn range_rev<'r>( &self, @@ -179,23 +218,28 @@ impl IDb for SqliteDb { .iter() .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - DbValueIterator::make::<&[&dyn rusqlite::ToSql]>( - self.db.lock().unwrap(), - &sql, - params.as_ref(), - ) + + trace!("range_rev {}: lock db", tree); + let db = self.db.lock().unwrap(); + trace!("range_rev {}: lock acquired", tree); + + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(db, &sql, params.as_ref()) } // ---- fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { let trees = self.trees.read().unwrap(); + + trace!("transaction: lock db"); let mut db = self.db.lock().unwrap(); + trace!("transaction: lock acquired"); + let tx = SqliteTx { tx: db.transaction()?, trees: trees.as_ref(), }; - match f.try_on(&tx) { + let res = match f.try_on(&tx) { TxFnResult::Ok => { tx.tx.commit()?; Ok(()) @@ -210,7 +254,10 @@ impl IDb for SqliteDb { "(this message will be discarded)".into(), ))) } - } + }; + + trace!("transaction done"); + res } } @@ -337,6 +384,7 @@ impl<'a> DbValueIterator<'a> { impl<'a> Drop for DbValueIterator<'a> { fn drop(&mut self) { + trace!("drop iter"); drop(self.iter.take()); drop(self.stmt.take()); } diff --git a/src/garage/server.rs b/src/garage/server.rs index b102067e..bd34456d 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -32,15 +32,32 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { let config = read_config(config_file).expect("Unable to read config file"); info!("Opening database..."); - let mut db_path = config.metadata_dir.clone(); - db_path.push("db"); - let db = db::sled_adapter::sled::Config::default() - .path(&db_path) - .cache_capacity(config.sled_cache_capacity) - .flush_every_ms(Some(config.sled_flush_every_ms)) - .open() - .expect("Unable to open sled DB"); - let db = db::sled_adapter::SledDb::init(db); + let db = match config.db_engine.as_str() { + "sled" => { + let mut db_path = config.metadata_dir.clone(); + db_path.push("db"); + let db = db::sled_adapter::sled::Config::default() + .path(&db_path) + .cache_capacity(config.sled_cache_capacity) + .flush_every_ms(Some(config.sled_flush_every_ms)) + .open() + .expect("Unable to open sled DB"); + db::sled_adapter::SledDb::init(db) + } + "sqlite" => { + let mut db_path = config.metadata_dir.clone(); + db_path.push("db.sqlite"); + let db = db::sqlite_adapter::rusqlite::Connection::open(db_path) + .expect("Unable to open sqlite DB"); + db::sqlite_adapter::SqliteDb::init(db) + } + e => { + return Err(Error::Message(format!( + "Unsupported DB engine: {} (options: sled, sqlite)", + e + ))); + } + }; info!("Initializing background runner..."); let watch_cancel = netapp::util::watch_ctrl_c(); diff --git a/src/table/merkle.rs b/src/table/merkle.rs index f7dca97b..48d2c5dd 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -110,9 +110,14 @@ where } fn updater_loop_iter(&self) -> Result { - if let Some(x) = self.data.merkle_todo.iter()?.next() { + // TODO undo this iter hack + let mut iter = self.data.merkle_todo.iter()?; + if let Some(x) = iter.next() { let (key, valhash) = x?; - self.update_item(&key[..], &valhash[..])?; + let key = key.to_vec(); + let valhash = valhash.to_vec(); + drop(iter); + self.update_item(&key, &valhash)?; Ok(true) } else { Ok(false) diff --git a/src/util/config.rs b/src/util/config.rs index 99ebce31..3b37adbb 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -64,14 +64,19 @@ pub struct Config { #[serde(default)] pub kubernetes_skip_crd: bool, + // -- DB + /// Database engine to use for metadata (options: sled, sqlite) + #[serde(default = "default_db_engine")] + pub db_engine: String, + /// Sled cache size, in bytes #[serde(default = "default_sled_cache_capacity")] pub sled_cache_capacity: u64, - /// Sled flush interval in milliseconds #[serde(default = "default_sled_flush_every_ms")] pub sled_flush_every_ms: u64, + // -- APIs /// Configuration for S3 api pub s3_api: S3ApiConfig, @@ -129,6 +134,10 @@ pub struct AdminConfig { pub trace_sink: Option, } +fn default_db_engine() -> String { + "sled".into() +} + fn default_sled_cache_capacity() -> u64 { 128 * 1024 * 1024 } -- 2.43.0 From 4bf706b170f149f54fd701576692cd5cd5bc9779 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 12:35:12 +0200 Subject: [PATCH 17/64] Slightly prettier code --- src/block/manager.rs | 19 ++++------- src/db/lib.rs | 9 ++++++ src/db/sqlite_adapter.rs | 16 +++++----- src/garage/repair.rs | 68 +++++++++++----------------------------- src/table/merkle.rs | 8 +---- 5 files changed, 42 insertions(+), 78 deletions(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index fbca74e2..53baede5 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -56,8 +56,6 @@ const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6; // to delete the block locally. pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600); -type OptKVPair = Option<(Vec, Vec)>; - /// RPC messages used to share blocks of data between nodes #[derive(Debug, Serialize, Deserialize)] pub enum BlockRpc { @@ -549,7 +547,12 @@ impl BlockManager { // - Ok(false) -> no block was processed, but we are ready for the next iteration // - Err(_) -> a Sled error occurred when reading/writing from resync_queue/resync_errors async fn resync_iter(&self, must_exit: &mut watch::Receiver) -> Result { - if let Some((time_bytes, hash_bytes)) = self.resync_get_next()? { + let next = match self.resync_queue.first()? { + Some((k, v)) => Some((k.into_vec(), v.into_vec())), + None => None, + }; + + if let Some((time_bytes, hash_bytes)) = next { let time_msec = u64::from_be_bytes(time_bytes[0..8].try_into().unwrap()); let now = now_msec(); @@ -642,16 +645,6 @@ impl BlockManager { } } - fn resync_get_next(&self) -> Result { - match self.resync_queue.iter()?.next() { - None => Ok(None), - Some(v) => { - let (time_bytes, hash_bytes) = v?; - Ok(Some((time_bytes.into_vec(), hash_bytes.into_vec()))) - } - } - } - async fn resync_block(&self, hash: &Hash) -> Result<(), Error> { let BlockStatus { exists, needed } = self .mutation_lock diff --git a/src/db/lib.rs b/src/db/lib.rs index 1b31df43..045c16c5 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -226,6 +226,15 @@ impl Tree { self.0.len(self.1) } + pub fn first(&self) -> Result, Value<'_>)>> { + self.iter()?.next().transpose() + } + pub fn get_gt>(&self, from: T) -> Result, Value<'_>)>> { + self.range((Bound::Excluded(from), Bound::Unbounded))? + .next() + .transpose() + } + pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { self.0.insert(self.1, key.as_ref(), value.as_ref()) } diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 49c07562..ab7efbf8 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -107,9 +107,9 @@ impl IDb for SqliteDb { fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; - trace!("get: lock db"); + trace!("get {}: lock db", tree); let db = self.db.lock().unwrap(); - trace!("get: lock acquired"); + trace!("get {}: lock acquired", tree); let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; let mut res_iter = stmt.query([key])?; @@ -122,9 +122,9 @@ impl IDb for SqliteDb { fn remove(&self, tree: usize, key: &[u8]) -> Result { let tree = self.get_tree(tree)?; - trace!("remove: lock db"); + trace!("remove {}: lock db", tree); let db = self.db.lock().unwrap(); - trace!("remove: lock acquired"); + trace!("remove {}: lock acquired", tree); let res = db.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; Ok(res > 0) @@ -133,9 +133,9 @@ impl IDb for SqliteDb { fn len(&self, tree: usize) -> Result { let tree = self.get_tree(tree)?; - trace!("len: lock db"); + trace!("len {}: lock db", tree); let db = self.db.lock().unwrap(); - trace!("len: lock acquired"); + trace!("len {}: lock acquired", tree); let mut stmt = db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; let mut res_iter = stmt.query([])?; @@ -148,9 +148,9 @@ impl IDb for SqliteDb { fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; - trace!("insert: lock db"); + trace!("insert {}: lock db", tree); let db = self.db.lock().unwrap(); - trace!("insert: lock acquired"); + trace!("insert {}: lock acquired", tree); db.execute( &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), diff --git a/src/garage/repair.rs b/src/garage/repair.rs index 5735d6d0..a90b4148 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -1,4 +1,3 @@ -use core::ops::Bound; use std::sync::Arc; use tokio::sync::watch; @@ -16,8 +15,6 @@ pub struct Repair { pub garage: Arc, } -type OptKVPair = Option<(Vec, Vec)>; - impl Repair { pub async fn repair_worker(&self, opt: RepairOpt, must_exit: watch::Receiver) { if let Err(e) = self.repair_worker_aux(opt, must_exit).await { @@ -68,8 +65,15 @@ impl Repair { async fn repair_versions(&self, must_exit: &watch::Receiver) -> Result<(), Error> { let mut pos = vec![]; - while let Some((item_key, item_bytes)) = self.get_next_version_after(&pos)? { - pos = item_key; + while *must_exit.borrow() { + let item_bytes = { + let (k, v) = match self.garage.version_table.data.store.get_gt(pos)? { + Some(pair) => pair, + None => break, + }; + pos = k.into_vec(); + v.into_vec() + }; let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?; if version.deleted.get() { @@ -99,36 +103,22 @@ impl Repair { )) .await?; } - - if *must_exit.borrow() { - break; - } } Ok(()) } - fn get_next_version_after(&self, pos: &[u8]) -> Result { - match self - .garage - .version_table - .data - .store - .range::<&[u8], _>((Bound::Excluded(pos), Bound::Unbounded))? - .next() - { - None => Ok(None), - Some(item) => { - let (item_key, item_bytes) = item?; - Ok(Some((item_key.into_vec(), item_bytes.into_vec()))) - } - } - } - async fn repair_block_ref(&self, must_exit: &watch::Receiver) -> Result<(), Error> { let mut pos = vec![]; - while let Some((item_key, item_bytes)) = self.get_next_block_ref_after(&pos)? { - pos = item_key; + while *must_exit.borrow() { + let item_bytes = { + let (k, v) = match self.garage.block_ref_table.data.store.get_gt(pos)? { + Some(pair) => pair, + None => break, + }; + pos = k.into_vec(); + v.into_vec() + }; let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?; if block_ref.deleted.get() { @@ -155,29 +145,7 @@ impl Repair { }) .await?; } - - if *must_exit.borrow() { - break; - } } Ok(()) } - - #[allow(clippy::type_complexity)] - fn get_next_block_ref_after(&self, pos: &[u8]) -> Result { - match self - .garage - .block_ref_table - .data - .store - .range::<&[u8], _>((Bound::Excluded(pos), Bound::Unbounded))? - .next() - { - None => Ok(None), - Some(item) => { - let (item_key, item_bytes) = item?; - Ok(Some((item_key.into_vec(), item_bytes.into_vec()))) - } - } - } } diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 48d2c5dd..c6653a64 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -110,13 +110,7 @@ where } fn updater_loop_iter(&self) -> Result { - // TODO undo this iter hack - let mut iter = self.data.merkle_todo.iter()?; - if let Some(x) = iter.next() { - let (key, valhash) = x?; - let key = key.to_vec(); - let valhash = valhash.to_vec(); - drop(iter); + if let Some((key, valhash)) = self.data.merkle_todo.first()? { self.update_item(&key, &valhash)?; Ok(true) } else { -- 2.43.0 From 295bc2741f427106a00c9a965d3ae50bcb36107d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 12:52:41 +0200 Subject: [PATCH 18/64] fix clipy lint --- src/block/manager.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index 53baede5..73dd453f 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -547,11 +547,10 @@ impl BlockManager { // - Ok(false) -> no block was processed, but we are ready for the next iteration // - Err(_) -> a Sled error occurred when reading/writing from resync_queue/resync_errors async fn resync_iter(&self, must_exit: &mut watch::Receiver) -> Result { - let next = match self.resync_queue.first()? { - Some((k, v)) => Some((k.into_vec(), v.into_vec())), - None => None, - }; - + let next = self + .resync_queue + .first()? + .map(|(k, v)| (k.into_vec(), v.into_vec())); if let Some((time_bytes, hash_bytes)) = next { let time_msec = u64::from_be_bytes(time_bytes[0..8].try_into().unwrap()); let now = now_msec(); -- 2.43.0 From 9bb58638f044722b58c7a88711fa474152d716b6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 12:56:35 +0200 Subject: [PATCH 19/64] Fix block repair to not deadlock with sqlite --- src/block/manager.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/block/manager.rs b/src/block/manager.rs index 73dd453f..decf33cc 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -218,9 +218,17 @@ impl BlockManager { /// to fix any mismatch between the two. pub async fn repair_data_store(&self, must_exit: &watch::Receiver) -> Result<(), Error> { // 1. Repair blocks from RC table. + // TODO don't do this like this + let mut hashes = vec![]; for (i, entry) in self.rc.rc.iter()?.enumerate() { let (hash, _) = entry?; let hash = Hash::try_from(&hash[..]).unwrap(); + hashes.push(hash); + if i & 0xFF == 0 && *must_exit.borrow() { + return Ok(()); + } + } + for (i, hash) in hashes.into_iter().enumerate() { self.put_to_resync(&hash, Duration::from_secs(0))?; if i & 0xFF == 0 && *must_exit.borrow() { return Ok(()); -- 2.43.0 From 3d18c9e183a1062d797a9cf1a9a6e3b1e6ae7853 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 13:35:02 +0200 Subject: [PATCH 20/64] whoops --- src/db/sqlite_adapter.rs | 1 + src/garage/repair.rs | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index ab7efbf8..4f79b34b 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -363,6 +363,7 @@ impl<'a> DbValueIterator<'a> { _pin: PhantomPinned, }; let mut boxed = Box::pin(res); + trace!("make iterator with sql: {}", sql); unsafe { let db = NonNull::from(&boxed.db); diff --git a/src/garage/repair.rs b/src/garage/repair.rs index a90b4148..cdcfc90f 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -64,8 +64,9 @@ impl Repair { async fn repair_versions(&self, must_exit: &watch::Receiver) -> Result<(), Error> { let mut pos = vec![]; + let mut i = 0; - while *must_exit.borrow() { + while !*must_exit.borrow() { let item_bytes = { let (k, v) = match self.garage.version_table.data.store.get_gt(pos)? { Some(pair) => pair, @@ -75,6 +76,11 @@ impl Repair { v.into_vec() }; + i += 1; + if i % 1000 == 0 { + info!("repair_versions: {}", i); + } + let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?; if version.deleted.get() { continue; @@ -104,13 +110,15 @@ impl Repair { .await?; } } + info!("repair_versions: finished, done {}", i); Ok(()) } async fn repair_block_ref(&self, must_exit: &watch::Receiver) -> Result<(), Error> { let mut pos = vec![]; + let mut i = 0; - while *must_exit.borrow() { + while !*must_exit.borrow() { let item_bytes = { let (k, v) = match self.garage.block_ref_table.data.store.get_gt(pos)? { Some(pair) => pair, @@ -120,6 +128,11 @@ impl Repair { v.into_vec() }; + i += 1; + if i % 1000 == 0 { + info!("repair_block_ref: {}", i); + } + let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?; if block_ref.deleted.get() { continue; @@ -146,6 +159,7 @@ impl Repair { .await?; } } + info!("repair_block_ref: finished, done {}", i); Ok(()) } } -- 2.43.0 From d3d7df098b4188ea26d3cec29f2cc226d55d783e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 14:00:13 +0200 Subject: [PATCH 21/64] small fixes --- src/block/rc.rs | 4 +--- src/garage/server.rs | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/block/rc.rs b/src/block/rc.rs index f6d8c2aa..7d85f67e 100644 --- a/src/block/rc.rs +++ b/src/block/rc.rs @@ -26,9 +26,7 @@ impl BlockRc { Some(x) => { tx.insert(&self.rc, &hash, x)?; } - None => { - tx.remove(&self.rc, &hash)?; - } + None => unreachable!(), }; tx.commit(old_rc) })?; diff --git a/src/garage/server.rs b/src/garage/server.rs index bd34456d..a56f124a 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -32,9 +32,9 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { let config = read_config(config_file).expect("Unable to read config file"); info!("Opening database..."); + let mut db_path = config.metadata_dir.clone(); let db = match config.db_engine.as_str() { "sled" => { - let mut db_path = config.metadata_dir.clone(); db_path.push("db"); let db = db::sled_adapter::sled::Config::default() .path(&db_path) @@ -44,8 +44,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { .expect("Unable to open sled DB"); db::sled_adapter::SledDb::init(db) } - "sqlite" => { - let mut db_path = config.metadata_dir.clone(); + "sqlite" | "sqlite3" | "rusqlite" => { db_path.push("db.sqlite"); let db = db::sqlite_adapter::rusqlite::Connection::open(db_path) .expect("Unable to open sqlite DB"); -- 2.43.0 From 16e0a655d0d01e3871aee81a0a9660102d6df74e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 14:16:30 +0200 Subject: [PATCH 22/64] Update rmp-serde --- Cargo.lock | 34 ++++++++++++++++++++++++++-------- src/block/Cargo.toml | 2 +- src/garage/Cargo.toml | 2 +- src/garage/repair.rs | 4 ++-- src/model/Cargo.toml | 2 +- src/model/index_counter.rs | 8 +++----- src/model/key_table.rs | 2 +- src/model/migrate.rs | 2 +- src/model/s3/object_table.rs | 2 +- src/model/s3/version_table.rs | 2 +- src/rpc/Cargo.toml | 2 +- src/table/Cargo.toml | 2 +- src/table/data.rs | 2 +- src/table/merkle.rs | 2 +- src/util/Cargo.toml | 2 +- src/util/data.rs | 2 +- src/util/persister.rs | 4 ++-- 17 files changed, 46 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c43aa81f..a953779b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -930,7 +930,7 @@ dependencies = [ "pretty_env_logger", "prometheus", "rand 0.8.5", - "rmp-serde 0.15.5", + "rmp-serde 1.1.0", "serde", "serde_bytes", "serde_json", @@ -1002,7 +1002,7 @@ dependencies = [ "hex", "opentelemetry", "rand 0.8.5", - "rmp-serde 0.15.5", + "rmp-serde 1.1.0", "serde", "serde_bytes", "tokio", @@ -1069,7 +1069,7 @@ dependencies = [ "netapp 0.4.4", "opentelemetry", "rand 0.8.5", - "rmp-serde 0.15.5", + "rmp-serde 1.1.0", "serde", "serde_bytes", "tokio", @@ -1126,7 +1126,7 @@ dependencies = [ "opentelemetry", "pnet_datalink", "rand 0.8.5", - "rmp-serde 0.15.5", + "rmp-serde 1.1.0", "schemars", "serde", "serde_bytes", @@ -1172,7 +1172,7 @@ dependencies = [ "hexdump", "opentelemetry", "rand 0.8.5", - "rmp-serde 0.15.5", + "rmp-serde 1.1.0", "serde", "serde_bytes", "tokio", @@ -1220,7 +1220,7 @@ dependencies = [ "netapp 0.4.4", "opentelemetry", "rand 0.8.5", - "rmp-serde 0.15.5", + "rmp-serde 1.1.0", "serde", "serde_json", "sha2", @@ -2249,6 +2249,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "paste" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" + [[package]] name = "pem" version = "0.8.3" @@ -2638,12 +2644,13 @@ dependencies = [ [[package]] name = "rmp" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f55e5fa1446c4d5dd1f5daeed2a4fe193071771a2636274d0d7a3b082aa7ad6" +checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" dependencies = [ "byteorder", "num-traits", + "paste", ] [[package]] @@ -2668,6 +2675,17 @@ dependencies = [ "serde", ] +[[package]] +name = "rmp-serde" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25786b0d276110195fa3d6f3f31299900cf71dfbd6c28450f3f58a0e7f7a347e" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + [[package]] name = "roxmltree" version = "0.14.1" diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index 80346aca..1a49947c 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -28,7 +28,7 @@ tracing = "0.1.30" rand = "0.8" zstd = { version = "0.9", default-features = false } -rmp-serde = "0.15" +rmp-serde = "1.1" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index eb643160..d5299291 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -37,7 +37,7 @@ rand = "0.8" async-trait = "0.1.7" sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } -rmp-serde = "0.15" +rmp-serde = "1.1" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" structopt = { version = "0.3", default-features = false } diff --git a/src/garage/repair.rs b/src/garage/repair.rs index cdcfc90f..1ae26181 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -81,7 +81,7 @@ impl Repair { info!("repair_versions: {}", i); } - let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?; + let version = rmp_serde::decode::from_slice::(&item_bytes)?; if version.deleted.get() { continue; } @@ -133,7 +133,7 @@ impl Repair { info!("repair_block_ref: {}", i); } - let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?; + let block_ref = rmp_serde::decode::from_slice::(&item_bytes)?; if block_ref.deleted.get() { continue; } diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index d908dc01..322feee1 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -31,7 +31,7 @@ tracing = "0.1.30" rand = "0.8" zstd = { version = "0.9", default-features = false } -rmp-serde = "0.15" +rmp-serde = "1.1" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/model/index_counter.rs b/src/model/index_counter.rs index 33de797d..9e343e5f 100644 --- a/src/model/index_counter.rs +++ b/src/model/index_counter.rs @@ -181,11 +181,9 @@ impl IndexCounter { let new_entry = self.local_counter.db().transaction(|tx| { let mut entry = match tx.get(&self.local_counter, &tree_key[..])? { - Some(old_bytes) => { - rmp_serde::decode::from_read_ref::<_, LocalCounterEntry>(&old_bytes) - .map_err(Error::RmpDecode) - .map_err(db::TxError::Abort)? - } + Some(old_bytes) => rmp_serde::decode::from_slice::(&old_bytes) + .map_err(Error::RmpDecode) + .map_err(db::TxError::Abort)?, None => LocalCounterEntry { values: BTreeMap::new(), }, diff --git a/src/model/key_table.rs b/src/model/key_table.rs index 330e83f0..852bf607 100644 --- a/src/model/key_table.rs +++ b/src/model/key_table.rs @@ -175,7 +175,7 @@ impl TableSchema for KeyTable { } fn try_migrate(bytes: &[u8]) -> Option { - let old_k = rmp_serde::decode::from_read_ref::<_, old::Key>(bytes).ok()?; + let old_k = rmp_serde::decode::from_slice::(bytes).ok()?; let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone()); let state = if old_k.deleted.get() { diff --git a/src/model/migrate.rs b/src/model/migrate.rs index 25acb4b0..bfd9845b 100644 --- a/src/model/migrate.rs +++ b/src/model/migrate.rs @@ -28,7 +28,7 @@ impl Migrate { let mut old_buckets = vec![]; for res in tree.iter().map_err(GarageError::from)? { let (_k, v) = res.map_err(GarageError::from)?; - let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..]) + let bucket = rmp_serde::decode::from_slice::(&v[..]) .map_err(GarageError::from)?; old_buckets.push(bucket); } diff --git a/src/model/s3/object_table.rs b/src/model/s3/object_table.rs index 3d9a89f7..c242b325 100644 --- a/src/model/s3/object_table.rs +++ b/src/model/s3/object_table.rs @@ -270,7 +270,7 @@ impl TableSchema for ObjectTable { } fn try_migrate(bytes: &[u8]) -> Option { - let old_obj = rmp_serde::decode::from_read_ref::<_, old::Object>(bytes).ok()?; + let old_obj = rmp_serde::decode::from_slice::(bytes).ok()?; Some(migrate_object(old_obj)) } } diff --git a/src/model/s3/version_table.rs b/src/model/s3/version_table.rs index ad096772..f67f2ff1 100644 --- a/src/model/s3/version_table.rs +++ b/src/model/s3/version_table.rs @@ -168,7 +168,7 @@ impl TableSchema for VersionTable { } fn try_migrate(bytes: &[u8]) -> Option { - let old = rmp_serde::decode::from_read_ref::<_, old::Version>(bytes).ok()?; + let old = rmp_serde::decode::from_slice::(bytes).ok()?; let blocks = old .blocks diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 73328993..93df02a4 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -26,7 +26,7 @@ rand = "0.8" sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } async-trait = "0.1.7" -rmp-serde = "0.15" +rmp-serde = "1.1" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" serde_json = "1.0" diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 6de37cda..690d5a71 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -26,7 +26,7 @@ hexdump = "0.1" tracing = "0.1.30" rand = "0.8" -rmp-serde = "0.15" +rmp-serde = "1.1" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/table/data.rs b/src/table/data.rs index 427ce763..17402bb6 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -297,7 +297,7 @@ where } pub fn decode_entry(&self, bytes: &[u8]) -> Result { - match rmp_serde::decode::from_read_ref::<_, F::E>(bytes) { + match rmp_serde::decode::from_slice::(bytes) { Ok(x) => Ok(x), Err(e) => match F::try_migrate(bytes) { Some(x) => Ok(x), diff --git a/src/table/merkle.rs b/src/table/merkle.rs index c6653a64..8c574d09 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -354,7 +354,7 @@ impl MerkleNode { fn decode_opt(ent: Option>) -> Result { match ent { None => Ok(MerkleNode::Empty), - Some(v) => Ok(rmp_serde::decode::from_read_ref::<_, MerkleNode>(&v[..])?), + Some(v) => Ok(rmp_serde::decode::from_slice::(&v[..])?), } } diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 5d073436..e68704ad 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -25,7 +25,7 @@ rand = "0.8" sha2 = "0.9" chrono = "0.4" -rmp-serde = "0.15" +rmp-serde = "1.1" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_json = "1.0" toml = "0.5" diff --git a/src/util/data.rs b/src/util/data.rs index 7715c2cc..2ef976a5 100644 --- a/src/util/data.rs +++ b/src/util/data.rs @@ -151,7 +151,7 @@ where let mut wr = Vec::with_capacity(128); let mut se = rmp_serde::Serializer::new(&mut wr) .with_struct_map() - .with_string_variants(); + .with_binary(); val.serialize(&mut se)?; Ok(wr) } diff --git a/src/util/persister.rs b/src/util/persister.rs index 9e1a1910..d0759db7 100644 --- a/src/util/persister.rs +++ b/src/util/persister.rs @@ -33,7 +33,7 @@ where let mut bytes = vec![]; file.read_to_end(&mut bytes)?; - let value = rmp_serde::decode::from_read_ref(&bytes[..])?; + let value = rmp_serde::decode::from_slice(&bytes[..])?; Ok(value) } @@ -57,7 +57,7 @@ where let mut bytes = vec![]; file.read_to_end(&mut bytes).await?; - let value = rmp_serde::decode::from_read_ref(&bytes[..])?; + let value = rmp_serde::decode::from_slice(&bytes[..])?; Ok(value) } -- 2.43.0 From 4e72c713f157ae9d5103a461c4c213b2aa6a84b9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 15:31:07 +0200 Subject: [PATCH 23/64] Start LMDB adapter, with fixed semantics --- Cargo.lock | 23 ++++ src/block/rc.rs | 6 +- src/db/Cargo.toml | 3 +- src/db/lib.rs | 38 +++--- src/db/lmdb_adapter.rs | 270 +++++++++++++++++++++++++++++++++++++ src/db/sled_adapter.rs | 22 +-- src/db/sqlite_adapter.rs | 22 +-- src/db/test.rs | 17 ++- src/model/index_counter.rs | 2 +- src/table/data.rs | 37 ++--- src/table/gc.rs | 14 +- src/table/merkle.rs | 24 ++-- 12 files changed, 394 insertions(+), 84 deletions(-) create mode 100644 src/db/lmdb_adapter.rs diff --git a/Cargo.lock b/Cargo.lock index a953779b..b051e72e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1017,6 +1017,7 @@ dependencies = [ "clap 3.1.18", "err-derive 0.3.1", "hexdump", + "lmdb", "log", "mktemp", "rusqlite", @@ -1822,6 +1823,28 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "lmdb" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0908efb5d6496aa977d96f91413da2635a902e5e31dbef0bfb88986c248539" +dependencies = [ + "bitflags", + "libc", + "lmdb-sys", +] + +[[package]] +name = "lmdb-sys" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5b392838cfe8858e86fac37cf97a0e8c55cc60ba0a18365cadc33092f128ce9" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "lock_api" version = "0.4.6" diff --git a/src/block/rc.rs b/src/block/rc.rs index 7d85f67e..e0b952fd 100644 --- a/src/block/rc.rs +++ b/src/block/rc.rs @@ -20,7 +20,7 @@ impl BlockRc { /// Increment the reference counter associated to a hash. /// Returns true if the RC goes from zero to nonzero. pub(crate) fn block_incref(&self, hash: &Hash) -> Result { - let old_rc = self.rc.db().transaction(|tx| { + let old_rc = self.rc.db().transaction(|mut tx| { let old_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?); match old_rc.increment().serialize() { Some(x) => { @@ -36,7 +36,7 @@ impl BlockRc { /// Decrement the reference counter associated to a hash. /// Returns true if the RC is now zero. pub(crate) fn block_decref(&self, hash: &Hash) -> Result { - let new_rc = self.rc.db().transaction(|tx| { + let new_rc = self.rc.db().transaction(|mut tx| { let new_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?).decrement(); match new_rc.serialize() { Some(x) => { @@ -60,7 +60,7 @@ impl BlockRc { /// deletion time has passed pub(crate) fn clear_deleted_block_rc(&self, hash: &Hash) -> Result<(), Error> { let now = now_msec(); - self.rc.db().transaction(|tx| { + self.rc.db().transaction(|mut tx| { let rcval = RcEntry::parse_opt(tx.get(&self.rc, &hash)?); match rcval { RcEntry::Deletable { at_time } if now > at_time => { diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 36b96229..b4601ff7 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -21,8 +21,9 @@ err-derive = "0.3" hexdump = "0.1" log = "0.4" -sled = "0.34" +lmdb = "0.8" rusqlite = "0.27" +sled = "0.34" # cli deps clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } diff --git a/src/db/lib.rs b/src/db/lib.rs index 045c16c5..86042eaf 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -1,3 +1,4 @@ +pub mod lmdb_adapter; pub mod sled_adapter; pub mod sqlite_adapter; @@ -15,8 +16,7 @@ use err_derive::Error; #[derive(Clone)] pub struct Db(pub(crate) Arc); -#[derive(Clone, Copy)] -pub struct Transaction<'a>(pub(crate) &'a dyn ITx<'a>); +pub struct Transaction<'a>(pub(crate) &'a mut dyn ITx); #[derive(Clone)] pub struct Tree(pub(crate) Arc, pub(crate) usize); @@ -271,7 +271,7 @@ impl Tree { #[allow(clippy::len_without_is_empty)] impl<'a> Transaction<'a> { - pub fn get>(&self, tree: &Tree, key: T) -> Result>> { + pub fn get>(&self, tree: &Tree, key: T) -> Result>> { self.0.get(tree.1, key.as_ref()) } pub fn len(&self, tree: &Tree) -> Result { @@ -279,25 +279,25 @@ impl<'a> Transaction<'a> { } pub fn insert, U: AsRef<[u8]>>( - &self, + &mut self, tree: &Tree, key: T, value: U, ) -> Result<()> { self.0.insert(tree.1, key.as_ref(), value.as_ref()) } - pub fn remove>(&self, tree: &Tree, key: T) -> Result { + pub fn remove>(&mut self, tree: &Tree, key: T) -> Result { self.0.remove(tree.1, key.as_ref()) } - pub fn iter(&self, tree: &Tree) -> Result> { + pub fn iter(&self, tree: &Tree) -> Result> { self.0.iter(tree.1) } - pub fn iter_rev(&self, tree: &Tree) -> Result> { + pub fn iter_rev(&self, tree: &Tree) -> Result> { self.0.iter_rev(tree.1) } - pub fn range(&self, tree: &Tree, range: R) -> Result> + pub fn range(&self, tree: &Tree, range: R) -> Result> where K: AsRef<[u8]>, R: RangeBounds, @@ -306,7 +306,7 @@ impl<'a> Transaction<'a> { let eb = range.end_bound(); self.0.range(tree.1, get_bound(sb), get_bound(eb)) } - pub fn range_rev(&self, tree: &Tree, range: R) -> Result> + pub fn range_rev(&self, tree: &Tree, range: R) -> Result> where K: AsRef<[u8]>, R: RangeBounds, @@ -358,32 +358,32 @@ pub(crate) trait IDb: Send + Sync { fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; } -pub(crate) trait ITx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> Result>>; +pub(crate) trait ITx { + fn get(&self, tree: usize, key: &[u8]) -> Result>>; fn len(&self, tree: usize) -> Result; - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; - fn remove(&self, tree: usize, key: &[u8]) -> Result; + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + fn remove(&mut self, tree: usize, key: &[u8]) -> Result; - fn iter(&self, tree: usize) -> Result>; - fn iter_rev(&self, tree: usize) -> Result>; + fn iter(&self, tree: usize) -> Result>; + fn iter_rev(&self, tree: usize) -> Result>; fn range<'r>( &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result>; + ) -> Result>; fn range_rev<'r>( &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result>; + ) -> Result>; } pub(crate) trait ITxFn { - fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult; + fn try_on(&self, tx: &mut dyn ITx) -> TxFnResult; } pub(crate) enum TxFnResult { @@ -404,7 +404,7 @@ impl ITxFn for TxFn where F: Fn(Transaction<'_>) -> TxResult, { - fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult { + fn try_on(&self, tx: &mut dyn ITx) -> TxFnResult { let res = (self.function)(Transaction(tx)); let res2 = match &res { Ok(_) => TxFnResult::Ok, diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs new file mode 100644 index 00000000..caf21517 --- /dev/null +++ b/src/db/lmdb_adapter.rs @@ -0,0 +1,270 @@ +use core::marker::PhantomPinned; +use core::ops::Bound; +use core::pin::Pin; +use core::ptr::NonNull; + +use std::cell::RefCell; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use lmdb::{ + Database, DatabaseFlags, Environment, RoTransaction, RwTransaction, Transaction, WriteFlags, +}; + +use crate::{ + Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter, +}; + +pub use lmdb; + +// -- err + +impl From for Error { + fn from(e: lmdb::Error) -> Error { + Error(format!("LMDB: {}", e).into()) + } +} + +impl From for TxError { + fn from(e: lmdb::Error) -> TxError { + TxError::Db(e.into()) + } +} + +// -- db + +pub struct LmdbDb { + db: lmdb::Environment, + trees: RwLock<(Vec, HashMap)>, +} + +impl LmdbDb { + pub fn init(db: lmdb::Environment) -> Db { + let s = Self { + db, + trees: RwLock::new((Vec::new(), HashMap::new())), + }; + Db(Arc::new(s)) + } + + fn get_tree(&self, i: usize) -> Result { + self.trees + .read() + .unwrap() + .0 + .get(i) + .cloned() + .ok_or_else(|| Error("invalid tree id".into())) + } +} + +impl IDb for LmdbDb { + fn open_tree(&self, name: &str) -> Result { + let mut trees = self.trees.write().unwrap(); + if let Some(i) = trees.1.get(name) { + Ok(*i) + } else { + let tree = self.db.create_db(Some(name), DatabaseFlags::empty())?; + let i = trees.0.len(); + trees.0.push(tree); + trees.1.insert(name.to_string(), i); + Ok(i) + } + } + + fn list_trees(&self) -> Result> { + unimplemented!() + } + + // ---- + + fn get(&self, tree: usize, key: &[u8]) -> Result>> { + let tree = self.get_tree(tree)?; + + let res = TxAndValue { + tx: self.db.begin_ro_txn()?, + value: NonNull::dangling(), + _pin: PhantomPinned, + }; + let mut boxed = Box::pin(res); + + unsafe { + let tx = NonNull::from(&boxed.tx); + let val = match tx.as_ref().get(tree, &key) { + Err(lmdb::Error::NotFound) => return Ok(None), + v => v?, + }; + + let mut_ref: Pin<&mut TxAndValue<'_>> = Pin::as_mut(&mut boxed); + Pin::get_unchecked_mut(mut_ref).value = NonNull::from(&val); + } + + Ok(Some(Value(Box::new(TxAndValuePin(boxed))))) + } + + fn remove(&self, tree: usize, key: &[u8]) -> Result { + unimplemented!() + } + + fn len(&self, tree: usize) -> Result { + unimplemented!() + } + + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.get_tree(tree)?; + let mut tx = self.db.begin_rw_txn()?; + tx.put(tree, &key, &value, WriteFlags::empty())?; + tx.commit()?; + Ok(()) + } + + fn iter(&self, tree: usize) -> Result> { + unimplemented!() + } + + fn iter_rev(&self, tree: usize) -> Result> { + unimplemented!() + } + + fn range<'r>( + &self, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result> { + unimplemented!() + } + fn range_rev<'r>( + &self, + tree: usize, + low: Bound<&'r [u8]>, + high: Bound<&'r [u8]>, + ) -> Result> { + unimplemented!() + } + + // ---- + + fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { + let trees = self.trees.read().unwrap(); + let mut tx = LmdbTx { + trees: &trees.0[..], + tx: self.db.begin_rw_txn()?, + }; + + let res = f.try_on(&mut tx); + match res { + TxFnResult::Ok => { + tx.tx.commit()?; + Ok(()) + } + TxFnResult::Abort => { + tx.tx.abort(); + Err(TxError::Abort(())) + } + TxFnResult::DbErr => { + tx.tx.abort(); + Err(TxError::Db(Error( + "(this message will be discarded)".into(), + ))) + } + } + } +} + +// ---- + +struct LmdbTx<'a, 'db> { + trees: &'db [Database], + tx: RwTransaction<'a>, +} + +impl<'a, 'db> LmdbTx<'a, 'db> { + fn get_tree(&self, i: usize) -> Result<&Database> { + self.trees.get(i).ok_or_else(|| { + Error( + "invalid tree id (it might have been openned after the transaction started)".into(), + ) + }) + } +} + +impl<'a, 'db> ITx for LmdbTx<'a, 'db> { + fn get(&self, tree: usize, key: &[u8]) -> Result>> { + let tree = self.get_tree(tree)?; + match self.tx.get::<'a, _>(*tree, &key) { + Err(lmdb::Error::NotFound) => Ok(None), + Err(e) => Err(e.into()), + Ok(v) => Ok(Some(Value(Box::new(v)))), + } + } + fn len(&self, _tree: usize) -> Result { + unimplemented!(".len() in transaction not supported with LMDB backend") + } + + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + let tree = self.get_tree(tree)?; + self.tx.put(*tree, &key, &value, WriteFlags::empty())?; + Ok(()) + } + fn remove(&mut self, tree: usize, key: &[u8]) -> Result { + let tree = self.get_tree(tree)?; + match self.tx.del::<'a, _>(*tree, &key, None) { + Ok(()) => Ok(true), + Err(lmdb::Error::NotFound) => Ok(false), + Err(e) => Err(e.into()), + } + } + + fn iter(&self, _tree: usize) -> Result> { + unimplemented!("Iterators in transactions not supported with LMDB backend"); + } + fn iter_rev(&self, _tree: usize) -> Result> { + unimplemented!("Iterators in transactions not supported with LMDB backend"); + } + + fn range<'r>( + &self, + _tree: usize, + _low: Bound<&'r [u8]>, + _high: Bound<&'r [u8]>, + ) -> Result> { + unimplemented!("Iterators in transactions not supported with LMDB backend"); + } + fn range_rev<'r>( + &self, + _tree: usize, + _low: Bound<&'r [u8]>, + _high: Bound<&'r [u8]>, + ) -> Result> { + unimplemented!("Iterators in transactions not supported with LMDB backend"); + } +} + +// ---- + +struct TxAndValue<'a> { + tx: RoTransaction<'a>, + value: NonNull<&'a [u8]>, + _pin: PhantomPinned, +} + +struct TxAndValuePin<'a>(Pin>>); + +impl<'a> IValue<'a> for TxAndValuePin<'a> { + fn take_maybe(&mut self) -> Vec { + self.as_ref().to_vec() + } +} + +impl<'a> AsRef<[u8]> for TxAndValuePin<'a> { + fn as_ref(&self) -> &[u8] { + unsafe { self.0.value.as_ref() } + } +} + +impl<'a> std::borrow::Borrow<[u8]> for TxAndValuePin<'a> { + fn borrow(&self) -> &[u8] { + self.as_ref() + } +} diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 3388b0ca..2953785e 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -19,7 +19,7 @@ pub use sled; impl From for Error { fn from(e: sled::Error) -> Error { - Error(format!("{}", e).into()) + Error(format!("Sled: {}", e).into()) } } @@ -162,11 +162,11 @@ impl IDb for SledDb { fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { let trees = self.trees.read().unwrap(); let res = trees.0.transaction(|txtrees| { - let tx = SledTx { + let mut tx = SledTx { trees: txtrees, err: Cell::new(None), }; - match f.try_on(&tx) { + match f.try_on(&mut tx) { TxFnResult::Ok => { assert!(tx.err.into_inner().is_none()); Ok(()) @@ -217,8 +217,8 @@ impl<'a> SledTx<'a> { } } -impl<'a> ITx<'a> for SledTx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> Result>> { +impl<'a> ITx for SledTx<'a> { + fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; let tmp = self.save_error(tree.get(key))?; Ok(tmp.map(From::from)) @@ -227,20 +227,20 @@ impl<'a> ITx<'a> for SledTx<'a> { unimplemented!(".len() in transaction not supported with Sled backend") } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; self.save_error(tree.insert(key, value))?; Ok(()) } - fn remove(&self, tree: usize, key: &[u8]) -> Result { + fn remove(&mut self, tree: usize, key: &[u8]) -> Result { let tree = self.get_tree(tree)?; Ok(self.save_error(tree.remove(key))?.is_some()) } - fn iter(&self, _tree: usize) -> Result> { + fn iter(&self, _tree: usize) -> Result> { unimplemented!("Iterators in transactions not supported with Sled backend"); } - fn iter_rev(&self, _tree: usize) -> Result> { + fn iter_rev(&self, _tree: usize) -> Result> { unimplemented!("Iterators in transactions not supported with Sled backend"); } @@ -249,7 +249,7 @@ impl<'a> ITx<'a> for SledTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> Result> { unimplemented!("Iterators in transactions not supported with Sled backend"); } fn range_rev<'r>( @@ -257,7 +257,7 @@ impl<'a> ITx<'a> for SledTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> Result> { unimplemented!("Iterators in transactions not supported with Sled backend"); } } diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 4f79b34b..9f2bf919 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -17,7 +17,7 @@ pub use rusqlite; impl From for Error { fn from(e: rusqlite::Error) -> Error { - Error(format!("{}", e).into()) + Error(format!("Sqlite: {}", e).into()) } } @@ -235,11 +235,11 @@ impl IDb for SqliteDb { let mut db = self.db.lock().unwrap(); trace!("transaction: lock acquired"); - let tx = SqliteTx { + let mut tx = SqliteTx { tx: db.transaction()?, trees: trees.as_ref(), }; - let res = match f.try_on(&tx) { + let res = match f.try_on(&mut tx) { TxFnResult::Ok => { tx.tx.commit()?; Ok(()) @@ -278,8 +278,8 @@ impl<'a> SqliteTx<'a> { } } -impl<'a> ITx<'a> for SqliteTx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> Result>> { +impl<'a> ITx for SqliteTx<'a> { + fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; let mut stmt = self .tx @@ -300,7 +300,7 @@ impl<'a> ITx<'a> for SqliteTx<'a> { } } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; self.tx.execute( &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), @@ -308,7 +308,7 @@ impl<'a> ITx<'a> for SqliteTx<'a> { )?; Ok(()) } - fn remove(&self, tree: usize, key: &[u8]) -> Result { + fn remove(&mut self, tree: usize, key: &[u8]) -> Result { let tree = self.get_tree(tree)?; let res = self .tx @@ -316,10 +316,10 @@ impl<'a> ITx<'a> for SqliteTx<'a> { Ok(res > 0) } - fn iter(&self, _tree: usize) -> Result> { + fn iter(&self, _tree: usize) -> Result> { unimplemented!(); } - fn iter_rev(&self, _tree: usize) -> Result> { + fn iter_rev(&self, _tree: usize) -> Result> { unimplemented!(); } @@ -328,7 +328,7 @@ impl<'a> ITx<'a> for SqliteTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> Result> { unimplemented!(); } fn range_rev<'r>( @@ -336,7 +336,7 @@ impl<'a> ITx<'a> for SqliteTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> Result> { unimplemented!(); } } diff --git a/src/db/test.rs b/src/db/test.rs index 75200cf1..e5b83ab5 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -1,5 +1,6 @@ use crate::*; +use crate::lmdb_adapter::LmdbDb; use crate::sled_adapter::SledDb; use crate::sqlite_adapter::SqliteDb; @@ -16,7 +17,7 @@ fn test_suite(db: Db) { tree.insert(ka, va).unwrap(); assert_eq!(tree.get(ka).unwrap().unwrap(), va); - let res = db.transaction::<_, (), _>(|tx| { + let res = db.transaction::<_, (), _>(|mut tx| { assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), va); tx.insert(&tree, ka, vb).unwrap(); @@ -28,7 +29,7 @@ fn test_suite(db: Db) { assert!(matches!(res, Ok(12))); assert_eq!(tree.get(ka).unwrap().unwrap(), vb); - let res = db.transaction::<(), _, _>(|tx| { + let res = db.transaction::<(), _, _>(|mut tx| { assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); tx.insert(&tree, ka, vc).unwrap(); @@ -78,6 +79,18 @@ fn test_suite(db: Db) { drop(iter); } +#[test] +fn test_lmdb_db() { + let path = mktemp::Temp::new_dir().unwrap(); + let db = lmdb::Environment::new() + .set_max_dbs(100) + .open(&path) + .unwrap(); + let db = LmdbDb::init(db); + test_suite(db); + drop(path); +} + #[test] fn test_sled_db() { let path = mktemp::Temp::new_dir().unwrap(); diff --git a/src/model/index_counter.rs b/src/model/index_counter.rs index 9e343e5f..d8c1229a 100644 --- a/src/model/index_counter.rs +++ b/src/model/index_counter.rs @@ -179,7 +179,7 @@ impl IndexCounter { pub fn count(&self, pk: &T::P, sk: &T::S, counts: &[(&str, i64)]) -> Result<(), Error> { let tree_key = self.table.data.tree_key(pk, sk); - let new_entry = self.local_counter.db().transaction(|tx| { + let new_entry = self.local_counter.db().transaction(|mut tx| { let mut entry = match tx.get(&self.local_counter, &tree_key[..])? { Some(old_bytes) => rmp_serde::decode::from_slice::(&old_bytes) .map_err(Error::RmpDecode) diff --git a/src/table/data.rs b/src/table/data.rs index 17402bb6..cca96f68 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -182,7 +182,7 @@ where tree_key: &[u8], f: impl Fn(Option) -> F::E, ) -> Result, Error> { - let changed = self.store.db().transaction(|tx| { + let changed = self.store.db().transaction(|mut tx| { let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, tree_key)? { Some(old_bytes) => { let old_entry = self.decode_entry(&old_bytes).map_err(db::TxError::Abort)?; @@ -203,6 +203,7 @@ where .map_err(Error::RmpEncode) .map_err(db::TxError::Abort)?; let encoding_changed = Some(&new_bytes[..]) != old_bytes.as_ref().map(|x| &x[..]); + drop(old_bytes); if value_changed || encoding_changed { let new_bytes_hash = blake2sum(&new_bytes[..]); @@ -241,15 +242,16 @@ where } pub(crate) fn delete_if_equal(self: &Arc, k: &[u8], v: &[u8]) -> Result { - let removed = self.store.db().transaction(|tx| { - if let Some(cur_v) = tx.get(&self.store, k)? { - if cur_v == v { - tx.remove(&self.store, k)?; - tx.insert(&self.merkle_todo, k, vec![])?; - return Ok(true); - } + let removed = self.store.db().transaction(|mut tx| { + let remove = match tx.get(&self.store, k)? { + Some(cur_v) if cur_v == v => true, + _ => false, + }; + if remove { + tx.remove(&self.store, k)?; + tx.insert(&self.merkle_todo, k, vec![])?; } - Ok(false) + Ok(remove) })?; if removed { @@ -267,15 +269,16 @@ where k: &[u8], vhash: Hash, ) -> Result { - let removed = self.store.db().transaction(|tx| { - if let Some(cur_v) = tx.get(&self.store, k)? { - if blake2sum(&cur_v[..]) == vhash { - tx.remove(&self.store, k)?; - tx.insert(&self.merkle_todo, k, vec![])?; - return Ok(Some(cur_v.into_vec())); - } + let removed = self.store.db().transaction(|mut tx| { + let remove_v = match tx.get(&self.store, k)? { + Some(cur_v) if blake2sum(&cur_v[..]) == vhash => Some(cur_v.into_vec()), + _ => None, + }; + if remove_v.is_some() { + tx.remove(&self.store, k)?; + tx.insert(&self.merkle_todo, k, vec![])?; } - Ok(None) + Ok(remove_v) })?; if let Some(old_v) = removed { diff --git a/src/table/gc.rs b/src/table/gc.rs index 260fecfa..e2611389 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -376,13 +376,13 @@ impl GcTodoEntry { /// what we have to do is still the same pub(crate) fn remove_if_equal(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { let key = self.todo_table_key(); - gc_todo_tree.db().transaction(|tx| { - let old_val = tx.get(gc_todo_tree, &key)?; - match old_val { - Some(ov) if ov == self.value_hash.as_slice() => { - tx.remove(gc_todo_tree, &key)?; - } - _ => (), + gc_todo_tree.db().transaction(|mut tx| { + let remove = match tx.get(gc_todo_tree, &key)? { + Some(ov) if ov == self.value_hash.as_slice() => true, + _ => false, + }; + if remove { + tx.remove(gc_todo_tree, &key)?; } tx.commit(()) })?; diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 8c574d09..92e1445b 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -137,17 +137,17 @@ where self.data .merkle_tree .db() - .transaction(|tx| self.update_item_rec(tx, k, &khash, &key, new_vhash))?; + .transaction(|mut tx| self.update_item_rec(&mut tx, k, &khash, &key, new_vhash))?; - let deleted = self.data.merkle_todo.db().transaction(|tx| { - let old_val = tx.get(&self.data.merkle_todo, k)?; - match old_val { - Some(ov) if ov == vhash_by => { - tx.remove(&self.data.merkle_todo, k)?; - tx.commit(true) - } - _ => tx.commit(false), + let deleted = self.data.merkle_todo.db().transaction(|mut tx| { + let remove = match tx.get(&self.data.merkle_todo, k)? { + Some(ov) if ov == vhash_by => true, + _ => false, + }; + if remove { + tx.remove(&self.data.merkle_todo, k)?; } + Ok(remove) })?; if !deleted { @@ -162,7 +162,7 @@ where fn update_item_rec( &self, - tx: db::Transaction<'_>, + tx: &mut db::Transaction<'_>, k: &[u8], khash: &Hash, key: &MerkleNodeKey, @@ -288,7 +288,7 @@ where fn read_node_txn( &self, - tx: db::Transaction<'_>, + tx: &mut db::Transaction<'_>, k: &MerkleNodeKey, ) -> db::TxResult { let ent = tx.get(&self.data.merkle_tree, k.encode())?; @@ -297,7 +297,7 @@ where fn put_node_txn( &self, - tx: db::Transaction<'_>, + tx: &mut db::Transaction<'_>, k: &MerkleNodeKey, v: &MerkleNode, ) -> db::TxResult { -- 2.43.0 From bd2997a4534a4d06bc0cc636a2c0b50fe793057a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 15:53:20 +0200 Subject: [PATCH 24/64] LMDB: use Heed wrapper (much more complete) --- Cargo.lock | 161 +++++++++++++++++++++++++++++++---------- src/db/Cargo.toml | 2 +- src/db/lmdb_adapter.rs | 66 ++++++++--------- src/db/test.rs | 4 +- 4 files changed, 157 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b051e72e..38e990fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -312,6 +312,15 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -344,6 +353,12 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +[[package]] +name = "bytemuck" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc" + [[package]] name = "byteorder" version = "1.4.3" @@ -381,6 +396,12 @@ dependencies = [ "jobserver", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -497,7 +518,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -506,8 +527,8 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.8", ] [[package]] @@ -517,20 +538,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" dependencies = [ "autocfg", - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.8", "lazy_static", "memoffset", "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +dependencies = [ + "crossbeam-utils 0.6.6", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", +] + [[package]] name = "crossbeam-utils" version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "lazy_static", ] @@ -614,7 +654,7 @@ version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "num_cpus", ] @@ -644,7 +684,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dirs-sys-next", ] @@ -683,7 +723,7 @@ version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1016,8 +1056,8 @@ version = "0.8.0" dependencies = [ "clap 3.1.18", "err-derive 0.3.1", + "heed", "hexdump", - "lmdb", "log", "mktemp", "rusqlite", @@ -1274,7 +1314,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.10.0+wasi-snapshot-preview1", ] @@ -1353,6 +1393,45 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +[[package]] +name = "heed" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392" +dependencies = [ + "bytemuck", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-rkv-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2" + +[[package]] +name = "heed-types" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd" +dependencies = [ + "bincode", + "bytemuck", + "byteorder", + "heed-traits", + "serde", + "serde_json", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1552,7 +1631,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1824,21 +1903,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] -name = "lmdb" -version = "0.8.0" +name = "lmdb-rkv-sys" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0908efb5d6496aa977d96f91413da2635a902e5e31dbef0bfb88986c248539" -dependencies = [ - "bitflags", - "libc", - "lmdb-sys", -] - -[[package]] -name = "lmdb-sys" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5b392838cfe8858e86fac37cf97a0e8c55cc60ba0a18365cadc33092f128ce9" +checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe" dependencies = [ "cc", "libc", @@ -1860,7 +1928,7 @@ version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -2018,7 +2086,7 @@ dependencies = [ "arc-swap", "async-trait", "bytes 0.6.0", - "cfg-if", + "cfg-if 1.0.0", "err-derive 0.2.4", "futures", "hex", @@ -2111,7 +2179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -2224,6 +2292,16 @@ version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435" +[[package]] +name = "page_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -2251,7 +2329,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", @@ -2265,7 +2343,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", @@ -2453,7 +2531,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fnv", "lazy_static", "memchr", @@ -3017,7 +3095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", "opaque-debug", @@ -3052,7 +3130,7 @@ checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" dependencies = [ "crc32fast", "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-utils 0.8.8", "fs2", "fxhash", "libc", @@ -3186,6 +3264,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synchronoise" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb" +dependencies = [ + "crossbeam-queue", +] + [[package]] name = "synstructure" version = "0.12.6" @@ -3204,7 +3291,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "libc", "redox_syscall", @@ -3503,7 +3590,7 @@ version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -3672,7 +3759,7 @@ version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index b4601ff7..ce9c9373 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -21,7 +21,7 @@ err-derive = "0.3" hexdump = "0.1" log = "0.4" -lmdb = "0.8" +heed = "0.11" rusqlite = "0.27" sled = "0.34" diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index caf21517..ec1a5444 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -7,26 +7,24 @@ use std::cell::RefCell; use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use lmdb::{ - Database, DatabaseFlags, Environment, RoTransaction, RwTransaction, Transaction, WriteFlags, -}; +use heed::{Env, RoTxn, RwTxn, UntypedDatabase as Database}; use crate::{ Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter, }; -pub use lmdb; +pub use heed; // -- err -impl From for Error { - fn from(e: lmdb::Error) -> Error { +impl From for Error { + fn from(e: heed::Error) -> Error { Error(format!("LMDB: {}", e).into()) } } -impl From for TxError { - fn from(e: lmdb::Error) -> TxError { +impl From for TxError { + fn from(e: heed::Error) -> TxError { TxError::Db(e.into()) } } @@ -34,12 +32,12 @@ impl From for TxError { // -- db pub struct LmdbDb { - db: lmdb::Environment, - trees: RwLock<(Vec, HashMap)>, + db: heed::Env, + trees: RwLock<(Vec, HashMap)>, } impl LmdbDb { - pub fn init(db: lmdb::Environment) -> Db { + pub fn init(db: Env) -> Db { let s = Self { db, trees: RwLock::new((Vec::new(), HashMap::new())), @@ -47,7 +45,7 @@ impl LmdbDb { Db(Arc::new(s)) } - fn get_tree(&self, i: usize) -> Result { + fn get_tree(&self, i: usize) -> Result { self.trees .read() .unwrap() @@ -64,7 +62,7 @@ impl IDb for LmdbDb { if let Some(i) = trees.1.get(name) { Ok(*i) } else { - let tree = self.db.create_db(Some(name), DatabaseFlags::empty())?; + let tree = self.db.create_database(Some(name))?; let i = trees.0.len(); trees.0.push(tree); trees.1.insert(name.to_string(), i); @@ -82,7 +80,7 @@ impl IDb for LmdbDb { let tree = self.get_tree(tree)?; let res = TxAndValue { - tx: self.db.begin_ro_txn()?, + tx: self.db.read_txn()?, value: NonNull::dangling(), _pin: PhantomPinned, }; @@ -90,9 +88,9 @@ impl IDb for LmdbDb { unsafe { let tx = NonNull::from(&boxed.tx); - let val = match tx.as_ref().get(tree, &key) { - Err(lmdb::Error::NotFound) => return Ok(None), - v => v?, + let val = match tree.get(tx.as_ref(), &key)? { + None => return Ok(None), + Some(v) => v, }; let mut_ref: Pin<&mut TxAndValue<'_>> = Pin::as_mut(&mut boxed); @@ -112,8 +110,8 @@ impl IDb for LmdbDb { fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; - let mut tx = self.db.begin_rw_txn()?; - tx.put(tree, &key, &value, WriteFlags::empty())?; + let mut tx = self.db.write_txn()?; + tree.put(&mut tx, &key, &value)?; tx.commit()?; Ok(()) } @@ -149,7 +147,7 @@ impl IDb for LmdbDb { let trees = self.trees.read().unwrap(); let mut tx = LmdbTx { trees: &trees.0[..], - tx: self.db.begin_rw_txn()?, + tx: self.db.write_txn()?, }; let res = f.try_on(&mut tx); @@ -159,11 +157,11 @@ impl IDb for LmdbDb { Ok(()) } TxFnResult::Abort => { - tx.tx.abort(); + tx.tx.abort()?; Err(TxError::Abort(())) } TxFnResult::DbErr => { - tx.tx.abort(); + tx.tx.abort()?; Err(TxError::Db(Error( "(this message will be discarded)".into(), ))) @@ -176,7 +174,7 @@ impl IDb for LmdbDb { struct LmdbTx<'a, 'db> { trees: &'db [Database], - tx: RwTransaction<'a>, + tx: RwTxn<'a, 'a>, } impl<'a, 'db> LmdbTx<'a, 'db> { @@ -192,10 +190,9 @@ impl<'a, 'db> LmdbTx<'a, 'db> { impl<'a, 'db> ITx for LmdbTx<'a, 'db> { fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; - match self.tx.get::<'a, _>(*tree, &key) { - Err(lmdb::Error::NotFound) => Ok(None), - Err(e) => Err(e.into()), - Ok(v) => Ok(Some(Value(Box::new(v)))), + match tree.get(&self.tx, &key)? { + Some(v) => Ok(Some(Value(Box::new(v)))), + None => Ok(None), } } fn len(&self, _tree: usize) -> Result { @@ -203,17 +200,14 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { } fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { - let tree = self.get_tree(tree)?; - self.tx.put(*tree, &key, &value, WriteFlags::empty())?; + let tree = *self.get_tree(tree)?; + tree.put(&mut self.tx, &key, &value)?; Ok(()) } fn remove(&mut self, tree: usize, key: &[u8]) -> Result { - let tree = self.get_tree(tree)?; - match self.tx.del::<'a, _>(*tree, &key, None) { - Ok(()) => Ok(true), - Err(lmdb::Error::NotFound) => Ok(false), - Err(e) => Err(e.into()), - } + let tree = *self.get_tree(tree)?; + let deleted = tree.delete(&mut self.tx, &key)?; + Ok(deleted) } fn iter(&self, _tree: usize) -> Result> { @@ -244,7 +238,7 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { // ---- struct TxAndValue<'a> { - tx: RoTransaction<'a>, + tx: RoTxn<'a>, value: NonNull<&'a [u8]>, _pin: PhantomPinned, } diff --git a/src/db/test.rs b/src/db/test.rs index e5b83ab5..31fea1f0 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -82,8 +82,8 @@ fn test_suite(db: Db) { #[test] fn test_lmdb_db() { let path = mktemp::Temp::new_dir().unwrap(); - let db = lmdb::Environment::new() - .set_max_dbs(100) + let db = heed::EnvOpenOptions::new() + .max_dbs(100) .open(&path) .unwrap(); let db = LmdbDb::init(db); -- 2.43.0 From f7a1c70089bf453554f5ae787da4865caf0ee5c4 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 16:18:56 +0200 Subject: [PATCH 25/64] Implement iterator for LMDB --- src/db/lmdb_adapter.rs | 106 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 8 deletions(-) diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index ec1a5444..d568b5c5 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -3,11 +3,12 @@ use core::ops::Bound; use core::pin::Pin; use core::ptr::NonNull; -use std::cell::RefCell; use std::collections::HashMap; +use std::convert::TryInto; use std::sync::{Arc, RwLock}; -use heed::{Env, RoTxn, RwTxn, UntypedDatabase as Database}; +use heed::types::ByteSlice; +use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; use crate::{ Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter, @@ -101,11 +102,17 @@ impl IDb for LmdbDb { } fn remove(&self, tree: usize, key: &[u8]) -> Result { - unimplemented!() + let tree = self.get_tree(tree)?; + let mut tx = self.db.write_txn()?; + let deleted = tree.delete(&mut tx, &key)?; + tx.commit()?; + Ok(deleted) } fn len(&self, tree: usize) -> Result { - unimplemented!() + let tree = self.get_tree(tree)?; + let tx = self.db.read_txn()?; + Ok(tree.len(&tx)?.try_into().unwrap()) } fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { @@ -117,11 +124,15 @@ impl IDb for LmdbDb { } fn iter(&self, tree: usize) -> Result> { - unimplemented!() + let tree = self.get_tree(tree)?; + let tx = self.db.read_txn()?; + TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) } fn iter_rev(&self, tree: usize) -> Result> { - unimplemented!() + let tree = self.get_tree(tree)?; + let tx = self.db.read_txn()?; + TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) } fn range<'r>( @@ -130,7 +141,9 @@ impl IDb for LmdbDb { low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, ) -> Result> { - unimplemented!() + let tree = self.get_tree(tree)?; + let tx = self.db.read_txn()?; + TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) } fn range_rev<'r>( &self, @@ -138,7 +151,9 @@ impl IDb for LmdbDb { low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, ) -> Result> { - unimplemented!() + let tree = self.get_tree(tree)?; + let tx = self.db.read_txn()?; + TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) } // ---- @@ -262,3 +277,78 @@ impl<'a> std::borrow::Borrow<[u8]> for TxAndValuePin<'a> { self.as_ref() } } + +// ---- + +type IteratorItem<'a> = heed::Result<( + >::DItem, + >::DItem, +)>; + +struct TxAndIterator<'a, I> +where + I: Iterator> + 'a, +{ + tx: RoTxn<'a>, + iter: Option, + _pin: PhantomPinned, +} + +impl<'a, I> TxAndIterator<'a, I> +where + I: Iterator> + 'a, +{ + fn make(tx: RoTxn<'a>, iterfun: F) -> Result> + where + F: FnOnce(&'a RoTxn<'a>) -> Result, + { + let res = TxAndIterator { + tx, + iter: None, + _pin: PhantomPinned, + }; + let mut boxed = Box::pin(res); + + unsafe { + let tx = NonNull::from(&boxed.tx); + let iter = iterfun(tx.as_ref())?; + + let mut_ref: Pin<&mut TxAndIterator<'a, I>> = Pin::as_mut(&mut boxed); + Pin::get_unchecked_mut(mut_ref).iter = Some(iter); + } + + Ok(Box::new(TxAndIteratorPin(boxed))) + } +} + +impl<'a, I> Drop for TxAndIterator<'a, I> +where + I: Iterator> + 'a, +{ + fn drop(&mut self) { + drop(self.iter.take()); + } +} + +struct TxAndIteratorPin<'a, I: Iterator> + 'a>( + Pin>>, +); + +impl<'a, I> Iterator for TxAndIteratorPin<'a, I> +where + I: Iterator> + 'a, +{ + type Item = Result<(Value<'a>, Value<'a>)>; + + fn next(&mut self) -> Option { + let iter_ref = unsafe { + let mut_ref: Pin<&mut TxAndIterator<'a, I>> = Pin::as_mut(&mut self.0); + Pin::get_unchecked_mut(mut_ref).iter.as_mut() + }; + match iter_ref.unwrap().next() { + None => None, + Some(Err(e)) => Some(Err(e.into())), + Some(Ok((k, v))) => Some(Ok((k.into(), v.into()))), + } + } +} -- 2.43.0 From 2d5541b0e7ed0939e99710f40d3859c05bc31175 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 16:40:38 +0200 Subject: [PATCH 26/64] Integrate LMDB with rest --- src/db/bin/convert.rs | 11 +++++++++++ src/db/lmdb_adapter.rs | 26 +++++++++++++++++++++++++- src/garage/server.rs | 22 +++++++++++++++++++++- src/util/config.rs | 2 +- 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs index 7525bcc9..5dd9e034 100644 --- a/src/db/bin/convert.rs +++ b/src/db/bin/convert.rs @@ -48,6 +48,17 @@ fn open_db(path: PathBuf, engine: String) -> Result { let db = sqlite_adapter::rusqlite::Connection::open(&path)?; Ok(sqlite_adapter::SqliteDb::init(db)) } + "lmdb" | "heed" => { + std::fs::create_dir_all(&path).map_err(|e| { + Error(format!("Unable to create LMDB data directory: {}", e).into()) + })?; + let db = lmdb_adapter::heed::EnvOpenOptions::new() + .max_dbs(100) + .map_size(1usize << 30) + .open(&path) + .unwrap(); + Ok(lmdb_adapter::LmdbDb::init(db)) + } e => Err(Error(format!("Invalid DB engine: {}", e).into())), } } diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index d568b5c5..095f512b 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -72,7 +72,31 @@ impl IDb for LmdbDb { } fn list_trees(&self) -> Result> { - unimplemented!() + let tree0 = match self.db.open_database::(None)? { + Some(x) => x, + None => return Ok(vec![]), + }; + + let mut ret = vec![]; + let tx = self.db.read_txn()?; + for item in tree0.iter(&tx)? { + let (tree_name, _) = item?; + ret.push(tree_name.to_string()); + } + drop(tx); + + let mut ret2 = vec![]; + for tree_name in ret { + if self + .db + .open_database::(Some(&tree_name))? + .is_some() + { + ret2.push(tree_name); + } + } + + Ok(ret2) } // ---- diff --git a/src/garage/server.rs b/src/garage/server.rs index a56f124a..9a1aa975 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -36,6 +36,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { let db = match config.db_engine.as_str() { "sled" => { db_path.push("db"); + info!("Opening Sled database at: {}", db_path.display()); let db = db::sled_adapter::sled::Config::default() .path(&db_path) .cache_capacity(config.sled_cache_capacity) @@ -46,13 +47,32 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { } "sqlite" | "sqlite3" | "rusqlite" => { db_path.push("db.sqlite"); + info!("Opening Sqlite database at: {}", db_path.display()); let db = db::sqlite_adapter::rusqlite::Connection::open(db_path) .expect("Unable to open sqlite DB"); db::sqlite_adapter::SqliteDb::init(db) } + "lmdb" | "heed" => { + db_path.push("db.lmdb"); + info!("Opening LMDB database at: {}", db_path.display()); + std::fs::create_dir_all(&db_path).expect("Unable to create LMDB data directory"); + let map_size = if u32::MAX as usize == usize::MAX { + warn!("LMDB is not recommended on 32-bit systems, database size will be limited"); + 1usize << 30 // 1GB for 32-bit systems + } else { + 1usize << 40 // 1TB for 64-bit systems + }; + + let db = db::lmdb_adapter::heed::EnvOpenOptions::new() + .max_dbs(100) + .map_size(map_size) + .open(&db_path) + .expect("Unable to open LMDB DB"); + db::lmdb_adapter::LmdbDb::init(db) + } e => { return Err(Error::Message(format!( - "Unsupported DB engine: {} (options: sled, sqlite)", + "Unsupported DB engine: {} (options: sled, sqlite, lmdb)", e ))); } diff --git a/src/util/config.rs b/src/util/config.rs index 3b37adbb..e8ef4fdd 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -65,7 +65,7 @@ pub struct Config { pub kubernetes_skip_crd: bool, // -- DB - /// Database engine to use for metadata (options: sled, sqlite) + /// Database engine to use for metadata (options: sled, sqlite, lmdb) #[serde(default = "default_db_engine")] pub db_engine: String, -- 2.43.0 From 4f5d17d46464b09a9e238bf959d14f3127a129cc Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 11:30:10 +0200 Subject: [PATCH 27/64] fixes --- Cargo.lock | 1 + src/db/Cargo.toml | 3 ++- src/db/bin/convert.rs | 2 ++ src/db/lib.rs | 27 ++++++++++++++++++--------- src/db/lmdb_adapter.rs | 11 +++++++++-- src/db/sqlite_adapter.rs | 1 + 6 files changed, 33 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38e990fe..51fa2a95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1060,6 +1060,7 @@ dependencies = [ "hexdump", "log", "mktemp", + "pretty_env_logger", "rusqlite", "sled", ] diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index ce9c9373..1a24979f 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -27,9 +27,10 @@ sled = "0.34" # cli deps clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } +pretty_env_logger = { version = "0.4", optional = true } [dev-dependencies] mktemp = "0.4" [features] -cli = ["clap"] +cli = ["clap", "pretty_env_logger"] diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs index 5dd9e034..636bd834 100644 --- a/src/db/bin/convert.rs +++ b/src/db/bin/convert.rs @@ -25,6 +25,8 @@ struct Args { fn main() { let args = Args::parse(); + pretty_env_logger::init(); + match do_conversion(args) { Ok(()) => println!("Success!"), Err(e) => eprintln!("Error: {}", e), diff --git a/src/db/lib.rs b/src/db/lib.rs index 86042eaf..2e847f7a 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -198,16 +198,25 @@ impl Db { let ex_tree = other.open_tree(&name)?; - let mut i = 0; - for item in ex_tree.iter()? { - let (k, v) = item?; - tree.insert(k, v)?; - i += 1; - if i % 1000 == 0 { - println!("{}: imported {}", name, i); + let tx_res = self.transaction(|mut tx| { + let mut i = 0; + for item in ex_tree.iter()? { + let (k, v) = item?; + tx.insert(&tree, k, v)?; + i += 1; + if i % 1000 == 0 { + println!("{}: imported {}", name, i); + } } - } - println!("{}: finished importing, {} items", name, i); + Ok::<_, TxError<()>>(i) + }); + let total = match tx_res { + Err(TxError::Db(e)) => return Err(e), + Err(TxError::Abort(_)) => unreachable!(), + Ok(x) => x, + }; + + println!("{}: finished importing, {} items", name, total); } Ok(()) } diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 095f512b..aa365733 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -230,7 +230,10 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; match tree.get(&self.tx, &key)? { - Some(v) => Ok(Some(Value(Box::new(v)))), + Some(v) => { + let v: &'_ [u8] = v; + Ok(Some(v.into())) + } None => Ok(None), } } @@ -372,7 +375,11 @@ where match iter_ref.unwrap().next() { None => None, Some(Err(e)) => Some(Err(e.into())), - Some(Ok((k, v))) => Some(Ok((k.into(), v.into()))), + Some(Ok((k, v))) => { + let k: &'a [u8] = k; + let v: &'a [u8] = v; + Some(Ok((k.into(), v.into()))) + } } } } diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 9f2bf919..10e3bfee 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -97,6 +97,7 @@ impl IDb for SqliteDb { while let Some(row) = rows.next()? { let name = row.get::<_, String>(0)?; let name = name.replace("_COLON_", ":"); + let name = name.strip_prefix("tree_").unwrap().to_string(); trees.push(name); } Ok(trees) -- 2.43.0 From 4539a6c2298cfb4578261060e4a5af739a45c99f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 12:02:42 +0200 Subject: [PATCH 28/64] Fix more sqlite deadlocks --- src/db/sqlite_adapter.rs | 133 +++++++++++++++++++-------------------- src/table/gc.rs | 20 +++--- 2 files changed, 79 insertions(+), 74 deletions(-) diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 10e3bfee..e945c31c 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -1,9 +1,10 @@ use core::ops::Bound; +use std::borrow::BorrowMut; use std::marker::PhantomPinned; use std::pin::Pin; use std::ptr::NonNull; -use std::sync::{Arc, Mutex, MutexGuard, RwLock}; +use std::sync::{Arc, Mutex, MutexGuard}; use log::trace; @@ -29,24 +30,26 @@ impl From for TxError { // -- db -pub struct SqliteDb { - db: Mutex, - trees: RwLock>, +pub struct SqliteDb(Mutex); + +struct SqliteDbInner { + db: Connection, + trees: Vec, } impl SqliteDb { pub fn init(db: rusqlite::Connection) -> Db { - let s = Self { - db: Mutex::new(db), - trees: RwLock::new(Vec::new()), - }; + let s = Self(Mutex::new(SqliteDbInner { + db, + trees: Vec::new(), + })); Db(Arc::new(s)) } +} +impl SqliteDbInner { fn get_tree(&self, i: usize) -> Result { self.trees - .read() - .unwrap() .get(i) .cloned() .ok_or_else(|| Error("invalid tree id".into())) @@ -56,16 +59,13 @@ impl SqliteDb { impl IDb for SqliteDb { fn open_tree(&self, name: &str) -> Result { let name = format!("tree_{}", name.replace(':', "_COLON_")); + let mut this = self.0.lock().unwrap(); - let mut trees = self.trees.write().unwrap(); - if let Some(i) = trees.iter().position(|x| x == &name) { + if let Some(i) = this.trees.iter().position(|x| x == &name) { Ok(i) } else { - trace!("open tree {}: lock db", name); - let db = self.db.lock().unwrap(); trace!("create table {}", name); - - db.execute( + this.db.execute( &format!( "CREATE TABLE IF NOT EXISTS {} ( k BLOB PRIMARY KEY, @@ -75,10 +75,10 @@ impl IDb for SqliteDb { ), [], )?; - trace!("table created: {}", name); + trace!("table created: {}, unlocking", name); - let i = trees.len(); - trees.push(name.to_string()); + let i = this.trees.len(); + this.trees.push(name.to_string()); Ok(i) } } @@ -87,10 +87,10 @@ impl IDb for SqliteDb { let mut trees = vec![]; trace!("list_trees: lock db"); - let db = self.db.lock().unwrap(); + let this = self.0.lock().unwrap(); trace!("list_trees: lock acquired"); - let mut stmt = db.prepare( + let mut stmt = this.db.prepare( "SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'", )?; let mut rows = stmt.query([])?; @@ -106,13 +106,15 @@ impl IDb for SqliteDb { // ---- fn get(&self, tree: usize, key: &[u8]) -> Result>> { - let tree = self.get_tree(tree)?; - trace!("get {}: lock db", tree); - let db = self.db.lock().unwrap(); + let this = self.0.lock().unwrap(); trace!("get {}: lock acquired", tree); - let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; + let tree = this.get_tree(tree)?; + + let mut stmt = this + .db + .prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; let mut res_iter = stmt.query([key])?; match res_iter.next()? { None => Ok(None), @@ -121,24 +123,24 @@ impl IDb for SqliteDb { } fn remove(&self, tree: usize, key: &[u8]) -> Result { - let tree = self.get_tree(tree)?; - trace!("remove {}: lock db", tree); - let db = self.db.lock().unwrap(); + let this = self.0.lock().unwrap(); trace!("remove {}: lock acquired", tree); - let res = db.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; + let tree = this.get_tree(tree)?; + let res = this + .db + .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; Ok(res > 0) } fn len(&self, tree: usize) -> Result { - let tree = self.get_tree(tree)?; - trace!("len {}: lock db", tree); - let db = self.db.lock().unwrap(); + let this = self.0.lock().unwrap(); trace!("len {}: lock acquired", tree); - let mut stmt = db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; + let tree = this.get_tree(tree)?; + let mut stmt = this.db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; let mut res_iter = stmt.query([])?; match res_iter.next()? { None => Ok(0), @@ -147,13 +149,12 @@ impl IDb for SqliteDb { } fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { - let tree = self.get_tree(tree)?; - trace!("insert {}: lock db", tree); - let db = self.db.lock().unwrap(); + let this = self.0.lock().unwrap(); trace!("insert {}: lock acquired", tree); - db.execute( + let tree = this.get_tree(tree)?; + this.db.execute( &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), params![key, value], )?; @@ -161,25 +162,23 @@ impl IDb for SqliteDb { } fn iter(&self, tree: usize) -> Result> { - let tree = self.get_tree(tree)?; - let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); - trace!("iter {}: lock db", tree); - let db = self.db.lock().unwrap(); + let this = self.0.lock().unwrap(); trace!("iter {}: lock acquired", tree); - DbValueIterator::make(db, &sql, []) + let tree = this.get_tree(tree)?; + let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree); + DbValueIterator::make(this, &sql, []) } fn iter_rev(&self, tree: usize) -> Result> { - let tree = self.get_tree(tree)?; - let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree); - trace!("iter_rev {}: lock db", tree); - let db = self.db.lock().unwrap(); + let this = self.0.lock().unwrap(); trace!("iter_rev {}: lock acquired", tree); - DbValueIterator::make(db, &sql, []) + let tree = this.get_tree(tree)?; + let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree); + DbValueIterator::make(this, &sql, []) } fn range<'r>( @@ -188,7 +187,11 @@ impl IDb for SqliteDb { low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, ) -> Result> { - let tree = self.get_tree(tree)?; + trace!("range {}: lock db", tree); + let this = self.0.lock().unwrap(); + trace!("range {}: lock acquired", tree); + + let tree = this.get_tree(tree)?; let (bounds_sql, params) = bounds_sql(low, high); let sql = format!("SELECT k, v FROM {} {} ORDER BY k ASC", tree, bounds_sql); @@ -198,11 +201,7 @@ impl IDb for SqliteDb { .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - trace!("range {}: lock db", tree); - let db = self.db.lock().unwrap(); - trace!("range {}: lock acquired", tree); - - DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(db, &sql, params.as_ref()) + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref()) } fn range_rev<'r>( &self, @@ -210,7 +209,11 @@ impl IDb for SqliteDb { low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, ) -> Result> { - let tree = self.get_tree(tree)?; + trace!("range_rev {}: lock db", tree); + let this = self.0.lock().unwrap(); + trace!("range_rev {}: lock acquired", tree); + + let tree = this.get_tree(tree)?; let (bounds_sql, params) = bounds_sql(low, high); let sql = format!("SELECT k, v FROM {} {} ORDER BY k DESC", tree, bounds_sql); @@ -220,25 +223,21 @@ impl IDb for SqliteDb { .map(|x| x as &dyn rusqlite::ToSql) .collect::>(); - trace!("range_rev {}: lock db", tree); - let db = self.db.lock().unwrap(); - trace!("range_rev {}: lock acquired", tree); - - DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(db, &sql, params.as_ref()) + DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref()) } // ---- fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> { - let trees = self.trees.read().unwrap(); - trace!("transaction: lock db"); - let mut db = self.db.lock().unwrap(); + let mut this = self.0.lock().unwrap(); trace!("transaction: lock acquired"); + let this_mut_ref: &mut SqliteDbInner = this.borrow_mut(); + let mut tx = SqliteTx { - tx: db.transaction()?, - trees: trees.as_ref(), + tx: this_mut_ref.db.transaction()?, + trees: &this_mut_ref.trees, }; let res = match f.try_on(&mut tx) { TxFnResult::Ok => { @@ -345,7 +344,7 @@ impl<'a> ITx for SqliteTx<'a> { // ---- struct DbValueIterator<'a> { - db: MutexGuard<'a, Connection>, + db: MutexGuard<'a, SqliteDbInner>, stmt: Option>, iter: Option>, _pin: PhantomPinned, @@ -353,7 +352,7 @@ struct DbValueIterator<'a> { impl<'a> DbValueIterator<'a> { fn make( - db: MutexGuard<'a, Connection>, + db: MutexGuard<'a, SqliteDbInner>, sql: &str, args: P, ) -> Result> { @@ -368,7 +367,7 @@ impl<'a> DbValueIterator<'a> { unsafe { let db = NonNull::from(&boxed.db); - let stmt = db.as_ref().prepare(sql)?; + let stmt = db.as_ref().db.prepare(sql)?; let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed); Pin::get_unchecked_mut(mut_ref).stmt = Some(stmt); diff --git a/src/table/gc.rs b/src/table/gc.rs index e2611389..2a23f4cc 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -101,18 +101,16 @@ where async fn gc_loop_iter(&self) -> Result, Error> { let now = now_msec(); - let mut entries = vec![]; - let mut excluded = vec![]; - // List entries in the GC todo list // These entries are put there when a tombstone is inserted in the table // (see update_entry in data.rs) + let mut candidates = vec![]; for entry_kv in self.data.gc_todo.iter()? { let (k, vhash) = entry_kv?; - let mut todo_entry = GcTodoEntry::parse(&k, &vhash); + let todo_entry = GcTodoEntry::parse(&k, &vhash); if todo_entry.deletion_time() > now { - if entries.is_empty() && excluded.is_empty() { + if candidates.is_empty() { // If the earliest entry in the todo list shouldn't yet be processed, // return a duration to wait in the loop return Ok(Some(Duration::from_millis( @@ -124,15 +122,23 @@ where } } - let vhash = Hash::try_from(&vhash[..]).unwrap(); + candidates.push(todo_entry); + if candidates.len() >= 2 * TABLE_GC_BATCH_SIZE { + break; + } + } + let mut entries = vec![]; + let mut excluded = vec![]; + for mut todo_entry in candidates { // Check if the tombstone is still the current value of the entry. // If not, we don't actually want to GC it, and we will remove it // from the gc_todo table later (below). + let vhash = todo_entry.value_hash; todo_entry.value = self .data .store - .get(&k[..])? + .get(&todo_entry.key[..])? .filter(|v| blake2sum(&v[..]) == vhash) .map(|v| v.to_vec()); -- 2.43.0 From 7f2cf0b809f1fc5741990e2bfff94dc3ec41a04f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 14:01:44 +0200 Subject: [PATCH 29/64] Safe choice: return Vec and not some fancy zero-copy type --- src/block/manager.rs | 12 ++--- src/db/lib.rs | 101 ++++----------------------------------- src/db/lmdb_adapter.rs | 70 +++++---------------------- src/db/sled_adapter.rs | 39 ++++----------- src/db/sqlite_adapter.rs | 12 ++--- src/garage/repair.rs | 8 ++-- src/table/data.rs | 4 +- src/table/merkle.rs | 6 +-- src/table/sync.rs | 2 +- 9 files changed, 49 insertions(+), 205 deletions(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index decf33cc..abc5063d 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -555,11 +555,7 @@ impl BlockManager { // - Ok(false) -> no block was processed, but we are ready for the next iteration // - Err(_) -> a Sled error occurred when reading/writing from resync_queue/resync_errors async fn resync_iter(&self, must_exit: &mut watch::Receiver) -> Result { - let next = self - .resync_queue - .first()? - .map(|(k, v)| (k.into_vec(), v.into_vec())); - if let Some((time_bytes, hash_bytes)) = next { + if let Some((time_bytes, hash_bytes)) = self.resync_queue.first()? { let time_msec = u64::from_be_bytes(time_bytes[0..8].try_into().unwrap()); let now = now_msec(); @@ -567,7 +563,7 @@ impl BlockManager { let hash = Hash::try_from(&hash_bytes[..]).unwrap(); if let Some(ec) = self.resync_errors.get(hash.as_slice())? { - let ec = ErrorCounter::decode(ec); + let ec = ErrorCounter::decode(&ec); if now < ec.next_try() { // if next retry after an error is not yet, // don't do resync and return early, but still @@ -608,7 +604,7 @@ impl BlockManager { warn!("Error when resyncing {:?}: {}", hash, e); let err_counter = match self.resync_errors.get(hash.as_slice())? { - Some(ec) => ErrorCounter::decode(ec).add1(now + 1), + Some(ec) => ErrorCounter::decode(&ec).add1(now + 1), None => ErrorCounter::new(now + 1), }; @@ -972,7 +968,7 @@ impl ErrorCounter { } } - fn decode(data: db::Value<'_>) -> Self { + fn decode(data: &db::Value) -> Self { Self { errors: u64::from_be_bytes(data[0..8].try_into().unwrap()), last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()), diff --git a/src/db/lib.rs b/src/db/lib.rs index 2e847f7a..b9c27d9a 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -21,93 +21,8 @@ pub struct Transaction<'a>(pub(crate) &'a mut dyn ITx); #[derive(Clone)] pub struct Tree(pub(crate) Arc, pub(crate) usize); -pub type ValueIter<'a> = Box, Value<'a>)>> + 'a>; - -// ---- - -pub struct Value<'a>(pub(crate) Box + 'a>); - -pub trait IValue<'a>: AsRef<[u8]> + core::borrow::Borrow<[u8]> { - fn take_maybe(&mut self) -> Vec; -} - -impl<'a> Value<'a> { - #[inline] - pub fn into_vec(mut self) -> Vec { - self.0.take_maybe() - } -} - -impl<'a> AsRef<[u8]> for Value<'a> { - #[inline] - fn as_ref(&self) -> &[u8] { - self.0.as_ref().as_ref() - } -} - -impl<'a> std::borrow::Borrow<[u8]> for Value<'a> { - #[inline] - fn borrow(&self) -> &[u8] { - self.0.as_ref().borrow() - } -} - -impl<'a> core::ops::Deref for Value<'a> { - type Target = [u8]; - #[inline] - fn deref(&self) -> &[u8] { - self.0.as_ref().as_ref() - } -} - -impl<'a, T> PartialEq for Value<'a> -where - T: AsRef<[u8]>, -{ - fn eq(&self, other: &T) -> bool { - self.as_ref() == other.as_ref() - } -} - -impl<'a> std::fmt::Debug for Value<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for line in hexdump::hexdump_iter(self.as_ref()) { - f.write_str(&line)?; - f.write_str("\n")?; - } - Ok(()) - } -} - -impl<'a> IValue<'a> for Vec { - fn take_maybe(&mut self) -> Vec { - std::mem::take(self) - } -} - -impl<'a> From> for Vec { - fn from(v: Value<'a>) -> Vec { - v.into_vec() - } -} - -impl<'a> From> for Value<'a> { - fn from(v: Vec) -> Value<'a> { - Value(Box::new(v)) - } -} - -impl<'a> From<&'a [u8]> for Value<'a> { - fn from(v: &'a [u8]) -> Value<'a> { - Value(Box::new(v)) - } -} - -impl<'a> IValue<'a> for &'a [u8] { - fn take_maybe(&mut self) -> Vec { - self.to_vec() - } -} +pub type Value = Vec; +pub type ValueIter<'a> = Box> + 'a>; // ---- @@ -228,17 +143,17 @@ impl Tree { Db(self.0.clone()) } - pub fn get>(&self, key: T) -> Result>> { + pub fn get>(&self, key: T) -> Result> { self.0.get(self.1, key.as_ref()) } pub fn len(&self) -> Result { self.0.len(self.1) } - pub fn first(&self) -> Result, Value<'_>)>> { + pub fn first(&self) -> Result> { self.iter()?.next().transpose() } - pub fn get_gt>(&self, from: T) -> Result, Value<'_>)>> { + pub fn get_gt>(&self, from: T) -> Result> { self.range((Bound::Excluded(from), Bound::Unbounded))? .next() .transpose() @@ -280,7 +195,7 @@ impl Tree { #[allow(clippy::len_without_is_empty)] impl<'a> Transaction<'a> { - pub fn get>(&self, tree: &Tree, key: T) -> Result>> { + pub fn get>(&self, tree: &Tree, key: T) -> Result> { self.0.get(tree.1, key.as_ref()) } pub fn len(&self, tree: &Tree) -> Result { @@ -342,7 +257,7 @@ pub(crate) trait IDb: Send + Sync { fn open_tree(&self, name: &str) -> Result; fn list_trees(&self) -> Result>; - fn get(&self, tree: usize, key: &[u8]) -> Result>>; + fn get(&self, tree: usize, key: &[u8]) -> Result>; fn len(&self, tree: usize) -> Result; fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; @@ -368,7 +283,7 @@ pub(crate) trait IDb: Send + Sync { } pub(crate) trait ITx { - fn get(&self, tree: usize, key: &[u8]) -> Result>>; + fn get(&self, tree: usize, key: &[u8]) -> Result>; fn len(&self, tree: usize) -> Result; fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index aa365733..3d0edb38 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -11,7 +11,7 @@ use heed::types::ByteSlice; use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; use crate::{ - Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter, + Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, }; pub use heed; @@ -101,28 +101,15 @@ impl IDb for LmdbDb { // ---- - fn get(&self, tree: usize, key: &[u8]) -> Result>> { + fn get(&self, tree: usize, key: &[u8]) -> Result> { let tree = self.get_tree(tree)?; - let res = TxAndValue { - tx: self.db.read_txn()?, - value: NonNull::dangling(), - _pin: PhantomPinned, - }; - let mut boxed = Box::pin(res); - - unsafe { - let tx = NonNull::from(&boxed.tx); - let val = match tree.get(tx.as_ref(), &key)? { - None => return Ok(None), - Some(v) => v, - }; - - let mut_ref: Pin<&mut TxAndValue<'_>> = Pin::as_mut(&mut boxed); - Pin::get_unchecked_mut(mut_ref).value = NonNull::from(&val); + let tx = self.db.read_txn()?; + let val = tree.get(&tx, &key)?; + match val { + None => Ok(None), + Some(v) => Ok(Some(v.to_vec())), } - - Ok(Some(Value(Box::new(TxAndValuePin(boxed))))) } fn remove(&self, tree: usize, key: &[u8]) -> Result { @@ -227,13 +214,10 @@ impl<'a, 'db> LmdbTx<'a, 'db> { } impl<'a, 'db> ITx for LmdbTx<'a, 'db> { - fn get(&self, tree: usize, key: &[u8]) -> Result>> { + fn get(&self, tree: usize, key: &[u8]) -> Result> { let tree = self.get_tree(tree)?; match tree.get(&self.tx, &key)? { - Some(v) => { - let v: &'_ [u8] = v; - Ok(Some(v.into())) - } + Some(v) => Ok(Some(v.to_vec())), None => Ok(None), } } @@ -279,34 +263,6 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { // ---- -struct TxAndValue<'a> { - tx: RoTxn<'a>, - value: NonNull<&'a [u8]>, - _pin: PhantomPinned, -} - -struct TxAndValuePin<'a>(Pin>>); - -impl<'a> IValue<'a> for TxAndValuePin<'a> { - fn take_maybe(&mut self) -> Vec { - self.as_ref().to_vec() - } -} - -impl<'a> AsRef<[u8]> for TxAndValuePin<'a> { - fn as_ref(&self) -> &[u8] { - unsafe { self.0.value.as_ref() } - } -} - -impl<'a> std::borrow::Borrow<[u8]> for TxAndValuePin<'a> { - fn borrow(&self) -> &[u8] { - self.as_ref() - } -} - -// ---- - type IteratorItem<'a> = heed::Result<( >::DItem, >::DItem, @@ -365,7 +321,7 @@ impl<'a, I> Iterator for TxAndIteratorPin<'a, I> where I: Iterator> + 'a, { - type Item = Result<(Value<'a>, Value<'a>)>; + type Item = Result<(Value, Value)>; fn next(&mut self) -> Option { let iter_ref = unsafe { @@ -375,11 +331,7 @@ where match iter_ref.unwrap().next() { None => None, Some(Err(e)) => Some(Err(e.into())), - Some(Ok((k, v))) => { - let k: &'a [u8] = k; - let v: &'a [u8] = v; - Some(Ok((k.into(), v.into()))) - } + Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))), } } } diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 2953785e..489fae61 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -10,7 +10,7 @@ use sled::transaction::{ }; use crate::{ - Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter, + Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, }; pub use sled; @@ -23,26 +23,6 @@ impl From for Error { } } -// -- val - -impl<'a> IValue<'a> for sled::IVec { - fn take_maybe(&mut self) -> Vec { - self.to_vec() - } -} - -impl<'a> From> for sled::IVec { - fn from(v: Value<'a>) -> sled::IVec { - sled::IVec::from(v.into_vec()) - } -} - -impl<'a> From for Value<'a> { - fn from(v: sled::IVec) -> Value<'a> { - Value(Box::new(v)) - } -} - // -- db pub struct SledDb { @@ -99,9 +79,10 @@ impl IDb for SledDb { // ---- - fn get(&self, tree: usize, key: &[u8]) -> Result>> { + fn get(&self, tree: usize, key: &[u8]) -> Result> { let tree = self.get_tree(tree)?; - Ok(tree.get(key)?.map(From::from)) + let val = tree.get(key)?; + Ok(val.map(|x| x.to_vec())) } fn remove(&self, tree: usize, key: &[u8]) -> Result { @@ -123,14 +104,14 @@ impl IDb for SledDb { fn iter(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.iter().map(|v| { - v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) + v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into) }))) } fn iter_rev(&self, tree: usize) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.iter().rev().map(|v| { - v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) + v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into) }))) } @@ -142,7 +123,7 @@ impl IDb for SledDb { ) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).map(|v| { - v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into) + v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into) }))) } fn range_rev<'r>( @@ -153,7 +134,7 @@ impl IDb for SledDb { ) -> Result> { let tree = self.get_tree(tree)?; Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).rev().map( - |v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into), + |v| v.map(|(x, y)| (x.to_vec(), y.to_vec())).map_err(Into::into), ))) } @@ -218,10 +199,10 @@ impl<'a> SledTx<'a> { } impl<'a> ITx for SledTx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> Result>> { + fn get(&self, tree: usize, key: &[u8]) -> Result> { let tree = self.get_tree(tree)?; let tmp = self.save_error(tree.get(key))?; - Ok(tmp.map(From::from)) + Ok(tmp.map(|x| x.to_vec())) } fn len(&self, _tree: usize) -> Result { unimplemented!(".len() in transaction not supported with Sled backend") diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index e945c31c..f0bca257 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -105,7 +105,7 @@ impl IDb for SqliteDb { // ---- - fn get(&self, tree: usize, key: &[u8]) -> Result>> { + fn get(&self, tree: usize, key: &[u8]) -> Result> { trace!("get {}: lock db", tree); let this = self.0.lock().unwrap(); trace!("get {}: lock acquired", tree); @@ -118,7 +118,7 @@ impl IDb for SqliteDb { let mut res_iter = stmt.query([key])?; match res_iter.next()? { None => Ok(None), - Some(v) => Ok(Some(v.get::<_, Vec>(0)?.into())), + Some(v) => Ok(Some(v.get::<_, Vec>(0)?)), } } @@ -279,7 +279,7 @@ impl<'a> SqliteTx<'a> { } impl<'a> ITx for SqliteTx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> Result>> { + fn get(&self, tree: usize, key: &[u8]) -> Result> { let tree = self.get_tree(tree)?; let mut stmt = self .tx @@ -287,7 +287,7 @@ impl<'a> ITx for SqliteTx<'a> { let mut res_iter = stmt.query([key])?; match res_iter.next()? { None => Ok(None), - Some(v) => Ok(Some(v.get::<_, Vec>(0)?.into())), + Some(v) => Ok(Some(v.get::<_, Vec>(0)?)), } } fn len(&self, tree: usize) -> Result { @@ -394,7 +394,7 @@ impl<'a> Drop for DbValueIterator<'a> { struct DbValueIteratorPin<'a>(Pin>>); impl<'a> Iterator for DbValueIteratorPin<'a> { - type Item = Result<(Value<'a>, Value<'a>)>; + type Item = Result<(Value, Value)>; fn next(&mut self) -> Option { let next = unsafe { @@ -414,7 +414,7 @@ impl<'a> Iterator for DbValueIteratorPin<'a> { Err(e) => return Some(Err(e.into())), Ok(y) => y, }; - Some(Ok((k.into(), v.into()))) + Some(Ok((k, v))) } } diff --git a/src/garage/repair.rs b/src/garage/repair.rs index 1ae26181..ae28f351 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -72,8 +72,8 @@ impl Repair { Some(pair) => pair, None => break, }; - pos = k.into_vec(); - v.into_vec() + pos = k; + v }; i += 1; @@ -124,8 +124,8 @@ impl Repair { Some(pair) => pair, None => break, }; - pos = k.into_vec(); - v.into_vec() + pos = k; + v }; i += 1; diff --git a/src/table/data.rs b/src/table/data.rs index cca96f68..fe8c5dbc 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -138,7 +138,7 @@ where } }; if keep { - ret.push(Arc::new(ByteBuf::from(value.as_ref()))); + ret.push(Arc::new(ByteBuf::from(value))); } if ret.len() >= limit { break; @@ -271,7 +271,7 @@ where ) -> Result { let removed = self.store.db().transaction(|mut tx| { let remove_v = match tx.get(&self.store, k)? { - Some(cur_v) if blake2sum(&cur_v[..]) == vhash => Some(cur_v.into_vec()), + Some(cur_v) if blake2sum(&cur_v[..]) == vhash => Some(cur_v), _ => None, }; if remove_v.is_some() { diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 92e1445b..dc67e8b7 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -292,7 +292,7 @@ where k: &MerkleNodeKey, ) -> db::TxResult { let ent = tx.get(&self.data.merkle_tree, k.encode())?; - MerkleNode::decode_opt(ent).map_err(db::TxError::Abort) + MerkleNode::decode_opt(&ent).map_err(db::TxError::Abort) } fn put_node_txn( @@ -316,7 +316,7 @@ where // Access a node in the Merkle tree, used by the sync protocol pub(crate) fn read_node(&self, k: &MerkleNodeKey) -> Result { let ent = self.data.merkle_tree.get(k.encode())?; - MerkleNode::decode_opt(ent) + MerkleNode::decode_opt(&ent) } pub fn merkle_tree_len(&self) -> usize { @@ -351,7 +351,7 @@ impl MerkleNodeKey { } impl MerkleNode { - fn decode_opt(ent: Option>) -> Result { + fn decode_opt(ent: &Option) -> Result { match ent { None => Ok(MerkleNode::Empty), Some(v) => Ok(rmp_serde::decode::from_slice::(&v[..])?), diff --git a/src/table/sync.rs b/src/table/sync.rs index 87dfd1d8..20066d73 100644 --- a/src/table/sync.rs +++ b/src/table/sync.rs @@ -260,7 +260,7 @@ where for item in self.data.store.range(begin.to_vec()..end.to_vec())? { let (key, value) = item?; - items.push((key.to_vec(), Arc::new(ByteBuf::from(value.as_ref())))); + items.push((key.to_vec(), Arc::new(ByteBuf::from(value)))); if items.len() >= 1024 { break; -- 2.43.0 From a77efd7ca6a02b9c668607e67efcaeeafd36a8bd Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 14:02:02 +0200 Subject: [PATCH 30/64] Safe choice: revert "Update rmp-serde" This reverts commit 16e0a655d0d01e3871aee81a0a9660102d6df74e. --- Cargo.lock | 34 ++++++++-------------------------- src/block/Cargo.toml | 2 +- src/garage/Cargo.toml | 2 +- src/garage/repair.rs | 4 ++-- src/model/Cargo.toml | 2 +- src/model/index_counter.rs | 8 +++++--- src/model/key_table.rs | 2 +- src/model/migrate.rs | 2 +- src/model/s3/object_table.rs | 2 +- src/model/s3/version_table.rs | 2 +- src/rpc/Cargo.toml | 2 +- src/table/Cargo.toml | 2 +- src/table/data.rs | 2 +- src/table/merkle.rs | 2 +- src/util/Cargo.toml | 2 +- src/util/data.rs | 2 +- src/util/persister.rs | 4 ++-- 17 files changed, 30 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51fa2a95..23e4f468 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -970,7 +970,7 @@ dependencies = [ "pretty_env_logger", "prometheus", "rand 0.8.5", - "rmp-serde 1.1.0", + "rmp-serde 0.15.5", "serde", "serde_bytes", "serde_json", @@ -1042,7 +1042,7 @@ dependencies = [ "hex", "opentelemetry", "rand 0.8.5", - "rmp-serde 1.1.0", + "rmp-serde 0.15.5", "serde", "serde_bytes", "tokio", @@ -1111,7 +1111,7 @@ dependencies = [ "netapp 0.4.4", "opentelemetry", "rand 0.8.5", - "rmp-serde 1.1.0", + "rmp-serde 0.15.5", "serde", "serde_bytes", "tokio", @@ -1168,7 +1168,7 @@ dependencies = [ "opentelemetry", "pnet_datalink", "rand 0.8.5", - "rmp-serde 1.1.0", + "rmp-serde 0.15.5", "schemars", "serde", "serde_bytes", @@ -1214,7 +1214,7 @@ dependencies = [ "hexdump", "opentelemetry", "rand 0.8.5", - "rmp-serde 1.1.0", + "rmp-serde 0.15.5", "serde", "serde_bytes", "tokio", @@ -1262,7 +1262,7 @@ dependencies = [ "netapp 0.4.4", "opentelemetry", "rand 0.8.5", - "rmp-serde 1.1.0", + "rmp-serde 0.15.5", "serde", "serde_json", "sha2", @@ -2351,12 +2351,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "paste" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" - [[package]] name = "pem" version = "0.8.3" @@ -2746,13 +2740,12 @@ dependencies = [ [[package]] name = "rmp" -version = "0.8.11" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" +checksum = "4f55e5fa1446c4d5dd1f5daeed2a4fe193071771a2636274d0d7a3b082aa7ad6" dependencies = [ "byteorder", "num-traits", - "paste", ] [[package]] @@ -2777,17 +2770,6 @@ dependencies = [ "serde", ] -[[package]] -name = "rmp-serde" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25786b0d276110195fa3d6f3f31299900cf71dfbd6c28450f3f58a0e7f7a347e" -dependencies = [ - "byteorder", - "rmp", - "serde", -] - [[package]] name = "roxmltree" version = "0.14.1" diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index 1a49947c..80346aca 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -28,7 +28,7 @@ tracing = "0.1.30" rand = "0.8" zstd = { version = "0.9", default-features = false } -rmp-serde = "1.1" +rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index d5299291..eb643160 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -37,7 +37,7 @@ rand = "0.8" async-trait = "0.1.7" sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } -rmp-serde = "1.1" +rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" structopt = { version = "0.3", default-features = false } diff --git a/src/garage/repair.rs b/src/garage/repair.rs index ae28f351..75828f24 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -81,7 +81,7 @@ impl Repair { info!("repair_versions: {}", i); } - let version = rmp_serde::decode::from_slice::(&item_bytes)?; + let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?; if version.deleted.get() { continue; } @@ -133,7 +133,7 @@ impl Repair { info!("repair_block_ref: {}", i); } - let block_ref = rmp_serde::decode::from_slice::(&item_bytes)?; + let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?; if block_ref.deleted.get() { continue; } diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 322feee1..d908dc01 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -31,7 +31,7 @@ tracing = "0.1.30" rand = "0.8" zstd = { version = "0.9", default-features = false } -rmp-serde = "1.1" +rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/model/index_counter.rs b/src/model/index_counter.rs index d8c1229a..6f81be3e 100644 --- a/src/model/index_counter.rs +++ b/src/model/index_counter.rs @@ -181,9 +181,11 @@ impl IndexCounter { let new_entry = self.local_counter.db().transaction(|mut tx| { let mut entry = match tx.get(&self.local_counter, &tree_key[..])? { - Some(old_bytes) => rmp_serde::decode::from_slice::(&old_bytes) - .map_err(Error::RmpDecode) - .map_err(db::TxError::Abort)?, + Some(old_bytes) => { + rmp_serde::decode::from_read_ref::<_, LocalCounterEntry>(&old_bytes) + .map_err(Error::RmpDecode) + .map_err(db::TxError::Abort)? + } None => LocalCounterEntry { values: BTreeMap::new(), }, diff --git a/src/model/key_table.rs b/src/model/key_table.rs index 852bf607..330e83f0 100644 --- a/src/model/key_table.rs +++ b/src/model/key_table.rs @@ -175,7 +175,7 @@ impl TableSchema for KeyTable { } fn try_migrate(bytes: &[u8]) -> Option { - let old_k = rmp_serde::decode::from_slice::(bytes).ok()?; + let old_k = rmp_serde::decode::from_read_ref::<_, old::Key>(bytes).ok()?; let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone()); let state = if old_k.deleted.get() { diff --git a/src/model/migrate.rs b/src/model/migrate.rs index bfd9845b..25acb4b0 100644 --- a/src/model/migrate.rs +++ b/src/model/migrate.rs @@ -28,7 +28,7 @@ impl Migrate { let mut old_buckets = vec![]; for res in tree.iter().map_err(GarageError::from)? { let (_k, v) = res.map_err(GarageError::from)?; - let bucket = rmp_serde::decode::from_slice::(&v[..]) + let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..]) .map_err(GarageError::from)?; old_buckets.push(bucket); } diff --git a/src/model/s3/object_table.rs b/src/model/s3/object_table.rs index c242b325..3d9a89f7 100644 --- a/src/model/s3/object_table.rs +++ b/src/model/s3/object_table.rs @@ -270,7 +270,7 @@ impl TableSchema for ObjectTable { } fn try_migrate(bytes: &[u8]) -> Option { - let old_obj = rmp_serde::decode::from_slice::(bytes).ok()?; + let old_obj = rmp_serde::decode::from_read_ref::<_, old::Object>(bytes).ok()?; Some(migrate_object(old_obj)) } } diff --git a/src/model/s3/version_table.rs b/src/model/s3/version_table.rs index f67f2ff1..ad096772 100644 --- a/src/model/s3/version_table.rs +++ b/src/model/s3/version_table.rs @@ -168,7 +168,7 @@ impl TableSchema for VersionTable { } fn try_migrate(bytes: &[u8]) -> Option { - let old = rmp_serde::decode::from_slice::(bytes).ok()?; + let old = rmp_serde::decode::from_read_ref::<_, old::Version>(bytes).ok()?; let blocks = old .blocks diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 93df02a4..73328993 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -26,7 +26,7 @@ rand = "0.8" sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } async-trait = "0.1.7" -rmp-serde = "1.1" +rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" serde_json = "1.0" diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 690d5a71..6de37cda 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -26,7 +26,7 @@ hexdump = "0.1" tracing = "0.1.30" rand = "0.8" -rmp-serde = "1.1" +rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_bytes = "0.11" diff --git a/src/table/data.rs b/src/table/data.rs index fe8c5dbc..64ca9f14 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -300,7 +300,7 @@ where } pub fn decode_entry(&self, bytes: &[u8]) -> Result { - match rmp_serde::decode::from_slice::(bytes) { + match rmp_serde::decode::from_read_ref::<_, F::E>(bytes) { Ok(x) => Ok(x), Err(e) => match F::try_migrate(bytes) { Some(x) => Ok(x), diff --git a/src/table/merkle.rs b/src/table/merkle.rs index dc67e8b7..138f3ca3 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -354,7 +354,7 @@ impl MerkleNode { fn decode_opt(ent: &Option) -> Result { match ent { None => Ok(MerkleNode::Empty), - Some(v) => Ok(rmp_serde::decode::from_slice::(&v[..])?), + Some(v) => Ok(rmp_serde::decode::from_read_ref::<_, MerkleNode>(&v[..])?), } } diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index e68704ad..5d073436 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -25,7 +25,7 @@ rand = "0.8" sha2 = "0.9" chrono = "0.4" -rmp-serde = "1.1" +rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_json = "1.0" toml = "0.5" diff --git a/src/util/data.rs b/src/util/data.rs index 2ef976a5..7715c2cc 100644 --- a/src/util/data.rs +++ b/src/util/data.rs @@ -151,7 +151,7 @@ where let mut wr = Vec::with_capacity(128); let mut se = rmp_serde::Serializer::new(&mut wr) .with_struct_map() - .with_binary(); + .with_string_variants(); val.serialize(&mut se)?; Ok(wr) } diff --git a/src/util/persister.rs b/src/util/persister.rs index d0759db7..9e1a1910 100644 --- a/src/util/persister.rs +++ b/src/util/persister.rs @@ -33,7 +33,7 @@ where let mut bytes = vec![]; file.read_to_end(&mut bytes)?; - let value = rmp_serde::decode::from_slice(&bytes[..])?; + let value = rmp_serde::decode::from_read_ref(&bytes[..])?; Ok(value) } @@ -57,7 +57,7 @@ where let mut bytes = vec![]; file.read_to_end(&mut bytes).await?; - let value = rmp_serde::decode::from_slice(&bytes[..])?; + let value = rmp_serde::decode::from_read_ref(&bytes[..])?; Ok(value) } -- 2.43.0 From 0ab21e780e8b944fb2e231c45b3777add27f368d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 14:08:19 +0200 Subject: [PATCH 31/64] Small syntax changes --- src/db/lmdb_adapter.rs | 4 +--- src/db/sled_adapter.rs | 4 +--- src/garage/repair.rs | 26 ++++++++++++-------------- src/table/data.rs | 5 +---- src/table/gc.rs | 6 ++---- src/table/merkle.rs | 5 +---- 6 files changed, 18 insertions(+), 32 deletions(-) diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 3d0edb38..04902d7f 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -10,9 +10,7 @@ use std::sync::{Arc, RwLock}; use heed::types::ByteSlice; use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; -use crate::{ - Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, -}; +use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; pub use heed; diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 489fae61..97fec2c7 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -9,9 +9,7 @@ use sled::transaction::{ UnabortableTransactionError, }; -use crate::{ - Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, -}; +use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; pub use sled; diff --git a/src/garage/repair.rs b/src/garage/repair.rs index 75828f24..17e14b8b 100644 --- a/src/garage/repair.rs +++ b/src/garage/repair.rs @@ -67,13 +67,12 @@ impl Repair { let mut i = 0; while !*must_exit.borrow() { - let item_bytes = { - let (k, v) = match self.garage.version_table.data.store.get_gt(pos)? { - Some(pair) => pair, - None => break, - }; - pos = k; - v + let item_bytes = match self.garage.version_table.data.store.get_gt(pos)? { + Some((k, v)) => { + pos = k; + v + } + None => break, }; i += 1; @@ -119,13 +118,12 @@ impl Repair { let mut i = 0; while !*must_exit.borrow() { - let item_bytes = { - let (k, v) = match self.garage.block_ref_table.data.store.get_gt(pos)? { - Some(pair) => pair, - None => break, - }; - pos = k; - v + let item_bytes = match self.garage.block_ref_table.data.store.get_gt(pos)? { + Some((k, v)) => { + pos = k; + v + } + None => break, }; i += 1; diff --git a/src/table/data.rs b/src/table/data.rs index 64ca9f14..f6d49b4d 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -243,10 +243,7 @@ where pub(crate) fn delete_if_equal(self: &Arc, k: &[u8], v: &[u8]) -> Result { let removed = self.store.db().transaction(|mut tx| { - let remove = match tx.get(&self.store, k)? { - Some(cur_v) if cur_v == v => true, - _ => false, - }; + let remove = matches!(tx.get(&self.store, k)?, Some(cur_v) if cur_v == v => true); if remove { tx.remove(&self.store, k)?; tx.insert(&self.merkle_todo, k, vec![])?; diff --git a/src/table/gc.rs b/src/table/gc.rs index 2a23f4cc..b9c589f4 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -383,10 +383,8 @@ impl GcTodoEntry { pub(crate) fn remove_if_equal(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { let key = self.todo_table_key(); gc_todo_tree.db().transaction(|mut tx| { - let remove = match tx.get(gc_todo_tree, &key)? { - Some(ov) if ov == self.value_hash.as_slice() => true, - _ => false, - }; + let remove = + matches!(tx.get(gc_todo_tree, &key)? Some(ov) if ov == self.value_hash.as_slice()); if remove { tx.remove(gc_todo_tree, &key)?; } diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 138f3ca3..e7f2442e 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -140,10 +140,7 @@ where .transaction(|mut tx| self.update_item_rec(&mut tx, k, &khash, &key, new_vhash))?; let deleted = self.data.merkle_todo.db().transaction(|mut tx| { - let remove = match tx.get(&self.data.merkle_todo, k)? { - Some(ov) if ov == vhash_by => true, - _ => false, - }; + let remove = matches!(tx.get(&self.data.merkle_todo, k)?, Some(ov) if ov == vhash_by); if remove { tx.remove(&self.data.merkle_todo, k)?; } -- 2.43.0 From 17918e2ef5dec8d131c4242111688b2a2e96aade Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 14:09:17 +0200 Subject: [PATCH 32/64] Whoops --- src/table/data.rs | 2 +- src/table/gc.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/table/data.rs b/src/table/data.rs index f6d49b4d..6352ac24 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -243,7 +243,7 @@ where pub(crate) fn delete_if_equal(self: &Arc, k: &[u8], v: &[u8]) -> Result { let removed = self.store.db().transaction(|mut tx| { - let remove = matches!(tx.get(&self.store, k)?, Some(cur_v) if cur_v == v => true); + let remove = matches!(tx.get(&self.store, k)?, Some(cur_v) if cur_v == v); if remove { tx.remove(&self.store, k)?; tx.insert(&self.merkle_todo, k, vec![])?; diff --git a/src/table/gc.rs b/src/table/gc.rs index b9c589f4..e8843339 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -384,7 +384,7 @@ impl GcTodoEntry { let key = self.todo_table_key(); gc_todo_tree.db().transaction(|mut tx| { let remove = - matches!(tx.get(gc_todo_tree, &key)? Some(ov) if ov == self.value_hash.as_slice()); + matches!(tx.get(gc_todo_tree, &key)?, Some(ov) if ov == self.value_hash.as_slice()); if remove { tx.remove(gc_todo_tree, &key)?; } -- 2.43.0 From a65049db8fbd373382f8ff73ef7a1c07c3df10e0 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 15:04:04 +0200 Subject: [PATCH 33/64] Simplify lmdb adapter (one less indirection) --- src/db/lmdb_adapter.rs | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 04902d7f..9173ad37 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -1,6 +1,4 @@ -use core::marker::PhantomPinned; use core::ops::Bound; -use core::pin::Pin; use core::ptr::NonNull; use std::collections::HashMap; @@ -272,7 +270,6 @@ where { tx: RoTxn<'a>, iter: Option, - _pin: PhantomPinned, } impl<'a, I> TxAndIterator<'a, I> @@ -283,22 +280,12 @@ where where F: FnOnce(&'a RoTxn<'a>) -> Result, { - let res = TxAndIterator { - tx, - iter: None, - _pin: PhantomPinned, - }; - let mut boxed = Box::pin(res); + let mut res = TxAndIterator { tx, iter: None }; - unsafe { - let tx = NonNull::from(&boxed.tx); - let iter = iterfun(tx.as_ref())?; + let tx = unsafe { NonNull::from(&res.tx).as_ref() }; + res.iter = Some(iterfun(tx)?); - let mut_ref: Pin<&mut TxAndIterator<'a, I>> = Pin::as_mut(&mut boxed); - Pin::get_unchecked_mut(mut_ref).iter = Some(iter); - } - - Ok(Box::new(TxAndIteratorPin(boxed))) + Ok(Box::new(res)) } } @@ -311,22 +298,14 @@ where } } -struct TxAndIteratorPin<'a, I: Iterator> + 'a>( - Pin>>, -); - -impl<'a, I> Iterator for TxAndIteratorPin<'a, I> +impl<'a, I> Iterator for TxAndIterator<'a, I> where I: Iterator> + 'a, { type Item = Result<(Value, Value)>; fn next(&mut self) -> Option { - let iter_ref = unsafe { - let mut_ref: Pin<&mut TxAndIterator<'a, I>> = Pin::as_mut(&mut self.0); - Pin::get_unchecked_mut(mut_ref).iter.as_mut() - }; - match iter_ref.unwrap().next() { + match self.iter.as_mut().unwrap().next() { None => None, Some(Err(e)) => Some(Err(e.into())), Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))), -- 2.43.0 From c56d858834bbfbe3edea2dc0c825bf3b5ce51c98 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 15:10:33 +0200 Subject: [PATCH 34/64] Fix clippy --- src/db/lmdb_adapter.rs | 12 ++++++------ src/garage/tests/bucket.rs | 8 +++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 9173ad37..3f468128 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -101,7 +101,7 @@ impl IDb for LmdbDb { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; - let val = tree.get(&tx, &key)?; + let val = tree.get(&tx, key)?; match val { None => Ok(None), Some(v) => Ok(Some(v.to_vec())), @@ -111,7 +111,7 @@ impl IDb for LmdbDb { fn remove(&self, tree: usize, key: &[u8]) -> Result { let tree = self.get_tree(tree)?; let mut tx = self.db.write_txn()?; - let deleted = tree.delete(&mut tx, &key)?; + let deleted = tree.delete(&mut tx, key)?; tx.commit()?; Ok(deleted) } @@ -125,7 +125,7 @@ impl IDb for LmdbDb { fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; let mut tx = self.db.write_txn()?; - tree.put(&mut tx, &key, &value)?; + tree.put(&mut tx, key, value)?; tx.commit()?; Ok(()) } @@ -212,7 +212,7 @@ impl<'a, 'db> LmdbTx<'a, 'db> { impl<'a, 'db> ITx for LmdbTx<'a, 'db> { fn get(&self, tree: usize, key: &[u8]) -> Result> { let tree = self.get_tree(tree)?; - match tree.get(&self.tx, &key)? { + match tree.get(&self.tx, key)? { Some(v) => Ok(Some(v.to_vec())), None => Ok(None), } @@ -223,12 +223,12 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = *self.get_tree(tree)?; - tree.put(&mut self.tx, &key, &value)?; + tree.put(&mut self.tx, key, value)?; Ok(()) } fn remove(&mut self, tree: usize, key: &[u8]) -> Result { let tree = *self.get_tree(tree)?; - let deleted = tree.delete(&mut self.tx, &key)?; + let deleted = tree.delete(&mut self.tx, key)?; Ok(deleted) } diff --git a/src/garage/tests/bucket.rs b/src/garage/tests/bucket.rs index ff5cc8da..b32af068 100644 --- a/src/garage/tests/bucket.rs +++ b/src/garage/tests/bucket.rs @@ -29,8 +29,7 @@ async fn test_bucket_all() { .unwrap() .iter() .filter(|x| x.name.as_ref().is_some()) - .find(|x| x.name.as_ref().unwrap() == "hello") - .is_some()); + .any(|x| x.name.as_ref().unwrap() == "hello")); } { // Get its location @@ -75,13 +74,12 @@ async fn test_bucket_all() { { // Check bucket is deleted with List buckets let r = ctx.client.list_buckets().send().await.unwrap(); - assert!(r + assert!(!r .buckets .as_ref() .unwrap() .iter() .filter(|x| x.name.as_ref().is_some()) - .find(|x| x.name.as_ref().unwrap() == "hello") - .is_none()); + .any(|x| x.name.as_ref().unwrap() == "hello")); } } -- 2.43.0 From 18978153585d870d567c5ad7d9e9c96d3b65a884 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 15:46:00 +0200 Subject: [PATCH 35/64] Table updated trigger now happens in transaction, this is waaaay better! --- src/block/manager.rs | 29 ++++++++++---- src/block/rc.rs | 42 +++++++++----------- src/model/index_counter.rs | 58 +++++++++++++++------------- src/model/k2v/item_table.rs | 25 ++++++++++-- src/model/s3/block_ref_table.rs | 20 ++++++---- src/model/s3/object_table.rs | 12 +++++- src/model/s3/version_table.rs | 13 ++++++- src/table/data.rs | 68 ++++++++++++++++++--------------- src/table/schema.rs | 19 ++++++--- 9 files changed, 177 insertions(+), 109 deletions(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index abc5063d..b7dcaf8a 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -289,28 +289,41 @@ impl BlockManager { /// Increment the number of time a block is used, putting it to resynchronization if it is /// required, but not known - pub fn block_incref(&self, hash: &Hash) -> Result<(), Error> { - if self.rc.block_incref(hash)? { + pub fn block_incref(self: &Arc, tx: &mut db::Transaction, hash: Hash) -> db::Result<()> { + if self.rc.block_incref(tx, &hash)? { // When the reference counter is incremented, there is // normally a node that is responsible for sending us the // data of the block. However that operation may fail, // so in all cases we add the block here to the todo list // to check later that it arrived correctly, and if not // we will fecth it from someone. - self.put_to_resync(hash, 2 * BLOCK_RW_TIMEOUT)?; + let this = self.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(1)).await; + if let Err(e) = this.put_to_resync(&hash, 2 * BLOCK_RW_TIMEOUT) { + error!("Block {:?} could not be put in resync queue: {}.", hash, e); + } + }); } Ok(()) } /// Decrement the number of time a block is used - pub fn block_decref(&self, hash: &Hash) -> Result<(), Error> { - if self.rc.block_decref(hash)? { + pub fn block_decref(self: &Arc, tx: &mut db::Transaction, hash: Hash) -> db::Result<()> { + if self.rc.block_decref(tx, &hash)? { // When the RC is decremented, it might drop to zero, // indicating that we don't need the block. // There is a delay before we garbage collect it; // make sure that it is handled in the resync loop // after that delay has passed. - self.put_to_resync(hash, BLOCK_GC_DELAY + Duration::from_secs(10))?; + let this = self.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(1)).await; + if let Err(e) = this.put_to_resync(&hash, BLOCK_GC_DELAY + Duration::from_secs(10)) + { + error!("Block {:?} could not be put in resync queue: {}.", hash, e); + } + }); } Ok(()) } @@ -510,12 +523,12 @@ impl BlockManager { }); } - fn put_to_resync(&self, hash: &Hash, delay: Duration) -> Result<(), db::Error> { + fn put_to_resync(&self, hash: &Hash, delay: Duration) -> db::Result<()> { let when = now_msec() + delay.as_millis() as u64; self.put_to_resync_at(hash, when) } - fn put_to_resync_at(&self, hash: &Hash, when: u64) -> Result<(), db::Error> { + fn put_to_resync_at(&self, hash: &Hash, when: u64) -> db::Result<()> { trace!("Put resync_queue: {} {:?}", when, hash); let mut key = u64::to_be_bytes(when).to_vec(); key.extend(hash.as_ref()); diff --git a/src/block/rc.rs b/src/block/rc.rs index e0b952fd..42cdf241 100644 --- a/src/block/rc.rs +++ b/src/block/rc.rs @@ -19,35 +19,29 @@ impl BlockRc { /// Increment the reference counter associated to a hash. /// Returns true if the RC goes from zero to nonzero. - pub(crate) fn block_incref(&self, hash: &Hash) -> Result { - let old_rc = self.rc.db().transaction(|mut tx| { - let old_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?); - match old_rc.increment().serialize() { - Some(x) => { - tx.insert(&self.rc, &hash, x)?; - } - None => unreachable!(), - }; - tx.commit(old_rc) - })?; + pub(crate) fn block_incref(&self, tx: &mut db::Transaction, hash: &Hash) -> db::Result { + let old_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?); + match old_rc.increment().serialize() { + Some(x) => { + tx.insert(&self.rc, &hash, x)?; + } + None => unreachable!(), + }; Ok(old_rc.is_zero()) } /// Decrement the reference counter associated to a hash. /// Returns true if the RC is now zero. - pub(crate) fn block_decref(&self, hash: &Hash) -> Result { - let new_rc = self.rc.db().transaction(|mut tx| { - let new_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?).decrement(); - match new_rc.serialize() { - Some(x) => { - tx.insert(&self.rc, &hash, x)?; - } - None => { - tx.remove(&self.rc, &hash)?; - } - }; - tx.commit(new_rc) - })?; + pub(crate) fn block_decref(&self, tx: &mut db::Transaction, hash: &Hash) -> db::Result { + let new_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?).decrement(); + match new_rc.serialize() { + Some(x) => { + tx.insert(&self.rc, &hash, x)?; + } + None => { + tx.remove(&self.rc, &hash)?; + } + }; Ok(matches!(new_rc, RcEntry::Deletable { .. })) } diff --git a/src/model/index_counter.rs b/src/model/index_counter.rs index 6f81be3e..4fec1138 100644 --- a/src/model/index_counter.rs +++ b/src/model/index_counter.rs @@ -116,8 +116,14 @@ impl TableSchema for CounterTable { type E = CounterEntry; type Filter = (DeletedFilter, Vec); - fn updated(&self, _old: Option<&Self::E>, _new: Option<&Self::E>) { + fn updated( + &self, + _tx: &mut db::Transaction, + _old: Option<&Self::E>, + _new: Option<&Self::E>, + ) -> db::Result<()> { // nothing for now + Ok(()) } fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { @@ -176,36 +182,36 @@ impl IndexCounter { this } - pub fn count(&self, pk: &T::P, sk: &T::S, counts: &[(&str, i64)]) -> Result<(), Error> { + pub fn count( + &self, + tx: &mut db::Transaction, + pk: &T::P, + sk: &T::S, + counts: &[(&str, i64)], + ) -> db::TxResult<(), Error> { let tree_key = self.table.data.tree_key(pk, sk); - let new_entry = self.local_counter.db().transaction(|mut tx| { - let mut entry = match tx.get(&self.local_counter, &tree_key[..])? { - Some(old_bytes) => { - rmp_serde::decode::from_read_ref::<_, LocalCounterEntry>(&old_bytes) - .map_err(Error::RmpDecode) - .map_err(db::TxError::Abort)? - } - None => LocalCounterEntry { - values: BTreeMap::new(), - }, - }; + let mut entry = match tx.get(&self.local_counter, &tree_key[..])? { + Some(old_bytes) => rmp_serde::decode::from_read_ref::<_, LocalCounterEntry>(&old_bytes) + .map_err(Error::RmpDecode) + .map_err(db::TxError::Abort)?, + None => LocalCounterEntry { + values: BTreeMap::new(), + }, + }; - for (s, inc) in counts.iter() { - let mut ent = entry.values.entry(s.to_string()).or_insert((0, 0)); - ent.0 += 1; - ent.1 += *inc; - } + for (s, inc) in counts.iter() { + let mut ent = entry.values.entry(s.to_string()).or_insert((0, 0)); + ent.0 += 1; + ent.1 += *inc; + } - let new_entry_bytes = rmp_to_vec_all_named(&entry) - .map_err(Error::RmpEncode) - .map_err(db::TxError::Abort)?; - tx.insert(&self.local_counter, &tree_key[..], new_entry_bytes)?; + let new_entry_bytes = rmp_to_vec_all_named(&entry) + .map_err(Error::RmpEncode) + .map_err(db::TxError::Abort)?; + tx.insert(&self.local_counter, &tree_key[..], new_entry_bytes)?; - Ok(entry) - })?; - - if let Err(e) = self.propagate_tx.send((pk.clone(), sk.clone(), new_entry)) { + if let Err(e) = self.propagate_tx.send((pk.clone(), sk.clone(), entry)) { error!( "Could not propagate updated counter values, failed to send to channel: {}", e diff --git a/src/model/k2v/item_table.rs b/src/model/k2v/item_table.rs index 8b7cc08a..77446f64 100644 --- a/src/model/k2v/item_table.rs +++ b/src/model/k2v/item_table.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::sync::Arc; +use garage_db as db; use garage_util::data::*; use garage_table::crdt::*; @@ -221,7 +222,12 @@ impl TableSchema for K2VItemTable { type E = K2VItem; type Filter = ItemFilter; - fn updated(&self, old: Option<&Self::E>, new: Option<&Self::E>) { + fn updated( + &self, + tx: &mut db::Transaction, + old: Option<&Self::E>, + new: Option<&Self::E>, + ) -> db::Result<()> { // 1. Count let (old_entries, old_conflicts, old_values, old_bytes) = match old { None => (0, 0, 0, 0), @@ -239,7 +245,8 @@ impl TableSchema for K2VItemTable { .map(|e| &e.partition.partition_key) .unwrap_or_else(|| &new.unwrap().partition.partition_key); - if let Err(e) = self.counter_table.count( + match self.counter_table.count( + tx, &count_pk, count_sk, &[ @@ -249,13 +256,25 @@ impl TableSchema for K2VItemTable { (BYTES, new_bytes - old_bytes), ], ) { - error!("Could not update K2V counter for bucket {:?} partition {}; counts will now be inconsistent. {}", count_pk, count_sk, e); + Ok(()) => (), + Err(db::TxError::Db(e)) => return Err(e), + Err(db::TxError::Abort(e)) => { + // This result can be returned by `counter_table.count()` for instance + // if messagepack serialization or deserialization fails at some step. + // Warn admin but ignore this error for now, that's all we can do. + error!( + "Unable to update K2V item counter for bucket {:?} partition {}: {}. Index values will be wrong!", + count_pk, count_sk, e + ); + } } // 2. Notify if let Some(new_ent) = new { self.subscriptions.notify(new_ent); } + + Ok(()) } #[allow(clippy::nonminimal_bool)] diff --git a/src/model/s3/block_ref_table.rs b/src/model/s3/block_ref_table.rs index 9b3991bf..2c06bc96 100644 --- a/src/model/s3/block_ref_table.rs +++ b/src/model/s3/block_ref_table.rs @@ -1,6 +1,8 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; +use garage_db as db; + use garage_util::data::*; use garage_table::crdt::Crdt; @@ -51,21 +53,23 @@ impl TableSchema for BlockRefTable { type E = BlockRef; type Filter = DeletedFilter; - fn updated(&self, old: Option<&Self::E>, new: Option<&Self::E>) { + fn updated( + &self, + tx: &mut db::Transaction, + old: Option<&Self::E>, + new: Option<&Self::E>, + ) -> db::Result<()> { #[allow(clippy::or_fun_call)] - let block = &old.or(new).unwrap().block; + let block = old.or(new).unwrap().block; let was_before = old.map(|x| !x.deleted.get()).unwrap_or(false); let is_after = new.map(|x| !x.deleted.get()).unwrap_or(false); if is_after && !was_before { - if let Err(e) = self.block_manager.block_incref(block) { - warn!("block_incref failed for block {:?}: {}", block, e); - } + self.block_manager.block_incref(tx, block)?; } if was_before && !is_after { - if let Err(e) = self.block_manager.block_decref(block) { - warn!("block_decref failed for block {:?}: {}", block, e); - } + self.block_manager.block_decref(tx, block)?; } + Ok(()) } fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { diff --git a/src/model/s3/object_table.rs b/src/model/s3/object_table.rs index 3d9a89f7..f3bd9892 100644 --- a/src/model/s3/object_table.rs +++ b/src/model/s3/object_table.rs @@ -2,6 +2,8 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::sync::Arc; +use garage_db as db; + use garage_util::background::BackgroundRunner; use garage_util::data::*; @@ -232,7 +234,12 @@ impl TableSchema for ObjectTable { type E = Object; type Filter = ObjectFilter; - fn updated(&self, old: Option<&Self::E>, new: Option<&Self::E>) { + fn updated( + &self, + _tx: &mut db::Transaction, + old: Option<&Self::E>, + new: Option<&Self::E>, + ) -> db::Result<()> { let version_table = self.version_table.clone(); let old = old.cloned(); let new = new.cloned(); @@ -259,7 +266,8 @@ impl TableSchema for ObjectTable { } } Ok(()) - }) + }); + Ok(()) } fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { diff --git a/src/model/s3/version_table.rs b/src/model/s3/version_table.rs index ad096772..d168c2c2 100644 --- a/src/model/s3/version_table.rs +++ b/src/model/s3/version_table.rs @@ -1,6 +1,8 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; +use garage_db as db; + use garage_util::background::BackgroundRunner; use garage_util::data::*; @@ -137,7 +139,12 @@ impl TableSchema for VersionTable { type E = Version; type Filter = DeletedFilter; - fn updated(&self, old: Option<&Self::E>, new: Option<&Self::E>) { + fn updated( + &self, + _tx: &mut db::Transaction, + old: Option<&Self::E>, + new: Option<&Self::E>, + ) -> db::Result<()> { let block_ref_table = self.block_ref_table.clone(); let old = old.cloned(); let new = new.cloned(); @@ -160,7 +167,9 @@ impl TableSchema for VersionTable { } } Ok(()) - }) + }); + + Ok(()) } fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { diff --git a/src/table/data.rs b/src/table/data.rs index 6352ac24..e688168f 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -209,17 +209,20 @@ where let new_bytes_hash = blake2sum(&new_bytes[..]); tx.insert(&self.merkle_todo, tree_key, new_bytes_hash.as_slice())?; tx.insert(&self.store, tree_key, new_bytes)?; - Ok(Some((old_entry, new_entry, new_bytes_hash))) + + self.instance + .updated(&mut tx, old_entry.as_ref(), Some(&new_entry))?; + + Ok(Some((new_entry, new_bytes_hash))) } else { Ok(None) } })?; - if let Some((old_entry, new_entry, new_bytes_hash)) = changed { + if let Some((new_entry, new_bytes_hash)) = changed { self.metrics.internal_update_counter.add(1); let is_tombstone = new_entry.is_tombstone(); - self.instance.updated(old_entry.as_ref(), Some(&new_entry)); self.merkle_todo_notify.notify_one(); if is_tombstone { // We are only responsible for GC'ing this item if we are the @@ -242,20 +245,23 @@ where } pub(crate) fn delete_if_equal(self: &Arc, k: &[u8], v: &[u8]) -> Result { - let removed = self.store.db().transaction(|mut tx| { - let remove = matches!(tx.get(&self.store, k)?, Some(cur_v) if cur_v == v); - if remove { - tx.remove(&self.store, k)?; - tx.insert(&self.merkle_todo, k, vec![])?; - } - Ok(remove) - })?; + let removed = self + .store + .db() + .transaction(|mut tx| match tx.get(&self.store, k)? { + Some(cur_v) if cur_v == v => { + tx.remove(&self.store, k)?; + tx.insert(&self.merkle_todo, k, vec![])?; + + let old_entry = self.decode_entry(v).map_err(db::TxError::Abort)?; + self.instance.updated(&mut tx, Some(&old_entry), None)?; + Ok(true) + } + _ => Ok(false), + })?; if removed { self.metrics.internal_delete_counter.add(1); - - let old_entry = self.decode_entry(v)?; - self.instance.updated(Some(&old_entry), None); self.merkle_todo_notify.notify_one(); } Ok(removed) @@ -266,26 +272,26 @@ where k: &[u8], vhash: Hash, ) -> Result { - let removed = self.store.db().transaction(|mut tx| { - let remove_v = match tx.get(&self.store, k)? { - Some(cur_v) if blake2sum(&cur_v[..]) == vhash => Some(cur_v), - _ => None, - }; - if remove_v.is_some() { - tx.remove(&self.store, k)?; - tx.insert(&self.merkle_todo, k, vec![])?; - } - Ok(remove_v) - })?; + let removed = self + .store + .db() + .transaction(|mut tx| match tx.get(&self.store, k)? { + Some(cur_v) if blake2sum(&cur_v[..]) == vhash => { + tx.remove(&self.store, k)?; + tx.insert(&self.merkle_todo, k, vec![])?; - if let Some(old_v) = removed { - let old_entry = self.decode_entry(&old_v[..])?; - self.instance.updated(Some(&old_entry), None); + let old_entry = self.decode_entry(&cur_v[..]).map_err(db::TxError::Abort)?; + self.instance.updated(&mut tx, Some(&old_entry), None)?; + Ok(true) + } + _ => Ok(false), + })?; + + if removed { + self.metrics.internal_delete_counter.add(1); self.merkle_todo_notify.notify_one(); - Ok(true) - } else { - Ok(false) } + Ok(removed) } // ---- Utility functions ---- diff --git a/src/table/schema.rs b/src/table/schema.rs index 37327037..393c7388 100644 --- a/src/table/schema.rs +++ b/src/table/schema.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; +use garage_db as db; use garage_util::data::*; use crate::crdt::Crdt; @@ -82,11 +83,19 @@ pub trait TableSchema: Send + Sync { None } - // Updated triggers some stuff downstream, but it is not supposed to block or fail, - // as the update itself is an unchangeable fact that will never go back - // due to CRDT logic. Typically errors in propagation of info should be logged - // to stderr. - fn updated(&self, _old: Option<&Self::E>, _new: Option<&Self::E>) {} + /// Actions triggered by data changing in a table. If such actions + /// include updates to the local database that should be applied + /// atomically with the item update itself, a db transaction is + /// provided on which these changes should be done. + /// This function can return a DB error but that's all. + fn updated( + &self, + _tx: &mut db::Transaction, + _old: Option<&Self::E>, + _new: Option<&Self::E>, + ) -> db::Result<()> { + Ok(()) + } fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool; } -- 2.43.0 From 9238fda9b245beb3ff535a16eda7859aed277bc6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 16:13:13 +0200 Subject: [PATCH 36/64] Update Cargo.nix --- Cargo.nix | 688 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 486 insertions(+), 202 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index d100e7bb..fec0f982 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4,6 +4,7 @@ args@{ release ? true, rootFeatures ? [ + "garage_db/default" "garage_util/default" "garage_rpc/default" "garage_table/default" @@ -45,6 +46,7 @@ in { cargo2nixVersion = "0.10.0"; workspace = { + garage_db = rustPackages.unknown.garage_db."0.8.0"; garage_util = rustPackages.unknown.garage_util."0.7.0"; garage_rpc = rustPackages.unknown.garage_rpc."0.7.0"; garage_table = rustPackages.unknown.garage_table."0.7.0"; @@ -55,6 +57,20 @@ in garage = rustPackages.unknown.garage."0.7.0"; k2v-client = rustPackages.unknown.k2v-client."0.1.0"; }; + "registry+https://github.com/rust-lang/crates.io-index".ahash."0.7.6" = overridableMkRustCrate (profileName: rec { + name = "ahash"; + version = "0.7.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"; }; + dependencies = { + ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "windows" || hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "ios" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "redox" || hostPlatform.parsed.kernel.name == "cloudabi" || hostPlatform.parsed.kernel.name == "haiku" || hostPlatform.parsed.kernel.name == "vxworks" || hostPlatform.parsed.kernel.name == "emscripten" || hostPlatform.parsed.kernel.name == "wasi" then "getrandom" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.5" { inherit profileName; }; + ${ if !((hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && hostPlatform.parsed.kernel.name == "none") then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; + }; + buildDependencies = { + version_check = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" = overridableMkRustCrate (profileName: rec { name = "aho-corasick"; version = "0.7.18"; @@ -433,6 +449,16 @@ in ]; }); + "registry+https://github.com/rust-lang/crates.io-index".bincode."1.3.3" = overridableMkRustCrate (profileName: rec { + name = "bincode"; + version = "1.3.3"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"; }; + dependencies = { + serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" = overridableMkRustCrate (profileName: rec { name = "bitflags"; version = "1.3.2"; @@ -479,6 +505,17 @@ in ]; }); + "registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.9.1" = overridableMkRustCrate (profileName: rec { + name = "bytemuck"; + version = "1.9.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc"; }; + features = builtins.concatLists [ + [ "extern_crate_alloc" ] + [ "extern_crate_std" ] + ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" = overridableMkRustCrate (profileName: rec { name = "byteorder"; version = "1.4.3"; @@ -541,6 +578,13 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".cfg-if."0.1.10" = overridableMkRustCrate (profileName: rec { + name = "cfg-if"; + version = "0.1.10"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" = overridableMkRustCrate (profileName: rec { name = "cfg-if"; version = "1.0.0"; @@ -561,22 +605,22 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"; }; features = builtins.concatLists [ - [ "clock" ] - [ "default" ] - [ "libc" ] - [ "oldtime" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "clock") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "libc") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "oldtime") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "serde") - [ "std" ] - [ "time" ] - [ "winapi" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "time") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "winapi") ]; dependencies = { - libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; - num_integer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }; - num_traits = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "num_integer" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "num_traits" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client" then "serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; - time = rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.1.44" { inherit profileName; }; - ${ if hostPlatform.isWindows then "winapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "time" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.1.44" { inherit profileName; }; + ${ if (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") && hostPlatform.isWindows then "winapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }; }; }); @@ -688,7 +732,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"; }; dependencies = { - ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; }; }); @@ -745,6 +789,32 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-queue."0.1.2" = overridableMkRustCrate (profileName: rec { + name = "crossbeam-queue"; + version = "0.1.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"; }; + dependencies = { + crossbeam_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.6.6" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.6.6" = overridableMkRustCrate (profileName: rec { + name = "crossbeam-utils"; + version = "0.6.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"; }; + features = builtins.concatLists [ + [ "default" ] + [ "lazy_static" ] + [ "std" ] + ]; + dependencies = { + cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."0.1.10" { inherit profileName; }; + lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.8" = overridableMkRustCrate (profileName: rec { name = "crossbeam-utils"; version = "0.8.8"; @@ -1018,6 +1088,24 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.2.0" = overridableMkRustCrate (profileName: rec { + name = "fallible-iterator"; + version = "0.2.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" = overridableMkRustCrate (profileName: rec { + name = "fallible-streaming-iterator"; + version = "0.1.9"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".fastrand."1.7.0" = overridableMkRustCrate (profileName: rec { name = "fastrand"; version = "1.7.0"; @@ -1214,32 +1302,32 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"; }; features = builtins.concatLists [ - [ "alloc" ] - [ "async-await" ] - [ "async-await-macro" ] - [ "channel" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "alloc") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "async-await") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "async-await-macro") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "channel") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") - [ "futures-channel" ] - [ "futures-io" ] - [ "futures-macro" ] - [ "futures-sink" ] - [ "io" ] - [ "memchr" ] - [ "sink" ] - [ "slab" ] - [ "std" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "futures-channel") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "futures-io") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "futures-macro") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "futures-sink") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "io") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "memchr") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "sink") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "slab") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "std") ]; dependencies = { - futures_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; - futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; - futures_io = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.21" { inherit profileName; }; - futures_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-macro."0.3.21" { profileName = "__noProfile"; }; - futures_sink = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.21" { inherit profileName; }; - futures_task = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.21" { inherit profileName; }; - memchr = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; - pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; - pin_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-utils."0.1.0" { inherit profileName; }; - slab = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_channel" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_io" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_macro" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-macro."0.3.21" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_sink" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_task" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "memchr" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_utils" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-utils."0.1.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "slab" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; }; }; }); @@ -1268,6 +1356,7 @@ in futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; garage_api = rustPackages."unknown".garage_api."0.7.0" { inherit profileName; }; + garage_db = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }; garage_model = rustPackages."unknown".garage_model."0.7.0" { inherit profileName; }; garage_rpc = rustPackages."unknown".garage_rpc."0.7.0" { inherit profileName; }; garage_table = rustPackages."unknown".garage_table."0.7.0" { inherit profileName; }; @@ -1285,7 +1374,6 @@ in rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; - sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; structopt = rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; toml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; }; @@ -1366,6 +1454,7 @@ in bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + garage_db = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }; garage_rpc = rustPackages."unknown".garage_rpc."0.7.0" { inherit profileName; }; garage_table = rustPackages."unknown".garage_table."0.7.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; @@ -1375,13 +1464,37 @@ in rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; - sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; zstd = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.2+zstd.1.5.1" { inherit profileName; }; }; }); + "unknown".garage_db."0.8.0" = overridableMkRustCrate (profileName: rec { + name = "garage_db"; + version = "0.8.0"; + registry = "unknown"; + src = fetchCrateLocal (workspaceSrc + "/src/db"); + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "garage_db") "clap") + (lib.optional (rootFeatures' ? "garage_db") "cli") + (lib.optional (rootFeatures' ? "garage_db") "pretty_env_logger") + ]; + dependencies = { + ${ if rootFeatures' ? "garage_db" then "clap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }; + err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }; + heed = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" { inherit profileName; }; + hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; + ${ if rootFeatures' ? "garage_db" then "pretty_env_logger" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; }; + rusqlite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.27.0" { inherit profileName; }; + sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; + }; + devDependencies = { + mktemp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.4.1" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".garage_model."0.5.1" = overridableMkRustCrate (profileName: rec { name = "garage_model"; version = "0.5.1"; @@ -1425,6 +1538,7 @@ in ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "futures" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_block" else null } = rustPackages."unknown".garage_block."0.7.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_db" else null } = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_model_050" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".garage_model."0.5.1" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_rpc" else null } = rustPackages."unknown".garage_rpc."0.7.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_table" else null } = rustPackages."unknown".garage_table."0.7.0" { inherit profileName; }; @@ -1436,7 +1550,6 @@ in ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "rmp_serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "serde_bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "sled" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "tracing" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "zstd" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.2+zstd.1.5.1" { inherit profileName; }; @@ -1546,6 +1659,7 @@ in bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + garage_db = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }; garage_rpc = rustPackages."unknown".garage_rpc."0.7.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }; @@ -1554,7 +1668,6 @@ in rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; - sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; @@ -1596,25 +1709,25 @@ in (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_util") "k2v") ]; dependencies = { - blake2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.9.2" { inherit profileName; }; - chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; }; - err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }; - futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; - hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; - http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; - hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }; - netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.4" { inherit profileName; }; - opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; - rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; - rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; - serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; - serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.81" { inherit profileName; }; - sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; }; - sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; - tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; - toml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; }; - tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; - xxhash_rust = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.4" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "blake2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.9.2" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "chrono" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "err_derive" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "garage_db" else null } = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "hex" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "http" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "hyper" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.4" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rmp_serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "serde_json" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.81" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "sha2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "toml" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tracing" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "xxhash_rust" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.4" { inherit profileName; }; }; }); @@ -1668,7 +1781,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"; }; features = builtins.concatLists [ - [ "std" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "std") ]; dependencies = { cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; @@ -1727,8 +1840,24 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"; }; features = builtins.concatLists [ - [ "raw" ] + [ "ahash" ] + [ "default" ] + [ "inline-more" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "raw") ]; + dependencies = { + ahash = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ahash."0.7.6" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".hashlink."0.7.0" = overridableMkRustCrate (profileName: rec { + name = "hashlink"; + version = "0.7.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"; }; + dependencies = { + hashbrown = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.11.2" { inherit profileName; }; + }; }); "registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" = overridableMkRustCrate (profileName: rec { @@ -1751,6 +1880,64 @@ in ]; }); + "registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" = overridableMkRustCrate (profileName: rec { + name = "heed"; + version = "0.11.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392"; }; + features = builtins.concatLists [ + [ "default" ] + [ "lmdb" ] + [ "lmdb-rkv-sys" ] + [ "serde" ] + [ "serde-bincode" ] + [ "serde-json" ] + ]; + dependencies = { + bytemuck = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.9.1" { inherit profileName; }; + byteorder = rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }; + heed_traits = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed-traits."0.8.0" { inherit profileName; }; + heed_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed-types."0.8.0" { inherit profileName; }; + libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + lmdb_sys = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lmdb-rkv-sys."0.11.2" { inherit profileName; }; + once_cell = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; + page_size = rustPackages."registry+https://github.com/rust-lang/crates.io-index".page_size."0.4.2" { inherit profileName; }; + serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; + synchronoise = rustPackages."registry+https://github.com/rust-lang/crates.io-index".synchronoise."1.0.0" { inherit profileName; }; + ${ if hostPlatform.isWindows then "url" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.2.2" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".heed-traits."0.8.0" = overridableMkRustCrate (profileName: rec { + name = "heed-traits"; + version = "0.8.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".heed-types."0.8.0" = overridableMkRustCrate (profileName: rec { + name = "heed-types"; + version = "0.8.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd"; }; + features = builtins.concatLists [ + [ "bincode" ] + [ "default" ] + [ "serde" ] + [ "serde-bincode" ] + [ "serde-json" ] + [ "serde_json" ] + ]; + dependencies = { + bincode = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bincode."1.3.3" { inherit profileName; }; + bytemuck = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.9.1" { inherit profileName; }; + byteorder = rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }; + heed_traits = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed-traits."0.8.0" { inherit profileName; }; + serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; + serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.81" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.1.19" = overridableMkRustCrate (profileName: rec { name = "hermit-abi"; version = "0.1.19"; @@ -1882,7 +2069,7 @@ in src = fetchCratesIo { inherit name version; sha256 = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2"; }; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "client") - [ "default" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "full") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "h2") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "http1") @@ -1894,22 +2081,22 @@ in (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "tcp") ]; dependencies = { - bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; - futures_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; - futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; - futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_channel" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "h2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; }; - http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; - http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; }; - httparse = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.6.0" { inherit profileName; }; - httpdate = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.2" { inherit profileName; }; - itoa = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; }; - pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "http" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "http_body" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "httparse" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.6.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "httpdate" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.2" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "itoa" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "socket2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.4" { inherit profileName; }; - tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; - tower_service = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.1" { inherit profileName; }; - tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; - want = rustPackages."registry+https://github.com/rust-lang/crates.io-index".want."0.3.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tower_service" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tracing" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "want" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".want."0.3.0" { inherit profileName; }; }; }); @@ -2376,6 +2563,23 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.24.2" = overridableMkRustCrate (profileName: rec { + name = "libsqlite3-sys"; + version = "0.24.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"; }; + features = builtins.concatLists [ + [ "default" ] + [ "min_sqlite_version_3_6_8" ] + [ "pkg-config" ] + [ "vcpkg" ] + ]; + buildDependencies = { + pkg_config = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.24" { profileName = "__noProfile"; }; + vcpkg = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".linked-hash-map."0.5.4" = overridableMkRustCrate (profileName: rec { name = "linked-hash-map"; version = "0.5.4"; @@ -2383,6 +2587,23 @@ in src = fetchCratesIo { inherit name version; sha256 = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"; }; }); + "registry+https://github.com/rust-lang/crates.io-index".lmdb-rkv-sys."0.11.2" = overridableMkRustCrate (profileName: rec { + name = "lmdb-rkv-sys"; + version = "0.11.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + }; + buildDependencies = { + cc = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.73" { profileName = "__noProfile"; }; + pkg_config = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.24" { profileName = "__noProfile"; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.6" = overridableMkRustCrate (profileName: rec { name = "lock_api"; version = "0.4.6"; @@ -2399,7 +2620,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage") "std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_db") "std") ]; dependencies = { cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; @@ -2512,6 +2733,16 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".mktemp."0.4.1" = overridableMkRustCrate (profileName: rec { + name = "mktemp"; + version = "0.4.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "975de676448231fcde04b9149d2543077e166b78fc29eae5aa219e7928410da2"; }; + dependencies = { + uuid = rustPackages."registry+https://github.com/rust-lang/crates.io-index".uuid."0.8.2" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".multer."2.0.2" = overridableMkRustCrate (profileName: rec { name = "multer"; version = "2.0.2"; @@ -2594,31 +2825,31 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "c6419a4b836774192e13fedb05c0e5f414ee8df9ca0c467456a0bacde06c29ee"; }; features = builtins.concatLists [ - [ "default" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "opentelemetry") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "opentelemetry-contrib") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rand") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "telemetry") ]; dependencies = { - arc_swap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }; - async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }; - bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."0.6.0" { inherit profileName; }; - cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; - err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.2.4" { profileName = "__noProfile"; }; - futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; - hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; - kuska_handshake = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" { inherit profileName; }; - sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "arc_swap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "async_trait" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."0.6.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "cfg_if" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "err_derive" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.2.4" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "hex" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "kuska_handshake" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "sodiumoxide" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "log" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "opentelemetry_contrib" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-contrib."0.9.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.5.6" { inherit profileName; }; - rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.14.4" { inherit profileName; }; - serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; - tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; - tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }; - tokio_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.9" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rmp_serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.14.4" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio_stream" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.9" { inherit profileName; }; }; }); @@ -2662,10 +2893,10 @@ in (lib.optional (rootFeatures' ? "garage") "std") ]; dependencies = { - num_traits = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "num_traits" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }; }; buildDependencies = { - autocfg = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "autocfg" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }; }; }); @@ -2900,6 +3131,17 @@ in ]; }); + "registry+https://github.com/rust-lang/crates.io-index".page_size."0.4.2" = overridableMkRustCrate (profileName: rec { + name = "page_size"; + version = "0.4.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd"; }; + dependencies = { + ${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + ${ if hostPlatform.isWindows then "winapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" = overridableMkRustCrate (profileName: rec { name = "parking_lot"; version = "0.11.2"; @@ -3320,19 +3562,19 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"; }; features = builtins.concatLists [ - [ "alloc" ] - [ "default" ] - [ "getrandom" ] - [ "libc" ] - [ "rand_chacha" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "alloc") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "getrandom") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "libc") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "rand_chacha") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "small_rng") - [ "std" ] - [ "std_rng" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "std_rng") ]; dependencies = { - ${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; - rand_chacha = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_chacha."0.3.1" { inherit profileName; }; - rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.3" { inherit profileName; }; + ${ if (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") && hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rand_chacha" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_chacha."0.3.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rand_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.3" { inherit profileName; }; }; }); @@ -3418,28 +3660,28 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "aho-corasick") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "default") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "memchr") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf-cache") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf-dfa") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf-inline") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf-literal") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "std") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-age") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-bool") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-case") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-gencat") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-perl") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-script") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-segment") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "aho-corasick") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "memchr") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "perf") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "perf-cache") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "perf-dfa") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "perf-inline") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "perf-literal") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "unicode") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "unicode-age") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "unicode-bool") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "unicode-case") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "unicode-gencat") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "unicode-perl") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "unicode-script") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web") "unicode-segment") ]; dependencies = { - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "aho_corasick" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "memchr" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "regex_syntax" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.25" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web" then "aho_corasick" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_web" then "memchr" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_db" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "regex_syntax" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.25" { inherit profileName; }; }; }); @@ -3484,7 +3726,7 @@ in ]; dependencies = { ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; }; untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; }; @@ -3625,6 +3867,22 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.27.0" = overridableMkRustCrate (profileName: rec { + name = "rusqlite"; + version = "0.27.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a"; }; + dependencies = { + bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; + fallible_iterator = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.2.0" { inherit profileName; }; + fallible_streaming_iterator = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" { inherit profileName; }; + hashlink = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashlink."0.7.0" { inherit profileName; }; + libsqlite3_sys = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.24.2" { inherit profileName; }; + memchr = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; + smallvec = rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.8.0" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" = overridableMkRustCrate (profileName: rec { name = "rustc_version"; version = "0.4.0"; @@ -3807,7 +4065,7 @@ in features = builtins.concatLists [ [ "default" ] [ "derive" ] - [ "rc" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "rc") [ "serde_derive" ] [ "std" ] ]; @@ -4136,7 +4394,7 @@ in [ "proc-macro" ] [ "quote" ] [ "visit" ] - [ "visit-mut" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "visit-mut") ]; dependencies = { proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; }; @@ -4145,6 +4403,16 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".synchronoise."1.0.0" = overridableMkRustCrate (profileName: rec { + name = "synchronoise"; + version = "1.0.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb"; }; + dependencies = { + crossbeam_queue = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-queue."0.1.2" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".synstructure."0.12.6" = overridableMkRustCrate (profileName: rec { name = "synstructure"; version = "0.12.6"; @@ -4283,44 +4551,44 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"; }; features = builtins.concatLists [ - [ "bytes" ] - [ "default" ] - [ "fs" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "bytes") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "fs") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "full") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "io-std") - [ "io-util" ] - [ "libc" ] - [ "macros" ] - [ "memchr" ] - [ "mio" ] - [ "net" ] - [ "num_cpus" ] - [ "once_cell" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "io-util") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "libc") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "macros") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "memchr") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "mio") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "net") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "num_cpus") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "once_cell") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "parking_lot") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "process") - [ "rt" ] - [ "rt-multi-thread" ] - [ "signal" ] - [ "signal-hook-registry" ] - [ "socket2" ] - [ "sync" ] - [ "time" ] - [ "tokio-macros" ] - [ "winapi" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "rt") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "rt-multi-thread") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "signal") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "signal-hook-registry") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "socket2") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "sync") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "time") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "tokio-macros") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "winapi") ]; dependencies = { - bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; - ${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; - memchr = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; - mio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mio."0.8.2" { inherit profileName; }; - num_cpus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.13.1" { inherit profileName; }; - once_cell = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + ${ if (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") && hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "memchr" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "mio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mio."0.8.2" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "num_cpus" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.13.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.0" { inherit profileName; }; - pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; - ${ if hostPlatform.isUnix then "signal_hook_registry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.0" { inherit profileName; }; - socket2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.4" { inherit profileName; }; - tokio_macros = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-macros."1.7.0" { profileName = "__noProfile"; }; - ${ if hostPlatform.isWindows then "winapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; + ${ if (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") && hostPlatform.isUnix then "signal_hook_registry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "socket2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.4" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio_macros" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-macros."1.7.0" { profileName = "__noProfile"; }; + ${ if (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") && hostPlatform.isWindows then "winapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }; }; }); @@ -4376,14 +4644,14 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"; }; features = builtins.concatLists [ - [ "default" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "net") - [ "time" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "time") ]; dependencies = { - futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; - pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; - tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; }; }); @@ -4394,22 +4662,22 @@ in src = fetchCratesIo { inherit name version; sha256 = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"; }; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "codec") - [ "compat" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "compat") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default") - [ "futures-io" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "futures-io") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc") "io") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc") "slab") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc") "time") ]; dependencies = { - bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; - futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; - futures_io = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.21" { inherit profileName; }; - futures_sink = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.21" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; - pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_io" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_sink" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "log" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "slab" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; }; - tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; }; }); @@ -4599,19 +4867,19 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"; }; features = builtins.concatLists [ - [ "attributes" ] - [ "default" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "attributes") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "log") (lib.optional (rootFeatures' ? "garage") "log-always") - [ "std" ] - [ "tracing-attributes" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "tracing-attributes") ]; dependencies = { - cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "cfg_if" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "log" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; - pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; - tracing_attributes = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-attributes."0.1.20" { profileName = "__noProfile"; }; - tracing_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.23" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tracing_attributes" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-attributes."0.1.20" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tracing_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.23" { inherit profileName; }; }; }); @@ -4758,6 +5026,22 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".uuid."0.8.2" = overridableMkRustCrate (profileName: rec { + name = "uuid"; + version = "0.8.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"; }; + features = builtins.concatLists [ + [ "default" ] + [ "getrandom" ] + [ "std" ] + [ "v4" ] + ]; + dependencies = { + getrandom = rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.5" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" = overridableMkRustCrate (profileName: rec { name = "vcpkg"; version = "0.2.15"; @@ -4940,49 +5224,49 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"; }; features = builtins.concatLists [ - [ "cfg" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "cfg") [ "consoleapi" ] [ "errhandlingapi" ] - [ "evntrace" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "evntrace") [ "fileapi" ] [ "handleapi" ] - [ "in6addr" ] - [ "inaddr" ] - [ "ioapiset" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "in6addr") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "inaddr") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "ioapiset") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "knownfolders") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "lmcons") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "minschannel") [ "minwinbase" ] [ "minwindef" ] - [ "mswsock" ] - [ "namedpipeapi" ] - [ "ntdef" ] - [ "ntsecapi" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "mswsock") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "namedpipeapi") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "ntdef") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "ntsecapi") [ "ntstatus" ] (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "objbase") [ "processenv" ] [ "processthreadsapi" ] - [ "profileapi" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "profileapi") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "schannel") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "securitybaseapi") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "shlobj") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "sspi") [ "std" ] - [ "synchapi" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "synchapi") [ "sysinfoapi" ] (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "threadpoollegacyapiset") - [ "timezoneapi" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "timezoneapi") [ "winbase" ] [ "wincon" ] (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "wincrypt") - [ "windef" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "windef") [ "winerror" ] - [ "winioctl" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "winioctl") [ "winnt" ] - [ "winsock2" ] - [ "ws2def" ] - [ "ws2ipdef" ] - [ "ws2tcpip" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "winsock2") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "ws2def") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "ws2ipdef") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "ws2tcpip") (lib.optional (rootFeatures' ? "garage") "wtypesbase") ]; dependencies = { @@ -5030,11 +5314,11 @@ in [ "default" ] ]; dependencies = { - ${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; }; }); -- 2.43.0 From 1dabd98330cedd108c42b7613a1f29fc4e4123ad Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 16:29:02 +0200 Subject: [PATCH 37/64] Improve things in block manager & correctly propagate .len() errors --- src/block/manager.rs | 54 +++++++++++++++++++++++++++++--------------- src/garage/admin.rs | 44 +++++++++++++++++++++++------------- src/table/data.rs | 4 ++-- src/table/merkle.rs | 8 +++---- 4 files changed, 70 insertions(+), 40 deletions(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index b7dcaf8a..ea984646 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -1,3 +1,5 @@ +use core::ops::Bound; + use std::convert::TryInto; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -218,19 +220,35 @@ impl BlockManager { /// to fix any mismatch between the two. pub async fn repair_data_store(&self, must_exit: &watch::Receiver) -> Result<(), Error> { // 1. Repair blocks from RC table. - // TODO don't do this like this - let mut hashes = vec![]; - for (i, entry) in self.rc.rc.iter()?.enumerate() { - let (hash, _) = entry?; - let hash = Hash::try_from(&hash[..]).unwrap(); - hashes.push(hash); - if i & 0xFF == 0 && *must_exit.borrow() { - return Ok(()); + let mut next_start: Option = None; + loop { + let mut batch_of_hashes = vec![]; + let start_bound = match next_start.as_ref() { + None => Bound::Unbounded, + Some(x) => Bound::Excluded(x.as_slice()), + }; + for entry in self + .rc + .rc + .range::<&[u8], _>((start_bound, Bound::Unbounded))? + { + let (hash, _) = entry?; + let hash = Hash::try_from(&hash[..]).unwrap(); + batch_of_hashes.push(hash); + if batch_of_hashes.len() >= 1000 { + break; + } } - } - for (i, hash) in hashes.into_iter().enumerate() { - self.put_to_resync(&hash, Duration::from_secs(0))?; - if i & 0xFF == 0 && *must_exit.borrow() { + if batch_of_hashes.is_empty() { + break; + } + + for hash in batch_of_hashes.into_iter() { + self.put_to_resync(&hash, Duration::from_secs(0))?; + next_start = Some(hash) + } + + if *must_exit.borrow() { return Ok(()); } } @@ -271,18 +289,18 @@ impl BlockManager { } /// Get lenght of resync queue - pub fn resync_queue_len(&self) -> usize { - self.resync_queue.len().unwrap() // TODO fix unwrap + pub fn resync_queue_len(&self) -> Result { + Ok(self.resync_queue.len()?) } /// Get number of blocks that have an error - pub fn resync_errors_len(&self) -> usize { - self.resync_errors.len().unwrap() // TODO fix unwrap + pub fn resync_errors_len(&self) -> Result { + Ok(self.resync_errors.len()?) } /// Get number of items in the refcount table - pub fn rc_len(&self) -> usize { - self.rc.rc.len().unwrap() // TODO fix unwrap + pub fn rc_len(&self) -> Result { + Ok(self.rc.rc.len()?) } //// ----- Managing the reference counter ---- diff --git a/src/garage/admin.rs b/src/garage/admin.rs index cce88b35..3af8b046 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -660,11 +660,11 @@ impl AdminRpcHandler { } Ok(AdminRpc::Ok(ret)) } else { - Ok(AdminRpc::Ok(self.gather_stats_local(opt))) + Ok(AdminRpc::Ok(self.gather_stats_local(opt)?)) } } - fn gather_stats_local(&self, opt: StatsOpt) -> String { + fn gather_stats_local(&self, opt: StatsOpt) -> Result { let mut ret = String::new(); writeln!( &mut ret, @@ -689,59 +689,71 @@ impl AdminRpcHandler { writeln!(&mut ret, " {:?} {}", n, c).unwrap(); } - self.gather_table_stats(&mut ret, &self.garage.bucket_table, &opt); - self.gather_table_stats(&mut ret, &self.garage.key_table, &opt); - self.gather_table_stats(&mut ret, &self.garage.object_table, &opt); - self.gather_table_stats(&mut ret, &self.garage.version_table, &opt); - self.gather_table_stats(&mut ret, &self.garage.block_ref_table, &opt); + self.gather_table_stats(&mut ret, &self.garage.bucket_table, &opt)?; + self.gather_table_stats(&mut ret, &self.garage.key_table, &opt)?; + self.gather_table_stats(&mut ret, &self.garage.object_table, &opt)?; + self.gather_table_stats(&mut ret, &self.garage.version_table, &opt)?; + self.gather_table_stats(&mut ret, &self.garage.block_ref_table, &opt)?; writeln!(&mut ret, "\nBlock manager stats:").unwrap(); if opt.detailed { writeln!( &mut ret, " number of RC entries (~= number of blocks): {}", - self.garage.block_manager.rc_len() + self.garage.block_manager.rc_len()? ) .unwrap(); } writeln!( &mut ret, " resync queue length: {}", - self.garage.block_manager.resync_queue_len() + self.garage.block_manager.resync_queue_len()? ) .unwrap(); writeln!( &mut ret, " blocks with resync errors: {}", - self.garage.block_manager.resync_errors_len() + self.garage.block_manager.resync_errors_len()? ) .unwrap(); - ret + Ok(ret) } - fn gather_table_stats(&self, to: &mut String, t: &Arc>, opt: &StatsOpt) + fn gather_table_stats( + &self, + to: &mut String, + t: &Arc>, + opt: &StatsOpt, + ) -> Result<(), Error> where F: TableSchema + 'static, R: TableReplication + 'static, { writeln!(to, "\nTable stats for {}", F::TABLE_NAME).unwrap(); if opt.detailed { - writeln!(to, " number of items: {}", t.data.store.len().unwrap()).unwrap(); // TODO fix len unwrap + writeln!( + to, + " number of items: {}", + t.data.store.len().map_err(GarageError::from)? + ) + .unwrap(); writeln!( to, " Merkle tree size: {}", - t.merkle_updater.merkle_tree_len() + t.merkle_updater.merkle_tree_len()? ) .unwrap(); } writeln!( to, " Merkle updater todo queue length: {}", - t.merkle_updater.todo_len() + t.merkle_updater.todo_len()? ) .unwrap(); - writeln!(to, " GC todo queue length: {}", t.data.gc_todo_len()).unwrap(); + writeln!(to, " GC todo queue length: {}", t.data.gc_todo_len()?).unwrap(); + + Ok(()) } } diff --git a/src/table/data.rs b/src/table/data.rs index e688168f..839dae94 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -318,7 +318,7 @@ where } } - pub fn gc_todo_len(&self) -> usize { - self.gc_todo.len().unwrap() // TODO fix unwrap + pub fn gc_todo_len(&self) -> Result { + Ok(self.gc_todo.len()?) } } diff --git a/src/table/merkle.rs b/src/table/merkle.rs index e7f2442e..7685b193 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -316,12 +316,12 @@ where MerkleNode::decode_opt(&ent) } - pub fn merkle_tree_len(&self) -> usize { - self.data.merkle_tree.len().unwrap() // TODO fix unwrap + pub fn merkle_tree_len(&self) -> Result { + Ok(self.data.merkle_tree.len()?) } - pub fn todo_len(&self) -> usize { - self.data.merkle_todo.len().unwrap() // TODO fix unwrap + pub fn todo_len(&self) -> Result { + Ok(self.data.merkle_todo.len()?) } } -- 2.43.0 From 72e6419b1b43fed5381b94c6ec01c53de34e5b17 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 16:31:43 +0200 Subject: [PATCH 38/64] add comment --- src/block/manager.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/block/manager.rs b/src/block/manager.rs index ea984646..17b45935 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -222,6 +222,15 @@ impl BlockManager { // 1. Repair blocks from RC table. let mut next_start: Option = None; loop { + // We have to do this complicated two-step process where we first read a bunch + // of hashes from the RC table, and then insert them in the to-resync queue, + // because of SQLite. Basically, as long as we have an iterator on a DB table, + // we can't do anything else on the DB. The naive approach (which we had previously) + // of just iterating on the RC table and inserting items one to one in the resync + // queue can't work here, it would just provoke a deadlock in the SQLite adapter code. + // This is mostly because the Rust bindings for SQLite assume a worst-case scenario + // where SQLite is not compiled in thread-safe mode, so we have to wrap everything + // in a mutex (see db/sqlite_adapter.rs). let mut batch_of_hashes = vec![]; let start_bound = match next_start.as_ref() { None => Bound::Unbounded, -- 2.43.0 From bbf68aa039678d844e7d88db1a23a128f877ba43 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 16:32:06 +0200 Subject: [PATCH 39/64] fix --- src/block/manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index 17b45935..48aed75d 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -230,7 +230,7 @@ impl BlockManager { // queue can't work here, it would just provoke a deadlock in the SQLite adapter code. // This is mostly because the Rust bindings for SQLite assume a worst-case scenario // where SQLite is not compiled in thread-safe mode, so we have to wrap everything - // in a mutex (see db/sqlite_adapter.rs). + // in a mutex (see db/sqlite_adapter.rs and discussion in PR #322). let mut batch_of_hashes = vec![]; let start_bound = match next_start.as_ref() { None => Bound::Unbounded, -- 2.43.0 From c3c8d59c996061a2a4280820915a6dcd94cf496c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 16:37:44 +0200 Subject: [PATCH 40/64] Fix more .unwrap()'s --- src/block/metrics.rs | 8 ++++++-- src/table/metrics.rs | 20 ++++++++++++-------- src/table/sync.rs | 13 ++++++++++--- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/block/metrics.rs b/src/block/metrics.rs index 1d4d0028..0bd50a18 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -28,7 +28,9 @@ impl BlockManagerMetrics { Self { _resync_queue_len: meter .u64_value_observer("block.resync_queue_length", move |observer| { - observer.observe(resync_queue.len().unwrap() as u64, &[]) // TODO fix unwrap + if let Ok(v) = resync_queue.len() { + observer.observe(v as u64, &[]); + } }) .with_description( "Number of block hashes queued for local check and possible resync", @@ -36,7 +38,9 @@ impl BlockManagerMetrics { .init(), _resync_errored_blocks: meter .u64_value_observer("block.resync_errored_blocks", move |observer| { - observer.observe(resync_errors.len().unwrap() as u64, &[]) // TODO fix unwrap + if let Ok(v) = resync_errors.len() { + observer.observe(v as u64, &[]); + } }) .with_description("Number of block hashes whose last resync resulted in an error") .init(), diff --git a/src/table/metrics.rs b/src/table/metrics.rs index 3318de88..13baf4c6 100644 --- a/src/table/metrics.rs +++ b/src/table/metrics.rs @@ -26,10 +26,12 @@ impl TableMetrics { .u64_value_observer( "table.merkle_updater_todo_queue_length", move |observer| { - observer.observe( - merkle_todo.len().unwrap() as u64, // TODO fix unwrap - &[KeyValue::new("table_name", table_name)], - ) + if let Ok(v) = merkle_todo.len() { + observer.observe( + v as u64, + &[KeyValue::new("table_name", table_name)], + ); + } }, ) .with_description("Merkle tree updater TODO queue length") @@ -38,10 +40,12 @@ impl TableMetrics { .u64_value_observer( "table.gc_todo_queue_length", move |observer| { - observer.observe( - gc_todo.len().unwrap() as u64, // TODO fix unwrap - &[KeyValue::new("table_name", table_name)], - ) + if let Ok(v) = gc_todo.len() { + observer.observe( + v as u64, + &[KeyValue::new("table_name", table_name)], + ); + } }, ) .with_description("Table garbage collector TODO queue length") diff --git a/src/table/sync.rs b/src/table/sync.rs index 20066d73..4c83e991 100644 --- a/src/table/sync.rs +++ b/src/table/sync.rs @@ -603,9 +603,16 @@ impl SyncTodo { let retain = nodes.contains(&my_id); if !retain { // Check if we have some data to send, otherwise skip - if data.store.range(begin..end).unwrap().next().is_none() { - // TODO fix unwrap - continue; + match data.store.range(begin..end) { + Ok(mut iter) => { + if iter.next().is_none() { + continue; + } + } + Err(e) => { + warn!("DB error in add_full_sync: {}", e); + continue; + } } } -- 2.43.0 From fd2fe18de3eb245aaab78abf924301e3d4a1734f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 16:44:47 +0200 Subject: [PATCH 41/64] in db convert: map size up to 1TB on 64-bit systems --- src/db/bin/convert.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs index 636bd834..e5778abb 100644 --- a/src/db/bin/convert.rs +++ b/src/db/bin/convert.rs @@ -54,9 +54,17 @@ fn open_db(path: PathBuf, engine: String) -> Result { std::fs::create_dir_all(&path).map_err(|e| { Error(format!("Unable to create LMDB data directory: {}", e).into()) })?; + + let map_size = if u32::MAX as usize == usize::MAX { + eprintln!("LMDB is not recommended on 32-bit systems, database size will be limited"); + 1usize << 30 // 1GB for 32-bit systems + } else { + 1usize << 40 // 1TB for 64-bit systems + }; + let db = lmdb_adapter::heed::EnvOpenOptions::new() .max_dbs(100) - .map_size(1usize << 30) + .map_size(map_size) .open(&path) .unwrap(); Ok(lmdb_adapter::LmdbDb::init(db)) -- 2.43.0 From 72a87c86867e6d440ff810513f572bb7d2a646b6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 16:51:00 +0200 Subject: [PATCH 42/64] Add inline marker on a bunch of db functions --- src/db/bin/convert.rs | 4 +++- src/db/lib.rs | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs index e5778abb..9e45e61f 100644 --- a/src/db/bin/convert.rs +++ b/src/db/bin/convert.rs @@ -56,7 +56,9 @@ fn open_db(path: PathBuf, engine: String) -> Result { })?; let map_size = if u32::MAX as usize == usize::MAX { - eprintln!("LMDB is not recommended on 32-bit systems, database size will be limited"); + eprintln!( + "LMDB is not recommended on 32-bit systems, database size will be limited" + ); 1usize << 30 // 1GB for 32-bit systems } else { 1usize << 40 // 1TB for 64-bit systems diff --git a/src/db/lib.rs b/src/db/lib.rs index b9c27d9a..afea9f55 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -139,40 +139,50 @@ impl Db { #[allow(clippy::len_without_is_empty)] impl Tree { + #[inline] pub fn db(&self) -> Db { Db(self.0.clone()) } + #[inline] pub fn get>(&self, key: T) -> Result> { self.0.get(self.1, key.as_ref()) } + #[inline] pub fn len(&self) -> Result { self.0.len(self.1) } + #[inline] pub fn first(&self) -> Result> { self.iter()?.next().transpose() } + #[inline] pub fn get_gt>(&self, from: T) -> Result> { self.range((Bound::Excluded(from), Bound::Unbounded))? .next() .transpose() } + #[inline] pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { self.0.insert(self.1, key.as_ref(), value.as_ref()) } + #[inline] pub fn remove>(&self, key: T) -> Result { self.0.remove(self.1, key.as_ref()) } + #[inline] pub fn iter(&self) -> Result> { self.0.iter(self.1) } + #[inline] pub fn iter_rev(&self) -> Result> { self.0.iter_rev(self.1) } + #[inline] pub fn range(&self, range: R) -> Result> where K: AsRef<[u8]>, @@ -182,6 +192,7 @@ impl Tree { let eb = range.end_bound(); self.0.range(self.1, get_bound(sb), get_bound(eb)) } + #[inline] pub fn range_rev(&self, range: R) -> Result> where K: AsRef<[u8]>, @@ -195,13 +206,16 @@ impl Tree { #[allow(clippy::len_without_is_empty)] impl<'a> Transaction<'a> { + #[inline] pub fn get>(&self, tree: &Tree, key: T) -> Result> { self.0.get(tree.1, key.as_ref()) } + #[inline] pub fn len(&self, tree: &Tree) -> Result { self.0.len(tree.1) } + #[inline] pub fn insert, U: AsRef<[u8]>>( &mut self, tree: &Tree, @@ -210,17 +224,21 @@ impl<'a> Transaction<'a> { ) -> Result<()> { self.0.insert(tree.1, key.as_ref(), value.as_ref()) } + #[inline] pub fn remove>(&mut self, tree: &Tree, key: T) -> Result { self.0.remove(tree.1, key.as_ref()) } + #[inline] pub fn iter(&self, tree: &Tree) -> Result> { self.0.iter(tree.1) } + #[inline] pub fn iter_rev(&self, tree: &Tree) -> Result> { self.0.iter_rev(tree.1) } + #[inline] pub fn range(&self, tree: &Tree, range: R) -> Result> where K: AsRef<[u8]>, @@ -230,6 +248,7 @@ impl<'a> Transaction<'a> { let eb = range.end_bound(); self.0.range(tree.1, get_bound(sb), get_bound(eb)) } + #[inline] pub fn range_rev(&self, tree: &Tree, range: R) -> Result> where K: AsRef<[u8]>, @@ -242,10 +261,12 @@ impl<'a> Transaction<'a> { // ---- + #[inline] pub fn abort(self, e: E) -> TxResult { Err(TxError::Abort(e)) } + #[inline] pub fn commit(self, r: R) -> TxResult { Ok(r) } -- 2.43.0 From 1f9a371b9a51ac0288b0137660b06a171853909b Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 17:01:49 +0200 Subject: [PATCH 43/64] Fix clippy lint --- src/block/manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index 48aed75d..8abff1b1 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -1008,7 +1008,7 @@ impl ErrorCounter { } } - fn decode(data: &db::Value) -> Self { + fn decode(data: &[u8]) -> Self { Self { errors: u64::from_be_bytes(data[0..8].try_into().unwrap()), last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()), -- 2.43.0 From e42c7bed386f604b5beeb0a4330defacff059233 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 17:06:22 +0200 Subject: [PATCH 44/64] Remove useless clones in sqlite --- src/db/sqlite_adapter.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index f0bca257..0c8a0746 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -48,10 +48,10 @@ impl SqliteDb { } impl SqliteDbInner { - fn get_tree(&self, i: usize) -> Result { + fn get_tree(&self, i: usize) -> Result<&'_ str> { self.trees .get(i) - .cloned() + .map(String::as_str) .ok_or_else(|| Error("invalid tree id".into())) } } @@ -269,8 +269,8 @@ struct SqliteTx<'a> { } impl<'a> SqliteTx<'a> { - fn get_tree(&self, i: usize) -> Result { - self.trees.get(i).cloned().ok_or_else(|| { + fn get_tree(&self, i: usize) -> Result<&'_ str> { + self.trees.get(i).map(String::as_ref).ok_or_else(|| { Error( "invalid tree id (it might have been openned after the transaction started)".into(), ) -- 2.43.0 From 35544cac3994449b331a085d957cc476b4f2e79c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 22:02:09 +0200 Subject: [PATCH 45/64] Add nix overrides to include lmdb and sqlite3 libraries --- default.nix | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/default.nix b/default.nix index de996ac1..4dc2db2f 100644 --- a/default.nix +++ b/default.nix @@ -95,6 +95,25 @@ in let features = if release then [ "kubernetes-discovery" ] else []; }; }) + + /* + We are now depending on sqlite3 and lmdb to provide alternative + storage backends for Garage. But these crates don't know what + system libraries they need, so we add them here manually: + `sqlite3` for `libsqlite3-sys` and `lmdb` for `lmdb-rkv-sys` + */ + (pkgs.rustBuilder.rustLib.makeOverride { + name = "libsqlite3-sys"; + overrideAttrs = drv: { + propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [ pkgs.sqlite ]; + }; + }) + (pkgs.rustBuilder.rustLib.makeOverride { + name = "lmdb-rkv-sys"; + overrideAttrs = drv: { + propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [ pkgs.lmdb ]; + }; + }) ]; packageFun = import ./Cargo.nix; -- 2.43.0 From 3de9320a0ed61d2ebfc4c7a2bfc34d846a581e9a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 22:35:46 +0200 Subject: [PATCH 46/64] bundled sqlite --- default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index 4dc2db2f..1a2386cf 100644 --- a/default.nix +++ b/default.nix @@ -104,8 +104,8 @@ in let */ (pkgs.rustBuilder.rustLib.makeOverride { name = "libsqlite3-sys"; - overrideAttrs = drv: { - propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [ pkgs.sqlite ]; + overrideArgs = old: { + features = old.features or [ ] ++ [ "bundled" ]; }; }) (pkgs.rustBuilder.rustLib.makeOverride { -- 2.43.0 From 5dbc79b77ebaec3778791f9882266fc809217f62 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Jun 2022 23:00:10 +0200 Subject: [PATCH 47/64] Try once again to fix sqlite bundling in nix --- Cargo.lock | 1 + Cargo.nix | 16 ++++++++++++---- default.nix | 19 ------------------- src/db/Cargo.toml | 2 +- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23e4f468..11aa070d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1893,6 +1893,7 @@ version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" dependencies = [ + "cc", "pkg-config", "vcpkg", ] diff --git a/Cargo.nix b/Cargo.nix index fec0f982..335651fc 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -732,7 +732,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"; }; dependencies = { - ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.config == "aarch64-linux-android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; }; }); @@ -2569,12 +2569,16 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"; }; features = builtins.concatLists [ + [ "bundled" ] + [ "bundled_bindings" ] + [ "cc" ] [ "default" ] [ "min_sqlite_version_3_6_8" ] [ "pkg-config" ] [ "vcpkg" ] ]; buildDependencies = { + cc = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.73" { profileName = "__noProfile"; }; pkg_config = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.24" { profileName = "__noProfile"; }; vcpkg = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }; }; @@ -2714,7 +2718,7 @@ in [ "os-poll" ] ]; dependencies = { - ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; ${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; }; ${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; }; @@ -3726,7 +3730,7 @@ in ]; dependencies = { ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; }; untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; }; @@ -3872,6 +3876,10 @@ in version = "0.27.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a"; }; + features = builtins.concatLists [ + [ "bundled" ] + [ "modern_sqlite" ] + ]; dependencies = { bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; fallible_iterator = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.2.0" { inherit profileName; }; @@ -5317,7 +5325,7 @@ in ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; }; }); diff --git a/default.nix b/default.nix index 1a2386cf..de996ac1 100644 --- a/default.nix +++ b/default.nix @@ -95,25 +95,6 @@ in let features = if release then [ "kubernetes-discovery" ] else []; }; }) - - /* - We are now depending on sqlite3 and lmdb to provide alternative - storage backends for Garage. But these crates don't know what - system libraries they need, so we add them here manually: - `sqlite3` for `libsqlite3-sys` and `lmdb` for `lmdb-rkv-sys` - */ - (pkgs.rustBuilder.rustLib.makeOverride { - name = "libsqlite3-sys"; - overrideArgs = old: { - features = old.features or [ ] ++ [ "bundled" ]; - }; - }) - (pkgs.rustBuilder.rustLib.makeOverride { - name = "lmdb-rkv-sys"; - overrideAttrs = drv: { - propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [ pkgs.lmdb ]; - }; - }) ]; packageFun = import ./Cargo.nix; diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 1a24979f..6d8f64be 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -22,7 +22,7 @@ hexdump = "0.1" log = "0.4" heed = "0.11" -rusqlite = "0.27" +rusqlite = { version = "0.27", features = ["bundled"] } sled = "0.34" # cli deps -- 2.43.0 From a9e79f848b704347601e082c8065c3ce042b283b Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 16:52:57 +0200 Subject: [PATCH 48/64] Bring back the counted tree hack for Sled (with caveat) caveat: it's not only for sled --- src/block/manager.rs | 17 +++-- src/block/metrics.rs | 12 ++-- src/db/counted_tree_hack.rs | 130 ++++++++++++++++++++++++++++++++++++ src/db/lib.rs | 16 +++-- src/db/lmdb_adapter.rs | 10 +-- src/db/sled_adapter.rs | 12 ++-- src/db/sqlite_adapter.rs | 43 +++++++----- src/table/data.rs | 6 +- src/table/gc.rs | 20 +++--- src/table/metrics.rs | 13 ++-- src/util/sled_counter.rs | 100 --------------------------- 11 files changed, 215 insertions(+), 164 deletions(-) create mode 100644 src/db/counted_tree_hack.rs delete mode 100644 src/util/sled_counter.rs diff --git a/src/block/manager.rs b/src/block/manager.rs index 8abff1b1..da86a2d5 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -20,6 +20,7 @@ use opentelemetry::{ }; use garage_db as db; +use garage_db::counted_tree_hack::CountedTree; use garage_util::data::*; use garage_util::error::*; @@ -94,9 +95,9 @@ pub struct BlockManager { rc: BlockRc, - resync_queue: db::Tree, + resync_queue: CountedTree, resync_notify: Notify, - resync_errors: db::Tree, + resync_errors: CountedTree, system: Arc, endpoint: Arc>, @@ -126,10 +127,14 @@ impl BlockManager { let resync_queue = db .open_tree("block_local_resync_queue") .expect("Unable to open block_local_resync_queue tree"); + let resync_queue = + CountedTree::new(resync_queue).expect("Could not count block_local_resync_queue"); let resync_errors = db .open_tree("block_local_resync_errors") .expect("Unable to open block_local_resync_errors tree"); + let resync_errors = + CountedTree::new(resync_errors).expect("Could not count block_local_resync_errors"); let endpoint = system .netapp @@ -299,12 +304,16 @@ impl BlockManager { /// Get lenght of resync queue pub fn resync_queue_len(&self) -> Result { - Ok(self.resync_queue.len()?) + // This currently can't return an error because the CountedTree hack + // doesn't error on .len(), but this will change when we remove the hack + // (hopefully someday!) + Ok(self.resync_queue.len()) } /// Get number of blocks that have an error pub fn resync_errors_len(&self) -> Result { - Ok(self.resync_errors.len()?) + // (see resync_queue_len comment) + Ok(self.resync_errors.len()) } /// Get number of items in the refcount table diff --git a/src/block/metrics.rs b/src/block/metrics.rs index 0bd50a18..1fc0962a 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -1,6 +1,6 @@ use opentelemetry::{global, metrics::*}; -use garage_db as db; +use garage_db::counted_tree_hack::CountedTree; /// TableMetrics reference all counter used for metrics pub struct BlockManagerMetrics { @@ -23,14 +23,12 @@ pub struct BlockManagerMetrics { } impl BlockManagerMetrics { - pub fn new(resync_queue: db::Tree, resync_errors: db::Tree) -> Self { + pub fn new(resync_queue: CountedTree, resync_errors: CountedTree) -> Self { let meter = global::meter("garage_model/block"); Self { _resync_queue_len: meter .u64_value_observer("block.resync_queue_length", move |observer| { - if let Ok(v) = resync_queue.len() { - observer.observe(v as u64, &[]); - } + observer.observe(resync_queue.len() as u64, &[]); }) .with_description( "Number of block hashes queued for local check and possible resync", @@ -38,9 +36,7 @@ impl BlockManagerMetrics { .init(), _resync_errored_blocks: meter .u64_value_observer("block.resync_errored_blocks", move |observer| { - if let Ok(v) = resync_errors.len() { - observer.observe(v as u64, &[]); - } + observer.observe(resync_errors.len() as u64, &[]); }) .with_description("Number of block hashes whose last resync resulted in an error") .init(), diff --git a/src/db/counted_tree_hack.rs b/src/db/counted_tree_hack.rs new file mode 100644 index 00000000..52893b41 --- /dev/null +++ b/src/db/counted_tree_hack.rs @@ -0,0 +1,130 @@ +//! This hack allows a db tree to keep in RAM a counter of the number of entries +//! it contains, which is used to call .len() on it. This is usefull only for +//! the sled backend where .len() otherwise would have to traverse the whole +//! tree to count items. For sqlite and lmdb, this is mostly useless (but +//! hopefully not harmfull!). Note that a CountedTree cannot be part of a +//! transaction. + +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +use crate::{Result, Tree, TxError, Value, ValueIter}; + +#[derive(Clone)] +pub struct CountedTree(Arc); + +struct CountedTreeInternal { + tree: Tree, + len: AtomicUsize, +} + +impl CountedTree { + pub fn new(tree: Tree) -> Result { + let len = tree.len()?; + Ok(Self(Arc::new(CountedTreeInternal { + tree, + len: AtomicUsize::new(len), + }))) + } + + pub fn len(&self) -> usize { + self.0.len.load(Ordering::Relaxed) + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn get>(&self, key: K) -> Result> { + self.0.tree.get(key) + } + + pub fn first(&self) -> Result> { + self.0.tree.first() + } + + pub fn iter(&self) -> Result> { + self.0.tree.iter() + } + + // ---- writing functions ---- + + pub fn insert(&self, key: K, value: V) -> Result + where + K: AsRef<[u8]>, + V: AsRef<[u8]>, + { + let inserted = self.0.tree.insert(key, value)?; + if inserted { + self.0.len.fetch_add(1, Ordering::Relaxed); + } + Ok(inserted) + } + + pub fn remove>(&self, key: K) -> Result { + let removed = self.0.tree.remove(key)?; + if removed { + self.0.len.fetch_sub(1, Ordering::Relaxed); + } + Ok(removed) + } + + /* + pub fn pop_min(&self) -> Result> { + let res = self.0.tree.pop_min(); + if let Ok(Some(_)) = &res { + self.0.len.fetch_sub(1, Ordering::Relaxed); + }; + res + } + */ + + pub fn compare_and_swap( + &self, + key: K, + expected_old: Option, + new: Option, + ) -> Result + where + K: AsRef<[u8]>, + OV: AsRef<[u8]>, + NV: AsRef<[u8]>, + { + let old_some = expected_old.is_some(); + let new_some = new.is_some(); + + match self.0.tree.db().transaction(|mut tx| { + let old_val = tx.get(&self.0.tree, &key)?; + if old_val.as_ref().map(|x| &x[..]) == expected_old.as_ref().map(AsRef::as_ref) { + match &new { + Some(v) => { + tx.insert(&self.0.tree, &key, v)?; + } + None => { + tx.remove(&self.0.tree, &key)?; + } + } + tx.commit(()) + } else { + tx.abort(()) + } + }) { + Ok(()) => { + match (old_some, new_some) { + (false, true) => { + self.0.len.fetch_add(1, Ordering::Relaxed); + } + (true, false) => { + self.0.len.fetch_sub(1, Ordering::Relaxed); + } + _ => (), + } + Ok(true) + } + Err(TxError::Abort(())) => Ok(false), + Err(TxError::Db(e)) => Err(e), + } + } +} diff --git a/src/db/lib.rs b/src/db/lib.rs index afea9f55..4543e53c 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -2,6 +2,8 @@ pub mod lmdb_adapter; pub mod sled_adapter; pub mod sqlite_adapter; +pub mod counted_tree_hack; + #[cfg(test)] pub mod test; @@ -164,10 +166,13 @@ impl Tree { .transpose() } + /// True if item didn't exist before, false if item already existed + /// and was replaced. #[inline] - pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result<()> { + pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result { self.0.insert(self.1, key.as_ref(), value.as_ref()) } + /// True if item was removed, false if item already didn't exist #[inline] pub fn remove>(&self, key: T) -> Result { self.0.remove(self.1, key.as_ref()) @@ -215,15 +220,18 @@ impl<'a> Transaction<'a> { self.0.len(tree.1) } + /// True if item didn't exist before, false if item already existed + /// and was replaced. #[inline] pub fn insert, U: AsRef<[u8]>>( &mut self, tree: &Tree, key: T, value: U, - ) -> Result<()> { + ) -> Result { self.0.insert(tree.1, key.as_ref(), value.as_ref()) } + /// True if item was removed, false if item already didn't exist #[inline] pub fn remove>(&mut self, tree: &Tree, key: T) -> Result { self.0.remove(tree.1, key.as_ref()) @@ -281,7 +289,7 @@ pub(crate) trait IDb: Send + Sync { fn get(&self, tree: usize, key: &[u8]) -> Result>; fn len(&self, tree: usize) -> Result; - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result; fn remove(&self, tree: usize, key: &[u8]) -> Result; fn iter(&self, tree: usize) -> Result>; @@ -307,7 +315,7 @@ pub(crate) trait ITx { fn get(&self, tree: usize, key: &[u8]) -> Result>; fn len(&self, tree: usize) -> Result; - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result; fn remove(&mut self, tree: usize, key: &[u8]) -> Result; fn iter(&self, tree: usize) -> Result>; diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 3f468128..d8e5bcd3 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -122,12 +122,13 @@ impl IDb for LmdbDb { Ok(tree.len(&tx)?.try_into().unwrap()) } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result { let tree = self.get_tree(tree)?; let mut tx = self.db.write_txn()?; + let old_val = tree.get(&tx, key)?.map(Vec::from); tree.put(&mut tx, key, value)?; tx.commit()?; - Ok(()) + Ok(old_val.is_none()) } fn iter(&self, tree: usize) -> Result> { @@ -221,10 +222,11 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { unimplemented!(".len() in transaction not supported with LMDB backend") } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result { let tree = *self.get_tree(tree)?; + let old_val = tree.get(&self.tx, key)?.map(Vec::from); tree.put(&mut self.tx, key, value)?; - Ok(()) + Ok(old_val.is_none()) } fn remove(&mut self, tree: usize, key: &[u8]) -> Result { let tree = *self.get_tree(tree)?; diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 97fec2c7..18f457c8 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -93,10 +93,10 @@ impl IDb for SledDb { Ok(tree.len()) } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result { let tree = self.get_tree(tree)?; - tree.insert(key, value)?; - Ok(()) + let old_val = tree.insert(key, value)?; + Ok(old_val.is_none()) } fn iter(&self, tree: usize) -> Result> { @@ -206,10 +206,10 @@ impl<'a> ITx for SledTx<'a> { unimplemented!(".len() in transaction not supported with Sled backend") } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result { let tree = self.get_tree(tree)?; - self.save_error(tree.insert(key, value))?; - Ok(()) + let old_val = self.save_error(tree.insert(key, value))?; + Ok(old_val.is_none()) } fn remove(&mut self, tree: usize, key: &[u8]) -> Result { let tree = self.get_tree(tree)?; diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 0c8a0746..32557a53 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -54,6 +54,17 @@ impl SqliteDbInner { .map(String::as_str) .ok_or_else(|| Error("invalid tree id".into())) } + + fn internal_get(&self, tree: &str, key: &[u8]) -> Result> { + let mut stmt = self + .db + .prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; + let mut res_iter = stmt.query([key])?; + match res_iter.next()? { + None => Ok(None), + Some(v) => Ok(Some(v.get::<_, Vec>(0)?)), + } + } } impl IDb for SqliteDb { @@ -111,15 +122,7 @@ impl IDb for SqliteDb { trace!("get {}: lock acquired", tree); let tree = this.get_tree(tree)?; - - let mut stmt = this - .db - .prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; - let mut res_iter = stmt.query([key])?; - match res_iter.next()? { - None => Ok(None), - Some(v) => Ok(Some(v.get::<_, Vec>(0)?)), - } + this.internal_get(tree, key) } fn remove(&self, tree: usize, key: &[u8]) -> Result { @@ -148,17 +151,18 @@ impl IDb for SqliteDb { } } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result { trace!("insert {}: lock db", tree); let this = self.0.lock().unwrap(); trace!("insert {}: lock acquired", tree); let tree = this.get_tree(tree)?; + let old_val = this.internal_get(tree, key)?; this.db.execute( &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), params![key, value], )?; - Ok(()) + Ok(old_val.is_none()) } fn iter(&self, tree: usize) -> Result> { @@ -276,11 +280,8 @@ impl<'a> SqliteTx<'a> { ) }) } -} -impl<'a> ITx for SqliteTx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> Result> { - let tree = self.get_tree(tree)?; + fn internal_get(&self, tree: &str, key: &[u8]) -> Result> { let mut stmt = self .tx .prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; @@ -290,6 +291,13 @@ impl<'a> ITx for SqliteTx<'a> { Some(v) => Ok(Some(v.get::<_, Vec>(0)?)), } } +} + +impl<'a> ITx for SqliteTx<'a> { + fn get(&self, tree: usize, key: &[u8]) -> Result> { + let tree = self.get_tree(tree)?; + self.internal_get(tree, key) + } fn len(&self, tree: usize) -> Result { let tree = self.get_tree(tree)?; let mut stmt = self.tx.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; @@ -300,13 +308,14 @@ impl<'a> ITx for SqliteTx<'a> { } } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result { let tree = self.get_tree(tree)?; + let old_val = self.internal_get(tree, key)?; self.tx.execute( &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), params![key, value], )?; - Ok(()) + Ok(old_val.is_none()) } fn remove(&mut self, tree: usize, key: &[u8]) -> Result { let tree = self.get_tree(tree)?; diff --git a/src/table/data.rs b/src/table/data.rs index 839dae94..3212e82b 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -6,6 +6,7 @@ use serde_bytes::ByteBuf; use tokio::sync::Notify; use garage_db as db; +use garage_db::counted_tree_hack::CountedTree; use garage_util::data::*; use garage_util::error::*; @@ -30,7 +31,7 @@ pub struct TableData { pub(crate) merkle_tree: db::Tree, pub(crate) merkle_todo: db::Tree, pub(crate) merkle_todo_notify: Notify, - pub(crate) gc_todo: db::Tree, + pub(crate) gc_todo: CountedTree, pub(crate) metrics: TableMetrics, } @@ -55,6 +56,7 @@ where let gc_todo = db .open_tree(&format!("{}:gc_todo_v2", F::TABLE_NAME)) .expect("Unable to open DB tree"); + let gc_todo = CountedTree::new(gc_todo).expect("Cannot count gc_todo_v2"); let metrics = TableMetrics::new(F::TABLE_NAME, merkle_todo.clone(), gc_todo.clone()); @@ -319,6 +321,6 @@ where } pub fn gc_todo_len(&self) -> Result { - Ok(self.gc_todo.len()?) + Ok(self.gc_todo.len()) } } diff --git a/src/table/gc.rs b/src/table/gc.rs index e8843339..e7fbbcb0 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -12,7 +12,7 @@ use futures::select; use futures_util::future::*; use tokio::sync::watch; -use garage_db as db; +use garage_db::counted_tree_hack::CountedTree; use garage_util::data::*; use garage_util::error::*; @@ -370,7 +370,7 @@ impl GcTodoEntry { } /// Saves the GcTodoEntry in the gc_todo tree - pub(crate) fn save(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { + pub(crate) fn save(&self, gc_todo_tree: &CountedTree) -> Result<(), Error> { gc_todo_tree.insert(self.todo_table_key(), self.value_hash.as_slice())?; Ok(()) } @@ -380,16 +380,12 @@ impl GcTodoEntry { /// This is usefull to remove a todo entry only under the condition /// that it has not changed since the time it was read, i.e. /// what we have to do is still the same - pub(crate) fn remove_if_equal(&self, gc_todo_tree: &db::Tree) -> Result<(), Error> { - let key = self.todo_table_key(); - gc_todo_tree.db().transaction(|mut tx| { - let remove = - matches!(tx.get(gc_todo_tree, &key)?, Some(ov) if ov == self.value_hash.as_slice()); - if remove { - tx.remove(gc_todo_tree, &key)?; - } - tx.commit(()) - })?; + pub(crate) fn remove_if_equal(&self, gc_todo_tree: &CountedTree) -> Result<(), Error> { + gc_todo_tree.compare_and_swap::<_, _, &[u8]>( + &self.todo_table_key(), + Some(self.value_hash), + None, + )?; Ok(()) } diff --git a/src/table/metrics.rs b/src/table/metrics.rs index 13baf4c6..3a1783e0 100644 --- a/src/table/metrics.rs +++ b/src/table/metrics.rs @@ -1,6 +1,7 @@ use opentelemetry::{global, metrics::*, KeyValue}; use garage_db as db; +use garage_db::counted_tree_hack::CountedTree; /// TableMetrics reference all counter used for metrics pub struct TableMetrics { @@ -19,7 +20,7 @@ pub struct TableMetrics { pub(crate) sync_items_received: Counter, } impl TableMetrics { - pub fn new(table_name: &'static str, merkle_todo: db::Tree, gc_todo: db::Tree) -> Self { + pub fn new(table_name: &'static str, merkle_todo: db::Tree, gc_todo: CountedTree) -> Self { let meter = global::meter(table_name); TableMetrics { _merkle_todo_len: meter @@ -40,12 +41,10 @@ impl TableMetrics { .u64_value_observer( "table.gc_todo_queue_length", move |observer| { - if let Ok(v) = gc_todo.len() { - observer.observe( - v as u64, - &[KeyValue::new("table_name", table_name)], - ); - } + observer.observe( + gc_todo.len() as u64, + &[KeyValue::new("table_name", table_name)], + ); }, ) .with_description("Table garbage collector TODO queue length") diff --git a/src/util/sled_counter.rs b/src/util/sled_counter.rs deleted file mode 100644 index bc54cea0..00000000 --- a/src/util/sled_counter.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, -}; - -use sled::{CompareAndSwapError, IVec, Iter, Result, Tree}; - -#[derive(Clone)] -pub struct SledCountedTree(Arc); - -struct SledCountedTreeInternal { - tree: Tree, - len: AtomicUsize, -} - -impl SledCountedTree { - pub fn new(tree: Tree) -> Self { - let len = tree.len(); - Self(Arc::new(SledCountedTreeInternal { - tree, - len: AtomicUsize::new(len), - })) - } - - pub fn len(&self) -> usize { - self.0.len.load(Ordering::Relaxed) - } - - pub fn is_empty(&self) -> bool { - self.0.tree.is_empty() - } - - pub fn get>(&self, key: K) -> Result> { - self.0.tree.get(key) - } - - pub fn iter(&self) -> Iter { - self.0.tree.iter() - } - - // ---- writing functions ---- - - pub fn insert(&self, key: K, value: V) -> Result> - where - K: AsRef<[u8]>, - V: Into, - { - let res = self.0.tree.insert(key, value); - if res == Ok(None) { - self.0.len.fetch_add(1, Ordering::Relaxed); - } - res - } - - pub fn remove>(&self, key: K) -> Result> { - let res = self.0.tree.remove(key); - if matches!(res, Ok(Some(_))) { - self.0.len.fetch_sub(1, Ordering::Relaxed); - } - res - } - - pub fn pop_min(&self) -> Result> { - let res = self.0.tree.pop_min(); - if let Ok(Some(_)) = &res { - self.0.len.fetch_sub(1, Ordering::Relaxed); - }; - res - } - - pub fn compare_and_swap( - &self, - key: K, - old: Option, - new: Option, - ) -> Result> - where - K: AsRef<[u8]>, - OV: AsRef<[u8]>, - NV: Into, - { - let old_some = old.is_some(); - let new_some = new.is_some(); - - let res = self.0.tree.compare_and_swap(key, old, new); - - if res == Ok(Ok(())) { - match (old_some, new_some) { - (false, true) => { - self.0.len.fetch_add(1, Ordering::Relaxed); - } - (true, false) => { - self.0.len.fetch_sub(1, Ordering::Relaxed); - } - _ => (), - } - } - res - } -} -- 2.43.0 From a3a01141ec8f8eefe1a31162bd53fbaa2d37e601 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 17:05:17 +0200 Subject: [PATCH 49/64] db abstraction: make .insert() and .remove() return the old value --- src/db/counted_tree_hack.rs | 26 +++------- src/db/lib.rs | 30 ++++++------ src/db/lmdb_adapter.rs | 32 ++++++------ src/db/sled_adapter.rs | 24 ++++----- src/db/sqlite_adapter.rs | 98 ++++++++++++++++++++++++++----------- src/db/test.rs | 8 +-- 6 files changed, 127 insertions(+), 91 deletions(-) diff --git a/src/db/counted_tree_hack.rs b/src/db/counted_tree_hack.rs index 52893b41..354a9dad 100644 --- a/src/db/counted_tree_hack.rs +++ b/src/db/counted_tree_hack.rs @@ -51,36 +51,26 @@ impl CountedTree { // ---- writing functions ---- - pub fn insert(&self, key: K, value: V) -> Result + pub fn insert(&self, key: K, value: V) -> Result> where K: AsRef<[u8]>, V: AsRef<[u8]>, { - let inserted = self.0.tree.insert(key, value)?; - if inserted { + let old_val = self.0.tree.insert(key, value)?; + if old_val.is_none() { self.0.len.fetch_add(1, Ordering::Relaxed); } - Ok(inserted) + Ok(old_val) } - pub fn remove>(&self, key: K) -> Result { - let removed = self.0.tree.remove(key)?; - if removed { + pub fn remove>(&self, key: K) -> Result> { + let old_val = self.0.tree.remove(key)?; + if old_val.is_some() { self.0.len.fetch_sub(1, Ordering::Relaxed); } - Ok(removed) + Ok(old_val) } - /* - pub fn pop_min(&self) -> Result> { - let res = self.0.tree.pop_min(); - if let Ok(Some(_)) = &res { - self.0.len.fetch_sub(1, Ordering::Relaxed); - }; - res - } - */ - pub fn compare_and_swap( &self, key: K, diff --git a/src/db/lib.rs b/src/db/lib.rs index 4543e53c..ed1bdb75 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -166,15 +166,18 @@ impl Tree { .transpose() } - /// True if item didn't exist before, false if item already existed - /// and was replaced. + /// Returns the old value if there was one #[inline] - pub fn insert, U: AsRef<[u8]>>(&self, key: T, value: U) -> Result { + pub fn insert, U: AsRef<[u8]>>( + &self, + key: T, + value: U, + ) -> Result> { self.0.insert(self.1, key.as_ref(), value.as_ref()) } - /// True if item was removed, false if item already didn't exist + /// Returns the old value if there was one #[inline] - pub fn remove>(&self, key: T) -> Result { + pub fn remove>(&self, key: T) -> Result> { self.0.remove(self.1, key.as_ref()) } @@ -220,20 +223,19 @@ impl<'a> Transaction<'a> { self.0.len(tree.1) } - /// True if item didn't exist before, false if item already existed - /// and was replaced. + /// Returns the old value if there was one #[inline] pub fn insert, U: AsRef<[u8]>>( &mut self, tree: &Tree, key: T, value: U, - ) -> Result { + ) -> Result> { self.0.insert(tree.1, key.as_ref(), value.as_ref()) } - /// True if item was removed, false if item already didn't exist + /// Returns the old value if there was one #[inline] - pub fn remove>(&mut self, tree: &Tree, key: T) -> Result { + pub fn remove>(&mut self, tree: &Tree, key: T) -> Result> { self.0.remove(tree.1, key.as_ref()) } @@ -289,8 +291,8 @@ pub(crate) trait IDb: Send + Sync { fn get(&self, tree: usize, key: &[u8]) -> Result>; fn len(&self, tree: usize) -> Result; - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result; - fn remove(&self, tree: usize, key: &[u8]) -> Result; + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result>; + fn remove(&self, tree: usize, key: &[u8]) -> Result>; fn iter(&self, tree: usize) -> Result>; fn iter_rev(&self, tree: usize) -> Result>; @@ -315,8 +317,8 @@ pub(crate) trait ITx { fn get(&self, tree: usize, key: &[u8]) -> Result>; fn len(&self, tree: usize) -> Result; - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result; - fn remove(&mut self, tree: usize, key: &[u8]) -> Result; + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result>; + fn remove(&mut self, tree: usize, key: &[u8]) -> Result>; fn iter(&self, tree: usize) -> Result>; fn iter_rev(&self, tree: usize) -> Result>; diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index d8e5bcd3..9e4306c8 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -108,27 +108,28 @@ impl IDb for LmdbDb { } } - fn remove(&self, tree: usize, key: &[u8]) -> Result { - let tree = self.get_tree(tree)?; - let mut tx = self.db.write_txn()?; - let deleted = tree.delete(&mut tx, key)?; - tx.commit()?; - Ok(deleted) - } - fn len(&self, tree: usize) -> Result { let tree = self.get_tree(tree)?; let tx = self.db.read_txn()?; Ok(tree.len(&tx)?.try_into().unwrap()) } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result { + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result> { let tree = self.get_tree(tree)?; let mut tx = self.db.write_txn()?; let old_val = tree.get(&tx, key)?.map(Vec::from); tree.put(&mut tx, key, value)?; tx.commit()?; - Ok(old_val.is_none()) + Ok(old_val) + } + + fn remove(&self, tree: usize, key: &[u8]) -> Result> { + let tree = self.get_tree(tree)?; + let mut tx = self.db.write_txn()?; + let old_val = tree.get(&tx, key)?.map(Vec::from); + tree.delete(&mut tx, key)?; + tx.commit()?; + Ok(old_val) } fn iter(&self, tree: usize) -> Result> { @@ -222,16 +223,17 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { unimplemented!(".len() in transaction not supported with LMDB backend") } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result> { let tree = *self.get_tree(tree)?; let old_val = tree.get(&self.tx, key)?.map(Vec::from); tree.put(&mut self.tx, key, value)?; - Ok(old_val.is_none()) + Ok(old_val) } - fn remove(&mut self, tree: usize, key: &[u8]) -> Result { + fn remove(&mut self, tree: usize, key: &[u8]) -> Result> { let tree = *self.get_tree(tree)?; - let deleted = tree.delete(&mut self.tx, key)?; - Ok(deleted) + let old_val = tree.get(&self.tx, key)?.map(Vec::from); + tree.delete(&mut self.tx, key)?; + Ok(old_val) } fn iter(&self, _tree: usize) -> Result> { diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 18f457c8..b07401c9 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -83,20 +83,21 @@ impl IDb for SledDb { Ok(val.map(|x| x.to_vec())) } - fn remove(&self, tree: usize, key: &[u8]) -> Result { - let tree = self.get_tree(tree)?; - Ok(tree.remove(key)?.is_some()) - } - fn len(&self, tree: usize) -> Result { let tree = self.get_tree(tree)?; Ok(tree.len()) } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result { + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result> { let tree = self.get_tree(tree)?; let old_val = tree.insert(key, value)?; - Ok(old_val.is_none()) + Ok(old_val.map(|x| x.to_vec())) + } + + fn remove(&self, tree: usize, key: &[u8]) -> Result> { + let tree = self.get_tree(tree)?; + let old_val = tree.remove(key)?; + Ok(old_val.map(|x| x.to_vec())) } fn iter(&self, tree: usize) -> Result> { @@ -206,14 +207,15 @@ impl<'a> ITx for SledTx<'a> { unimplemented!(".len() in transaction not supported with Sled backend") } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result> { let tree = self.get_tree(tree)?; let old_val = self.save_error(tree.insert(key, value))?; - Ok(old_val.is_none()) + Ok(old_val.map(|x| x.to_vec())) } - fn remove(&mut self, tree: usize, key: &[u8]) -> Result { + fn remove(&mut self, tree: usize, key: &[u8]) -> Result> { let tree = self.get_tree(tree)?; - Ok(self.save_error(tree.remove(key))?.is_some()) + let old_val = self.save_error(tree.remove(key))?; + Ok(old_val.map(|x| x.to_vec())) } fn iter(&self, _tree: usize) -> Result> { diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 32557a53..b23885bd 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -125,18 +125,6 @@ impl IDb for SqliteDb { this.internal_get(tree, key) } - fn remove(&self, tree: usize, key: &[u8]) -> Result { - trace!("remove {}: lock db", tree); - let this = self.0.lock().unwrap(); - trace!("remove {}: lock acquired", tree); - - let tree = this.get_tree(tree)?; - let res = this - .db - .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; - Ok(res > 0) - } - fn len(&self, tree: usize) -> Result { trace!("len {}: lock db", tree); let this = self.0.lock().unwrap(); @@ -151,18 +139,50 @@ impl IDb for SqliteDb { } } - fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result { + fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result> { trace!("insert {}: lock db", tree); let this = self.0.lock().unwrap(); trace!("insert {}: lock acquired", tree); let tree = this.get_tree(tree)?; let old_val = this.internal_get(tree, key)?; - this.db.execute( - &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), - params![key, value], - )?; - Ok(old_val.is_none()) + + match &old_val { + Some(_) => { + let n = this.db.execute( + &format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), + params![key, value], + )?; + assert_eq!(n, 1); + } + None => { + let n = this.db.execute( + &format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), + params![key, value], + )?; + assert_eq!(n, 1); + } + } + + Ok(old_val) + } + + fn remove(&self, tree: usize, key: &[u8]) -> Result> { + trace!("remove {}: lock db", tree); + let this = self.0.lock().unwrap(); + trace!("remove {}: lock acquired", tree); + + let tree = this.get_tree(tree)?; + let old_val = this.internal_get(tree, key)?; + + if old_val.is_some() { + let n = this + .db + .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; + assert_eq!(n, 1); + } + + Ok(old_val) } fn iter(&self, tree: usize) -> Result> { @@ -308,21 +328,41 @@ impl<'a> ITx for SqliteTx<'a> { } } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result> { let tree = self.get_tree(tree)?; let old_val = self.internal_get(tree, key)?; - self.tx.execute( - &format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree), - params![key, value], - )?; - Ok(old_val.is_none()) + + match &old_val { + Some(_) => { + let n = self.tx.execute( + &format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), + params![key, value], + )?; + assert_eq!(n, 1); + } + None => { + let n = self.tx.execute( + &format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), + params![key, value], + )?; + assert_eq!(n, 1); + } + } + + Ok(old_val) } - fn remove(&mut self, tree: usize, key: &[u8]) -> Result { + fn remove(&mut self, tree: usize, key: &[u8]) -> Result> { let tree = self.get_tree(tree)?; - let res = self - .tx - .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; - Ok(res > 0) + let old_val = self.internal_get(tree, key)?; + + if old_val.is_some() { + let n = self + .tx + .execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?; + assert_eq!(n, 1); + } + + Ok(old_val) } fn iter(&self, _tree: usize) -> Result> { diff --git a/src/db/test.rs b/src/db/test.rs index 31fea1f0..cfcee643 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -14,13 +14,13 @@ fn test_suite(db: Db) { let vb: &[u8] = &b"plip"[..]; let vc: &[u8] = &b"plup"[..]; - tree.insert(ka, va).unwrap(); + assert!(tree.insert(ka, va).unwrap().is_none()); assert_eq!(tree.get(ka).unwrap().unwrap(), va); let res = db.transaction::<_, (), _>(|mut tx| { assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), va); - tx.insert(&tree, ka, vb).unwrap(); + assert_eq!(tx.insert(&tree, ka, vb).unwrap().unwrap(), va); assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); @@ -32,7 +32,7 @@ fn test_suite(db: Db) { let res = db.transaction::<(), _, _>(|mut tx| { assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb); - tx.insert(&tree, ka, vc).unwrap(); + assert_eq!(tx.insert(&tree, ka, vc).unwrap().unwrap(), vb); assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vc); @@ -47,7 +47,7 @@ fn test_suite(db: Db) { assert!(iter.next().is_none()); drop(iter); - tree.insert(kb, vc).unwrap(); + assert!(tree.insert(kb, vc).unwrap().is_none()); assert_eq!(tree.get(kb).unwrap().unwrap(), vc); let mut iter = tree.iter().unwrap(); -- 2.43.0 From 1d408d52b9abd2705f68bbdb4c28b7990d147e53 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 17:10:22 +0200 Subject: [PATCH 50/64] Simplify --- src/block/manager.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index da86a2d5..aa0969f4 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -335,7 +335,6 @@ impl BlockManager { // we will fecth it from someone. let this = self.clone(); tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(1)).await; if let Err(e) = this.put_to_resync(&hash, 2 * BLOCK_RW_TIMEOUT) { error!("Block {:?} could not be put in resync queue: {}.", hash, e); } @@ -354,7 +353,6 @@ impl BlockManager { // after that delay has passed. let this = self.clone(); tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(1)).await; if let Err(e) = this.put_to_resync(&hash, BLOCK_GC_DELAY + Duration::from_secs(10)) { error!("Block {:?} could not be put in resync queue: {}.", hash, e); -- 2.43.0 From cdc03da0b83aeb39d1b16aef252d6b18c5bf1fa9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 17:11:55 +0200 Subject: [PATCH 51/64] make things more like before --- src/block/metrics.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/block/metrics.rs b/src/block/metrics.rs index 1fc0962a..477add66 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -28,7 +28,7 @@ impl BlockManagerMetrics { Self { _resync_queue_len: meter .u64_value_observer("block.resync_queue_length", move |observer| { - observer.observe(resync_queue.len() as u64, &[]); + observer.observe(resync_queue.len() as u64, &[]) }) .with_description( "Number of block hashes queued for local check and possible resync", @@ -36,7 +36,7 @@ impl BlockManagerMetrics { .init(), _resync_errored_blocks: meter .u64_value_observer("block.resync_errored_blocks", move |observer| { - observer.observe(resync_errors.len() as u64, &[]); + observer.observe(resync_errors.len() as u64, &[]) }) .with_description("Number of block hashes whose last resync resulted in an error") .init(), -- 2.43.0 From 8c6f690fa584e6f1bccc428d2b5525ba219e16c2 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 17:14:16 +0200 Subject: [PATCH 52/64] less verbose code --- src/block/rc.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/block/rc.rs b/src/block/rc.rs index 42cdf241..f82595b7 100644 --- a/src/block/rc.rs +++ b/src/block/rc.rs @@ -22,9 +22,7 @@ impl BlockRc { pub(crate) fn block_incref(&self, tx: &mut db::Transaction, hash: &Hash) -> db::Result { let old_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?); match old_rc.increment().serialize() { - Some(x) => { - tx.insert(&self.rc, &hash, x)?; - } + Some(x) => tx.insert(&self.rc, &hash, x)?, None => unreachable!(), }; Ok(old_rc.is_zero()) @@ -35,12 +33,8 @@ impl BlockRc { pub(crate) fn block_decref(&self, tx: &mut db::Transaction, hash: &Hash) -> db::Result { let new_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?).decrement(); match new_rc.serialize() { - Some(x) => { - tx.insert(&self.rc, &hash, x)?; - } - None => { - tx.remove(&self.rc, &hash)?; - } + Some(x) => tx.insert(&self.rc, &hash, x)?, + None => tx.remove(&self.rc, &hash)?, }; Ok(matches!(new_rc, RcEntry::Deletable { .. })) } -- 2.43.0 From 6f3d43b75b45dc9aca03c0598f81f4cb2c2013b2 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 17:21:01 +0200 Subject: [PATCH 53/64] prettier code --- src/db/counted_tree_hack.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/db/counted_tree_hack.rs b/src/db/counted_tree_hack.rs index 354a9dad..bbe943a2 100644 --- a/src/db/counted_tree_hack.rs +++ b/src/db/counted_tree_hack.rs @@ -30,7 +30,7 @@ impl CountedTree { } pub fn len(&self) -> usize { - self.0.len.load(Ordering::Relaxed) + self.0.len.load(Ordering::SeqCst) } pub fn is_empty(&self) -> bool { @@ -58,7 +58,7 @@ impl CountedTree { { let old_val = self.0.tree.insert(key, value)?; if old_val.is_none() { - self.0.len.fetch_add(1, Ordering::Relaxed); + self.0.len.fetch_add(1, Ordering::SeqCst); } Ok(old_val) } @@ -66,7 +66,7 @@ impl CountedTree { pub fn remove>(&self, key: K) -> Result> { let old_val = self.0.tree.remove(key)?; if old_val.is_some() { - self.0.len.fetch_sub(1, Ordering::Relaxed); + self.0.len.fetch_sub(1, Ordering::SeqCst); } Ok(old_val) } @@ -85,9 +85,14 @@ impl CountedTree { let old_some = expected_old.is_some(); let new_some = new.is_some(); - match self.0.tree.db().transaction(|mut tx| { + let tx_res = self.0.tree.db().transaction(|mut tx| { let old_val = tx.get(&self.0.tree, &key)?; - if old_val.as_ref().map(|x| &x[..]) == expected_old.as_ref().map(AsRef::as_ref) { + let is_same = match (&old_val, &expected_old) { + (None, None) => true, + (Some(x), Some(y)) if x == y.as_ref() => true, + _ => false, + }; + if is_same { match &new { Some(v) => { tx.insert(&self.0.tree, &key, v)?; @@ -100,14 +105,16 @@ impl CountedTree { } else { tx.abort(()) } - }) { + }); + + match tx_res { Ok(()) => { match (old_some, new_some) { (false, true) => { - self.0.len.fetch_add(1, Ordering::Relaxed); + self.0.len.fetch_add(1, Ordering::SeqCst); } (true, false) => { - self.0.len.fetch_sub(1, Ordering::Relaxed); + self.0.len.fetch_sub(1, Ordering::SeqCst); } _ => (), } -- 2.43.0 From 1bbe0794f363eb59c56548cca672013fd78f361a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 17:23:17 +0200 Subject: [PATCH 54/64] less pub(crate) --- src/db/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/db/lib.rs b/src/db/lib.rs index ed1bdb75..a7b6197c 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -18,10 +18,10 @@ use err_derive::Error; #[derive(Clone)] pub struct Db(pub(crate) Arc); -pub struct Transaction<'a>(pub(crate) &'a mut dyn ITx); +pub struct Transaction<'a>(&'a mut dyn ITx); #[derive(Clone)] -pub struct Tree(pub(crate) Arc, pub(crate) usize); +pub struct Tree(Arc, usize); pub type Value = Vec; pub type ValueIter<'a> = Box> + 'a>; -- 2.43.0 From 0543cb345320a15280a5af7db941bb9fbffb4cd6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 17:50:10 +0200 Subject: [PATCH 55/64] Cleaner error management (less error-prone api) --- src/block/manager.rs | 12 +++++-- src/block/rc.rs | 12 +++++-- src/db/lib.rs | 57 ++++++++++++++++++++------------- src/db/lmdb_adapter.rs | 45 +++++++++++++++----------- src/db/sled_adapter.rs | 40 +++++++++++++++-------- src/db/sqlite_adapter.rs | 47 +++++++++++++++------------ src/model/index_counter.rs | 2 +- src/model/k2v/item_table.rs | 25 +++++++-------- src/model/s3/block_ref_table.rs | 2 +- src/model/s3/object_table.rs | 2 +- src/model/s3/version_table.rs | 2 +- src/table/schema.rs | 2 +- 12 files changed, 149 insertions(+), 99 deletions(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index aa0969f4..32ba0431 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -325,7 +325,11 @@ impl BlockManager { /// Increment the number of time a block is used, putting it to resynchronization if it is /// required, but not known - pub fn block_incref(self: &Arc, tx: &mut db::Transaction, hash: Hash) -> db::Result<()> { + pub fn block_incref( + self: &Arc, + tx: &mut db::Transaction, + hash: Hash, + ) -> db::TxOpResult<()> { if self.rc.block_incref(tx, &hash)? { // When the reference counter is incremented, there is // normally a node that is responsible for sending us the @@ -344,7 +348,11 @@ impl BlockManager { } /// Decrement the number of time a block is used - pub fn block_decref(self: &Arc, tx: &mut db::Transaction, hash: Hash) -> db::Result<()> { + pub fn block_decref( + self: &Arc, + tx: &mut db::Transaction, + hash: Hash, + ) -> db::TxOpResult<()> { if self.rc.block_decref(tx, &hash)? { // When the RC is decremented, it might drop to zero, // indicating that we don't need the block. diff --git a/src/block/rc.rs b/src/block/rc.rs index f82595b7..ce6defad 100644 --- a/src/block/rc.rs +++ b/src/block/rc.rs @@ -19,7 +19,11 @@ impl BlockRc { /// Increment the reference counter associated to a hash. /// Returns true if the RC goes from zero to nonzero. - pub(crate) fn block_incref(&self, tx: &mut db::Transaction, hash: &Hash) -> db::Result { + pub(crate) fn block_incref( + &self, + tx: &mut db::Transaction, + hash: &Hash, + ) -> db::TxOpResult { let old_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?); match old_rc.increment().serialize() { Some(x) => tx.insert(&self.rc, &hash, x)?, @@ -30,7 +34,11 @@ impl BlockRc { /// Decrement the reference counter associated to a hash. /// Returns true if the RC is now zero. - pub(crate) fn block_decref(&self, tx: &mut db::Transaction, hash: &Hash) -> db::Result { + pub(crate) fn block_decref( + &self, + tx: &mut db::Transaction, + hash: &Hash, + ) -> db::TxOpResult { let new_rc = RcEntry::parse_opt(tx.get(&self.rc, &hash)?).decrement(); match new_rc.serialize() { Some(x) => tx.insert(&self.rc, &hash, x)?, diff --git a/src/db/lib.rs b/src/db/lib.rs index a7b6197c..a4da4086 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -34,6 +34,9 @@ pub struct Error(pub Cow<'static, str>); pub type Result = std::result::Result; +pub struct TxOpError(pub(crate) Error); +pub type TxOpResult = std::result::Result; + #[derive(Debug)] pub enum TxError { Abort(E), @@ -41,9 +44,17 @@ pub enum TxError { } pub type TxResult = std::result::Result>; -impl From for TxError { - fn from(e: Error) -> TxError { - TxError::Db(e) +impl From for TxError { + fn from(e: TxOpError) -> TxError { + TxError::Db(e.0) + } +} + +pub fn unabort(res: TxResult) -> TxOpResult> { + match res { + Ok(v) => Ok(Ok(v)), + Err(TxError::Abort(e)) => Ok(Err(e)), + Err(TxError::Db(e)) => Err(TxOpError(e)), } } @@ -117,19 +128,19 @@ impl Db { let tx_res = self.transaction(|mut tx| { let mut i = 0; - for item in ex_tree.iter()? { - let (k, v) = item?; + for item in ex_tree.iter().map_err(TxError::Abort)? { + let (k, v) = item.map_err(TxError::Abort)?; tx.insert(&tree, k, v)?; i += 1; if i % 1000 == 0 { println!("{}: imported {}", name, i); } } - Ok::<_, TxError<()>>(i) + tx.commit(i) }); let total = match tx_res { Err(TxError::Db(e)) => return Err(e), - Err(TxError::Abort(_)) => unreachable!(), + Err(TxError::Abort(e)) => return Err(e), Ok(x) => x, }; @@ -215,11 +226,11 @@ impl Tree { #[allow(clippy::len_without_is_empty)] impl<'a> Transaction<'a> { #[inline] - pub fn get>(&self, tree: &Tree, key: T) -> Result> { + pub fn get>(&self, tree: &Tree, key: T) -> TxOpResult> { self.0.get(tree.1, key.as_ref()) } #[inline] - pub fn len(&self, tree: &Tree) -> Result { + pub fn len(&self, tree: &Tree) -> TxOpResult { self.0.len(tree.1) } @@ -230,26 +241,26 @@ impl<'a> Transaction<'a> { tree: &Tree, key: T, value: U, - ) -> Result> { + ) -> TxOpResult> { self.0.insert(tree.1, key.as_ref(), value.as_ref()) } /// Returns the old value if there was one #[inline] - pub fn remove>(&mut self, tree: &Tree, key: T) -> Result> { + pub fn remove>(&mut self, tree: &Tree, key: T) -> TxOpResult> { self.0.remove(tree.1, key.as_ref()) } #[inline] - pub fn iter(&self, tree: &Tree) -> Result> { + pub fn iter(&self, tree: &Tree) -> TxOpResult> { self.0.iter(tree.1) } #[inline] - pub fn iter_rev(&self, tree: &Tree) -> Result> { + pub fn iter_rev(&self, tree: &Tree) -> TxOpResult> { self.0.iter_rev(tree.1) } #[inline] - pub fn range(&self, tree: &Tree, range: R) -> Result> + pub fn range(&self, tree: &Tree, range: R) -> TxOpResult> where K: AsRef<[u8]>, R: RangeBounds, @@ -259,7 +270,7 @@ impl<'a> Transaction<'a> { self.0.range(tree.1, get_bound(sb), get_bound(eb)) } #[inline] - pub fn range_rev(&self, tree: &Tree, range: R) -> Result> + pub fn range_rev(&self, tree: &Tree, range: R) -> TxOpResult> where K: AsRef<[u8]>, R: RangeBounds, @@ -314,27 +325,27 @@ pub(crate) trait IDb: Send + Sync { } pub(crate) trait ITx { - fn get(&self, tree: usize, key: &[u8]) -> Result>; - fn len(&self, tree: usize) -> Result; + fn get(&self, tree: usize, key: &[u8]) -> TxOpResult>; + fn len(&self, tree: usize) -> TxOpResult; - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result>; - fn remove(&mut self, tree: usize, key: &[u8]) -> Result>; + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult>; + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult>; - fn iter(&self, tree: usize) -> Result>; - fn iter_rev(&self, tree: usize) -> Result>; + fn iter(&self, tree: usize) -> TxOpResult>; + fn iter_rev(&self, tree: usize) -> TxOpResult>; fn range<'r>( &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result>; + ) -> TxOpResult>; fn range_rev<'r>( &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> Result>; + ) -> TxOpResult>; } pub(crate) trait ITxFn { diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 9e4306c8..d1efd216 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -8,7 +8,10 @@ use std::sync::{Arc, RwLock}; use heed::types::ByteSlice; use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; -use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; +use crate::{ + Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, + Value, ValueIter, +}; pub use heed; @@ -20,9 +23,9 @@ impl From for Error { } } -impl From for TxError { - fn from(e: heed::Error) -> TxError { - TxError::Db(e.into()) +impl From for TxOpError { + fn from(e: heed::Error) -> TxOpError { + TxOpError(e.into()) } } @@ -171,21 +174,25 @@ impl IDb for LmdbDb { let trees = self.trees.read().unwrap(); let mut tx = LmdbTx { trees: &trees.0[..], - tx: self.db.write_txn()?, + tx: self + .db + .write_txn() + .map_err(Error::from) + .map_err(TxError::Db)?, }; let res = f.try_on(&mut tx); match res { TxFnResult::Ok => { - tx.tx.commit()?; + tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?; Ok(()) } TxFnResult::Abort => { - tx.tx.abort()?; + tx.tx.abort().map_err(Error::from).map_err(TxError::Db)?; Err(TxError::Abort(())) } TxFnResult::DbErr => { - tx.tx.abort()?; + tx.tx.abort().map_err(Error::from).map_err(TxError::Db)?; Err(TxError::Db(Error( "(this message will be discarded)".into(), ))) @@ -202,44 +209,44 @@ struct LmdbTx<'a, 'db> { } impl<'a, 'db> LmdbTx<'a, 'db> { - fn get_tree(&self, i: usize) -> Result<&Database> { + fn get_tree(&self, i: usize) -> TxOpResult<&Database> { self.trees.get(i).ok_or_else(|| { - Error( + TxOpError(Error( "invalid tree id (it might have been openned after the transaction started)".into(), - ) + )) }) } } impl<'a, 'db> ITx for LmdbTx<'a, 'db> { - fn get(&self, tree: usize, key: &[u8]) -> Result> { + fn get(&self, tree: usize, key: &[u8]) -> TxOpResult> { let tree = self.get_tree(tree)?; match tree.get(&self.tx, key)? { Some(v) => Ok(Some(v.to_vec())), None => Ok(None), } } - fn len(&self, _tree: usize) -> Result { + fn len(&self, _tree: usize) -> TxOpResult { unimplemented!(".len() in transaction not supported with LMDB backend") } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult> { let tree = *self.get_tree(tree)?; let old_val = tree.get(&self.tx, key)?.map(Vec::from); tree.put(&mut self.tx, key, value)?; Ok(old_val) } - fn remove(&mut self, tree: usize, key: &[u8]) -> Result> { + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult> { let tree = *self.get_tree(tree)?; let old_val = tree.get(&self.tx, key)?.map(Vec::from); tree.delete(&mut self.tx, key)?; Ok(old_val) } - fn iter(&self, _tree: usize) -> Result> { + fn iter(&self, _tree: usize) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with LMDB backend"); } - fn iter_rev(&self, _tree: usize) -> Result> { + fn iter_rev(&self, _tree: usize) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with LMDB backend"); } @@ -248,7 +255,7 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with LMDB backend"); } fn range_rev<'r>( @@ -256,7 +263,7 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with LMDB backend"); } } diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index b07401c9..d0d9e9c0 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -9,7 +9,10 @@ use sled::transaction::{ UnabortableTransactionError, }; -use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; +use crate::{ + Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, + Value, ValueIter, +}; pub use sled; @@ -21,6 +24,12 @@ impl From for Error { } } +impl From for TxOpError { + fn from(e: sled::Error) -> TxOpError { + TxOpError(e.into()) + } +} + // -- db pub struct SledDb { @@ -177,51 +186,54 @@ struct SledTx<'a> { } impl<'a> SledTx<'a> { - fn get_tree(&self, i: usize) -> Result<&TransactionalTree> { + fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalTree> { self.trees.get(i).ok_or_else(|| { - Error( + TxOpError(Error( "invalid tree id (it might have been openned after the transaction started)".into(), - ) + )) }) } - fn save_error(&self, v: std::result::Result) -> Result { + fn save_error( + &self, + v: std::result::Result, + ) -> TxOpResult { match v { Ok(x) => Ok(x), Err(e) => { let txt = format!("{}", e); self.err.set(Some(e)); - Err(Error(txt.into())) + Err(TxOpError(Error(txt.into()))) } } } } impl<'a> ITx for SledTx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> Result> { + fn get(&self, tree: usize, key: &[u8]) -> TxOpResult> { let tree = self.get_tree(tree)?; let tmp = self.save_error(tree.get(key))?; Ok(tmp.map(|x| x.to_vec())) } - fn len(&self, _tree: usize) -> Result { + fn len(&self, _tree: usize) -> TxOpResult { unimplemented!(".len() in transaction not supported with Sled backend") } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult> { let tree = self.get_tree(tree)?; let old_val = self.save_error(tree.insert(key, value))?; Ok(old_val.map(|x| x.to_vec())) } - fn remove(&mut self, tree: usize, key: &[u8]) -> Result> { + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult> { let tree = self.get_tree(tree)?; let old_val = self.save_error(tree.remove(key))?; Ok(old_val.map(|x| x.to_vec())) } - fn iter(&self, _tree: usize) -> Result> { + fn iter(&self, _tree: usize) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with Sled backend"); } - fn iter_rev(&self, _tree: usize) -> Result> { + fn iter_rev(&self, _tree: usize) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with Sled backend"); } @@ -230,7 +242,7 @@ impl<'a> ITx for SledTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with Sled backend"); } fn range_rev<'r>( @@ -238,7 +250,7 @@ impl<'a> ITx for SledTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with Sled backend"); } } diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index b23885bd..8d6bd714 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -10,7 +10,10 @@ use log::trace; use rusqlite::{params, Connection, Rows, Statement, Transaction}; -use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; +use crate::{ + Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, + Value, ValueIter, +}; pub use rusqlite; @@ -22,9 +25,9 @@ impl From for Error { } } -impl From for TxError { - fn from(e: rusqlite::Error) -> TxError { - TxError::Db(e.into()) +impl From for TxOpError { + fn from(e: rusqlite::Error) -> TxOpError { + TxOpError(e.into()) } } @@ -260,20 +263,24 @@ impl IDb for SqliteDb { let this_mut_ref: &mut SqliteDbInner = this.borrow_mut(); let mut tx = SqliteTx { - tx: this_mut_ref.db.transaction()?, + tx: this_mut_ref + .db + .transaction() + .map_err(Error::from) + .map_err(TxError::Db)?, trees: &this_mut_ref.trees, }; let res = match f.try_on(&mut tx) { TxFnResult::Ok => { - tx.tx.commit()?; + tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?; Ok(()) } TxFnResult::Abort => { - tx.tx.rollback()?; + tx.tx.rollback().map_err(Error::from).map_err(TxError::Db)?; Err(TxError::Abort(())) } TxFnResult::DbErr => { - tx.tx.rollback()?; + tx.tx.rollback().map_err(Error::from).map_err(TxError::Db)?; Err(TxError::Db(Error( "(this message will be discarded)".into(), ))) @@ -293,15 +300,15 @@ struct SqliteTx<'a> { } impl<'a> SqliteTx<'a> { - fn get_tree(&self, i: usize) -> Result<&'_ str> { + fn get_tree(&self, i: usize) -> TxOpResult<&'_ str> { self.trees.get(i).map(String::as_ref).ok_or_else(|| { - Error( + TxOpError(Error( "invalid tree id (it might have been openned after the transaction started)".into(), - ) + )) }) } - fn internal_get(&self, tree: &str, key: &[u8]) -> Result> { + fn internal_get(&self, tree: &str, key: &[u8]) -> TxOpResult> { let mut stmt = self .tx .prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?; @@ -314,11 +321,11 @@ impl<'a> SqliteTx<'a> { } impl<'a> ITx for SqliteTx<'a> { - fn get(&self, tree: usize, key: &[u8]) -> Result> { + fn get(&self, tree: usize, key: &[u8]) -> TxOpResult> { let tree = self.get_tree(tree)?; self.internal_get(tree, key) } - fn len(&self, tree: usize) -> Result { + fn len(&self, tree: usize) -> TxOpResult { let tree = self.get_tree(tree)?; let mut stmt = self.tx.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?; let mut res_iter = stmt.query([])?; @@ -328,7 +335,7 @@ impl<'a> ITx for SqliteTx<'a> { } } - fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result> { + fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult> { let tree = self.get_tree(tree)?; let old_val = self.internal_get(tree, key)?; @@ -351,7 +358,7 @@ impl<'a> ITx for SqliteTx<'a> { Ok(old_val) } - fn remove(&mut self, tree: usize, key: &[u8]) -> Result> { + fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult> { let tree = self.get_tree(tree)?; let old_val = self.internal_get(tree, key)?; @@ -365,10 +372,10 @@ impl<'a> ITx for SqliteTx<'a> { Ok(old_val) } - fn iter(&self, _tree: usize) -> Result> { + fn iter(&self, _tree: usize) -> TxOpResult> { unimplemented!(); } - fn iter_rev(&self, _tree: usize) -> Result> { + fn iter_rev(&self, _tree: usize) -> TxOpResult> { unimplemented!(); } @@ -377,7 +384,7 @@ impl<'a> ITx for SqliteTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> TxOpResult> { unimplemented!(); } fn range_rev<'r>( @@ -385,7 +392,7 @@ impl<'a> ITx for SqliteTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> Result> { + ) -> TxOpResult> { unimplemented!(); } } diff --git a/src/model/index_counter.rs b/src/model/index_counter.rs index 4fec1138..48f616f7 100644 --- a/src/model/index_counter.rs +++ b/src/model/index_counter.rs @@ -121,7 +121,7 @@ impl TableSchema for CounterTable { _tx: &mut db::Transaction, _old: Option<&Self::E>, _new: Option<&Self::E>, - ) -> db::Result<()> { + ) -> db::TxOpResult<()> { // nothing for now Ok(()) } diff --git a/src/model/k2v/item_table.rs b/src/model/k2v/item_table.rs index 77446f64..991fe66d 100644 --- a/src/model/k2v/item_table.rs +++ b/src/model/k2v/item_table.rs @@ -227,7 +227,7 @@ impl TableSchema for K2VItemTable { tx: &mut db::Transaction, old: Option<&Self::E>, new: Option<&Self::E>, - ) -> db::Result<()> { + ) -> db::TxOpResult<()> { // 1. Count let (old_entries, old_conflicts, old_values, old_bytes) = match old { None => (0, 0, 0, 0), @@ -245,7 +245,7 @@ impl TableSchema for K2VItemTable { .map(|e| &e.partition.partition_key) .unwrap_or_else(|| &new.unwrap().partition.partition_key); - match self.counter_table.count( + let counter_res = self.counter_table.count( tx, &count_pk, count_sk, @@ -255,18 +255,15 @@ impl TableSchema for K2VItemTable { (VALUES, new_values - old_values), (BYTES, new_bytes - old_bytes), ], - ) { - Ok(()) => (), - Err(db::TxError::Db(e)) => return Err(e), - Err(db::TxError::Abort(e)) => { - // This result can be returned by `counter_table.count()` for instance - // if messagepack serialization or deserialization fails at some step. - // Warn admin but ignore this error for now, that's all we can do. - error!( - "Unable to update K2V item counter for bucket {:?} partition {}: {}. Index values will be wrong!", - count_pk, count_sk, e - ); - } + ); + if let Err(e) = db::unabort(counter_res)? { + // This result can be returned by `counter_table.count()` for instance + // if messagepack serialization or deserialization fails at some step. + // Warn admin but ignore this error for now, that's all we can do. + error!( + "Unable to update K2V item counter for bucket {:?} partition {}: {}. Index values will be wrong!", + count_pk, count_sk, e + ); } // 2. Notify diff --git a/src/model/s3/block_ref_table.rs b/src/model/s3/block_ref_table.rs index 2c06bc96..1134922e 100644 --- a/src/model/s3/block_ref_table.rs +++ b/src/model/s3/block_ref_table.rs @@ -58,7 +58,7 @@ impl TableSchema for BlockRefTable { tx: &mut db::Transaction, old: Option<&Self::E>, new: Option<&Self::E>, - ) -> db::Result<()> { + ) -> db::TxOpResult<()> { #[allow(clippy::or_fun_call)] let block = old.or(new).unwrap().block; let was_before = old.map(|x| !x.deleted.get()).unwrap_or(false); diff --git a/src/model/s3/object_table.rs b/src/model/s3/object_table.rs index f3bd9892..62f5d8d9 100644 --- a/src/model/s3/object_table.rs +++ b/src/model/s3/object_table.rs @@ -239,7 +239,7 @@ impl TableSchema for ObjectTable { _tx: &mut db::Transaction, old: Option<&Self::E>, new: Option<&Self::E>, - ) -> db::Result<()> { + ) -> db::TxOpResult<()> { let version_table = self.version_table.clone(); let old = old.cloned(); let new = new.cloned(); diff --git a/src/model/s3/version_table.rs b/src/model/s3/version_table.rs index d168c2c2..881c245a 100644 --- a/src/model/s3/version_table.rs +++ b/src/model/s3/version_table.rs @@ -144,7 +144,7 @@ impl TableSchema for VersionTable { _tx: &mut db::Transaction, old: Option<&Self::E>, new: Option<&Self::E>, - ) -> db::Result<()> { + ) -> db::TxOpResult<()> { let block_ref_table = self.block_ref_table.clone(); let old = old.cloned(); let new = new.cloned(); diff --git a/src/table/schema.rs b/src/table/schema.rs index 393c7388..74f57798 100644 --- a/src/table/schema.rs +++ b/src/table/schema.rs @@ -93,7 +93,7 @@ pub trait TableSchema: Send + Sync { _tx: &mut db::Transaction, _old: Option<&Self::E>, _new: Option<&Self::E>, - ) -> db::Result<()> { + ) -> db::TxOpResult<()> { Ok(()) } -- 2.43.0 From c103ca8bf2f7c8168f4224ae610aed7d82c19618 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 17:56:11 +0200 Subject: [PATCH 56/64] Correct error type for (unimplemented) iterators in transactions --- src/db/lib.rs | 17 +++++++++-------- src/db/lmdb_adapter.rs | 10 +++++----- src/db/sled_adapter.rs | 10 +++++----- src/db/sqlite_adapter.rs | 10 +++++----- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/db/lib.rs b/src/db/lib.rs index a4da4086..d4b5c519 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -25,6 +25,7 @@ pub struct Tree(Arc, usize); pub type Value = Vec; pub type ValueIter<'a> = Box> + 'a>; +pub type TxValueIter<'a> = Box> + 'a>; // ---- @@ -251,16 +252,16 @@ impl<'a> Transaction<'a> { } #[inline] - pub fn iter(&self, tree: &Tree) -> TxOpResult> { + pub fn iter(&self, tree: &Tree) -> TxOpResult> { self.0.iter(tree.1) } #[inline] - pub fn iter_rev(&self, tree: &Tree) -> TxOpResult> { + pub fn iter_rev(&self, tree: &Tree) -> TxOpResult> { self.0.iter_rev(tree.1) } #[inline] - pub fn range(&self, tree: &Tree, range: R) -> TxOpResult> + pub fn range(&self, tree: &Tree, range: R) -> TxOpResult> where K: AsRef<[u8]>, R: RangeBounds, @@ -270,7 +271,7 @@ impl<'a> Transaction<'a> { self.0.range(tree.1, get_bound(sb), get_bound(eb)) } #[inline] - pub fn range_rev(&self, tree: &Tree, range: R) -> TxOpResult> + pub fn range_rev(&self, tree: &Tree, range: R) -> TxOpResult> where K: AsRef<[u8]>, R: RangeBounds, @@ -331,21 +332,21 @@ pub(crate) trait ITx { fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> TxOpResult>; fn remove(&mut self, tree: usize, key: &[u8]) -> TxOpResult>; - fn iter(&self, tree: usize) -> TxOpResult>; - fn iter_rev(&self, tree: usize) -> TxOpResult>; + fn iter(&self, tree: usize) -> TxOpResult>; + fn iter_rev(&self, tree: usize) -> TxOpResult>; fn range<'r>( &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> TxOpResult>; + ) -> TxOpResult>; fn range_rev<'r>( &self, tree: usize, low: Bound<&'r [u8]>, high: Bound<&'r [u8]>, - ) -> TxOpResult>; + ) -> TxOpResult>; } pub(crate) trait ITxFn { diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index d1efd216..50885dad 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -10,7 +10,7 @@ use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; use crate::{ Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, - Value, ValueIter, + Value, ValueIter, TxValueIter }; pub use heed; @@ -243,10 +243,10 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { Ok(old_val) } - fn iter(&self, _tree: usize) -> TxOpResult> { + fn iter(&self, _tree: usize) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with LMDB backend"); } - fn iter_rev(&self, _tree: usize) -> TxOpResult> { + fn iter_rev(&self, _tree: usize) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with LMDB backend"); } @@ -255,7 +255,7 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> TxOpResult> { + ) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with LMDB backend"); } fn range_rev<'r>( @@ -263,7 +263,7 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> TxOpResult> { + ) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with LMDB backend"); } } diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index d0d9e9c0..bcda32a6 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -11,7 +11,7 @@ use sled::transaction::{ use crate::{ Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, - Value, ValueIter, + Value, ValueIter, TxValueIter }; pub use sled; @@ -230,10 +230,10 @@ impl<'a> ITx for SledTx<'a> { Ok(old_val.map(|x| x.to_vec())) } - fn iter(&self, _tree: usize) -> TxOpResult> { + fn iter(&self, _tree: usize) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with Sled backend"); } - fn iter_rev(&self, _tree: usize) -> TxOpResult> { + fn iter_rev(&self, _tree: usize) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with Sled backend"); } @@ -242,7 +242,7 @@ impl<'a> ITx for SledTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> TxOpResult> { + ) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with Sled backend"); } fn range_rev<'r>( @@ -250,7 +250,7 @@ impl<'a> ITx for SledTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> TxOpResult> { + ) -> TxOpResult> { unimplemented!("Iterators in transactions not supported with Sled backend"); } } diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 8d6bd714..402ff7a4 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -12,7 +12,7 @@ use rusqlite::{params, Connection, Rows, Statement, Transaction}; use crate::{ Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, - Value, ValueIter, + Value, ValueIter, TxValueIter }; pub use rusqlite; @@ -372,10 +372,10 @@ impl<'a> ITx for SqliteTx<'a> { Ok(old_val) } - fn iter(&self, _tree: usize) -> TxOpResult> { + fn iter(&self, _tree: usize) -> TxOpResult> { unimplemented!(); } - fn iter_rev(&self, _tree: usize) -> TxOpResult> { + fn iter_rev(&self, _tree: usize) -> TxOpResult> { unimplemented!(); } @@ -384,7 +384,7 @@ impl<'a> ITx for SqliteTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> TxOpResult> { + ) -> TxOpResult> { unimplemented!(); } fn range_rev<'r>( @@ -392,7 +392,7 @@ impl<'a> ITx for SqliteTx<'a> { _tree: usize, _low: Bound<&'r [u8]>, _high: Bound<&'r [u8]>, - ) -> TxOpResult> { + ) -> TxOpResult> { unimplemented!(); } } -- 2.43.0 From daec7995c33e9d8213dec1f6256855737a83fd0c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 17:56:29 +0200 Subject: [PATCH 57/64] cargo fmt --- src/db/lmdb_adapter.rs | 2 +- src/db/sled_adapter.rs | 2 +- src/db/sqlite_adapter.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 50885dad..b82add75 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -10,7 +10,7 @@ use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; use crate::{ Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, - Value, ValueIter, TxValueIter + TxValueIter, Value, ValueIter, }; pub use heed; diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index bcda32a6..fb9353f0 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -11,7 +11,7 @@ use sled::transaction::{ use crate::{ Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, - Value, ValueIter, TxValueIter + TxValueIter, Value, ValueIter, }; pub use sled; diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 402ff7a4..251dcafb 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -12,7 +12,7 @@ use rusqlite::{params, Connection, Rows, Statement, Transaction}; use crate::{ Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxOpError, TxOpResult, TxResult, - Value, ValueIter, TxValueIter + TxValueIter, Value, ValueIter, }; pub use rusqlite; -- 2.43.0 From d41a67c4eea3dbb1e40e5eb0fcee28dd916c8f29 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 18:06:30 +0200 Subject: [PATCH 58/64] simplify & fix db tests --- src/db/lib.rs | 3 ++- src/db/lmdb_adapter.rs | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/db/lib.rs b/src/db/lib.rs index d4b5c519..9fae7cc9 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -35,10 +35,11 @@ pub struct Error(pub Cow<'static, str>); pub type Result = std::result::Result; +#[derive(Debug, Error)] +#[error(display = "{}", _0)] pub struct TxOpError(pub(crate) Error); pub type TxOpResult = std::result::Result; -#[derive(Debug)] pub enum TxError { Abort(E), Db(Error), diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index b82add75..23b8c87e 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -203,12 +203,12 @@ impl IDb for LmdbDb { // ---- -struct LmdbTx<'a, 'db> { - trees: &'db [Database], +struct LmdbTx<'a> { + trees: &'a [Database], tx: RwTxn<'a, 'a>, } -impl<'a, 'db> LmdbTx<'a, 'db> { +impl<'a> LmdbTx<'a> { fn get_tree(&self, i: usize) -> TxOpResult<&Database> { self.trees.get(i).ok_or_else(|| { TxOpError(Error( @@ -218,7 +218,7 @@ impl<'a, 'db> LmdbTx<'a, 'db> { } } -impl<'a, 'db> ITx for LmdbTx<'a, 'db> { +impl<'a> ITx for LmdbTx<'a> { fn get(&self, tree: usize, key: &[u8]) -> TxOpResult> { let tree = self.get_tree(tree)?; match tree.get(&self.tx, key)? { -- 2.43.0 From 845c344231280edc16c3e6a1ab65b97eb623134c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 18:20:51 +0200 Subject: [PATCH 59/64] Inform user of what DB engine Garage is running on --- src/api/admin/cluster.rs | 2 ++ src/db/lib.rs | 5 +++++ src/db/lmdb_adapter.rs | 4 ++++ src/db/sled_adapter.rs | 4 ++++ src/db/sqlite_adapter.rs | 4 ++++ src/garage/admin.rs | 1 + 6 files changed, 20 insertions(+) diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 6d01317d..4b7716a3 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -19,6 +19,7 @@ pub async fn handle_get_cluster_status(garage: &Arc) -> Result) -> GetClusterLayoutResponse { struct GetClusterStatusResponse { node: String, garage_version: &'static str, + db_engine: String, known_nodes: HashMap, layout: GetClusterLayoutResponse, } diff --git a/src/db/lib.rs b/src/db/lib.rs index 9fae7cc9..e9d3ea18 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -63,6 +63,10 @@ pub fn unabort(res: TxResult) -> TxOpResult String { + self.0.engine() + } + pub fn open_tree>(&self, name: S) -> Result { let tree_id = self.0.open_tree(name.as_ref())?; Ok(Tree(self.0.clone(), tree_id)) @@ -298,6 +302,7 @@ impl<'a> Transaction<'a> { // ---- Internal interfaces pub(crate) trait IDb: Send + Sync { + fn engine(&self) -> String; fn open_tree(&self, name: &str) -> Result; fn list_trees(&self) -> Result>; diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index 23b8c87e..74622919 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -57,6 +57,10 @@ impl LmdbDb { } impl IDb for LmdbDb { + fn engine(&self) -> String { + "LMDB (using Heed crate)".into() + } + fn open_tree(&self, name: &str) -> Result { let mut trees = self.trees.write().unwrap(); if let Some(i) = trees.1.get(name) { diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index fb9353f0..982f8d82 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -58,6 +58,10 @@ impl SledDb { } impl IDb for SledDb { + fn engine(&self) -> String { + "Sled".into() + } + fn open_tree(&self, name: &str) -> Result { let mut trees = self.trees.write().unwrap(); if let Some(i) = trees.1.get(name) { diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 251dcafb..84883772 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -71,6 +71,10 @@ impl SqliteDbInner { } impl IDb for SqliteDb { + fn engine(&self) -> String { + "Sqlite3 (using rusqlite crate)".into() + } + fn open_tree(&self, name: &str) -> Result { let name = format!("tree_{}", name.replace(':', "_COLON_")); let mut this = self.0.lock().unwrap(); diff --git a/src/garage/admin.rs b/src/garage/admin.rs index 3af8b046..c662aa00 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -672,6 +672,7 @@ impl AdminRpcHandler { self.garage.system.garage_version(), ) .unwrap(); + writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap(); // Gather ring statistics let ring = self.garage.system.ring.borrow().clone(); -- 2.43.0 From cc4f1aca978d9d4c71d89553e3bcf990c8efab18 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 18:22:27 +0200 Subject: [PATCH 60/64] sqlite can actually tell us its version --- src/db/sqlite_adapter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 84883772..e11a3e60 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -72,7 +72,7 @@ impl SqliteDbInner { impl IDb for SqliteDb { fn engine(&self) -> String { - "Sqlite3 (using rusqlite crate)".into() + format!("sqlite3 v{} (using rusqlite crate)", rusqlite::version()) } fn open_tree(&self, name: &str) -> Result { -- 2.43.0 From 48928d2d70ac34b0ea639d199a3e5c71322a5fd8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 18:26:28 +0200 Subject: [PATCH 61/64] Simplify sqlite --- src/db/sqlite_adapter.rs | 44 +++++++++++----------------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index e11a3e60..14bf35ff 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -154,22 +154,12 @@ impl IDb for SqliteDb { let tree = this.get_tree(tree)?; let old_val = this.internal_get(tree, key)?; - match &old_val { - Some(_) => { - let n = this.db.execute( - &format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), - params![key, value], - )?; - assert_eq!(n, 1); - } - None => { - let n = this.db.execute( - &format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), - params![key, value], - )?; - assert_eq!(n, 1); - } - } + let sql = match &old_val { + Some(_) => format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), + None => format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), + }; + let n = this.db.execute(&sql, params![key, value])?; + assert_eq!(n, 1); Ok(old_val) } @@ -343,22 +333,12 @@ impl<'a> ITx for SqliteTx<'a> { let tree = self.get_tree(tree)?; let old_val = self.internal_get(tree, key)?; - match &old_val { - Some(_) => { - let n = self.tx.execute( - &format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), - params![key, value], - )?; - assert_eq!(n, 1); - } - None => { - let n = self.tx.execute( - &format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), - params![key, value], - )?; - assert_eq!(n, 1); - } - } + let sql = match &old_val { + Some(_) => format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree), + None => format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree), + }; + let n = self.tx.execute(&sql, params![key, value])?; + assert_eq!(n, 1); Ok(old_val) } -- 2.43.0 From 0c0a02ad03eabceed6f213ac9e380ac20841fc17 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 18:36:13 +0200 Subject: [PATCH 62/64] Remove useless function --- src/model/index_counter.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/model/index_counter.rs b/src/model/index_counter.rs index 48f616f7..2602d5d9 100644 --- a/src/model/index_counter.rs +++ b/src/model/index_counter.rs @@ -116,16 +116,6 @@ impl TableSchema for CounterTable { type E = CounterEntry; type Filter = (DeletedFilter, Vec); - fn updated( - &self, - _tx: &mut db::Transaction, - _old: Option<&Self::E>, - _new: Option<&Self::E>, - ) -> db::TxOpResult<()> { - // nothing for now - Ok(()) - } - fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { if filter.0 == DeletedFilter::Any { return true; -- 2.43.0 From 1e4814568b3be937b7ddc55686a6aa2a34c587f6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 18:48:32 +0200 Subject: [PATCH 63/64] remove useless clippy allow --- src/model/s3/block_ref_table.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/model/s3/block_ref_table.rs b/src/model/s3/block_ref_table.rs index 1134922e..9589b4aa 100644 --- a/src/model/s3/block_ref_table.rs +++ b/src/model/s3/block_ref_table.rs @@ -59,7 +59,6 @@ impl TableSchema for BlockRefTable { old: Option<&Self::E>, new: Option<&Self::E>, ) -> db::TxOpResult<()> { - #[allow(clippy::or_fun_call)] let block = old.or(new).unwrap().block; let was_before = old.map(|x| !x.deleted.get()).unwrap_or(false); let is_after = new.map(|x| !x.deleted.get()).unwrap_or(false); -- 2.43.0 From e682478d753d63e34595d7a2b1e120cee658a3a8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 7 Jun 2022 19:11:11 +0200 Subject: [PATCH 64/64] Create meta directory in server (or else sqlite fails) --- src/garage/server.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/garage/server.rs b/src/garage/server.rs index 9a1aa975..697d3358 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -33,6 +33,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Opening database..."); let mut db_path = config.metadata_dir.clone(); + std::fs::create_dir_all(&db_path).expect("Unable to create Garage meta data directory"); let db = match config.db_engine.as_str() { "sled" => { db_path.push("db"); -- 2.43.0