Abstract database behind generic interface and implement alternative drivers #322

Merged
lx merged 64 commits from db-abstraction into main 2022-06-08 08:01:56 +00:00
11 changed files with 166 additions and 45 deletions
Showing only changes of commit f25309e58f - Show all commits

1
Cargo.lock generated
View file

@ -1015,6 +1015,7 @@ name = "garage_db"
version = "0.8.0"
dependencies = [
"err-derive 0.3.1",
"hexdump",
"mktemp",
"rusqlite",
"sled",

View file

@ -645,7 +645,7 @@ impl BlockManager {
None => Ok(None),
Some(v) => {
let (time_bytes, hash_bytes) = v?;
Ok(Some((time_bytes.into_owned(), hash_bytes.into_owned())))
Ok(Some((time_bytes.into_vec(), hash_bytes.into_vec())))
}
}
}

View file

@ -15,6 +15,7 @@ path = "lib.rs"
[dependencies]
err-derive = "0.3"
hexdump = "0.1"
sled = "0.34"
rusqlite = "0.27"

View file

@ -21,13 +21,98 @@ pub struct Transaction<'a>(pub(crate) &'a dyn ITx<'a>);
#[derive(Clone)]
pub struct Tree(pub(crate) Arc<dyn IDb>, pub(crate) usize);
pub type Value<'a> = Cow<'a, [u8]>;
pub type ValueIter<'a> = Box<dyn std::iter::Iterator<Item = Result<(Value<'a>, Value<'a>)>> + 'a>;
pub type Exporter<'a> = Box<dyn std::iter::Iterator<Item = Result<(String, ValueIter<'a>)>> + 'a>;
// ----
pub struct Value<'a>(pub(crate) Box<dyn IValue<'a> + 'a>);
pub trait IValue<'a>: AsRef<[u8]> {
fn into_vec(&mut self) -> Vec<u8>;
}
impl<'a> Value<'a> {
#[inline]
pub fn into_vec(mut self) -> Vec<u8> {
self.0.into_vec()
}
}
impl<'a> AsRef<[u8]> for Value<'a> {
#[inline]
fn as_ref(&self) -> &[u8] {
self.0.as_ref().as_ref()
}
}
impl<'a> std::borrow::Borrow<[u8]> for Value<'a> {
#[inline]
fn borrow(&self) -> &[u8] {
self.0.as_ref().as_ref()
}
}
impl<'a> core::ops::Deref for Value<'a> {
type Target = [u8];
#[inline]
fn deref(&self) -> &[u8] {
self.0.as_ref().as_ref()
}
}
impl<'a, T> PartialEq<T> for Value<'a>
where
T: AsRef<[u8]>,
{
fn eq(&self, other: &T) -> bool {
self.as_ref() == other.as_ref()
}
}
impl<'a> std::fmt::Debug for Value<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
for line in hexdump::hexdump_iter(self.as_ref()) {
f.write_str(&line)?;
f.write_str("\n")?;
}
Ok(())
}
}
impl<'a> IValue<'a> for Vec<u8> {
fn into_vec(&mut self) -> Vec<u8> {
std::mem::take(self)
}
}
impl<'a> From<Value<'a>> for Vec<u8> {
fn from(v: Value<'a>) -> Vec<u8> {
v.into_vec()
}
}
impl<'a> From<Vec<u8>> for Value<'a> {
fn from(v: Vec<u8>) -> Value<'a> {
Value(Box::new(v))
}
}
impl<'a> From<&'a [u8]> for Value<'a> {
fn from(v: &'a [u8]) -> Value<'a> {
Value(Box::new(v))
}
}
impl<'a> IValue<'a> for &'a [u8] {
fn into_vec(&mut self) -> Vec<u8> {
self.to_vec()
}
}
// ----
#[derive(Debug, Error)]
#[error(display = "{}", _0)]
pub struct Error(Cow<'static, str>);

View file

@ -10,17 +10,42 @@ use sled::transaction::{
};
use crate::{
Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
Db, Error, Exporter, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value,
ValueIter,
};
pub use sled;
// -- err
impl From<sled::Error> for Error {
fn from(e: sled::Error) -> Error {
Error(format!("{}", e).into())
}
}
// -- val
impl<'a> IValue<'a> for sled::IVec {
fn into_vec(&mut self) -> Vec<u8> {
self.to_vec()
}
}
impl<'a> From<Value<'a>> for sled::IVec {
fn from(v: Value<'a>) -> sled::IVec {
sled::IVec::from(v.into_vec())
}
}
impl<'a> From<sled::IVec> for Value<'a> {
fn from(v: sled::IVec) -> Value<'a> {
Value(Box::new(v))
}
}
// -- db
pub struct SledDb {
db: sled::Db,
trees: RwLock<(Vec<sled::Tree>, HashMap<String, usize>)>,
@ -64,7 +89,7 @@ impl IDb for SledDb {
fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result<Option<Value<'a>>> {
let tree = self.get_tree(tree)?;
Ok(tree.get(key)?.map(|v| v.to_vec().into()))
Ok(tree.get(key)?.map(From::from))
}
fn remove(&self, tree: usize, key: &[u8]) -> Result<bool> {
@ -86,16 +111,14 @@ impl IDb for SledDb {
fn iter<'a>(&'a self, tree: usize) -> Result<ValueIter<'a>> {
let tree = self.get_tree(tree)?;
Ok(Box::new(tree.iter().map(|v| {
v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into()))
.map_err(Into::into)
v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)
})))
}
fn iter_rev<'a>(&'a self, tree: usize) -> Result<ValueIter<'a>> {
let tree = self.get_tree(tree)?;
Ok(Box::new(tree.iter().rev().map(|v| {
v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into()))
.map_err(Into::into)
v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)
})))
}
@ -107,8 +130,7 @@ impl IDb for SledDb {
) -> Result<ValueIter<'a>> {
let tree = self.get_tree(tree)?;
Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).map(|v| {
v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into()))
.map_err(Into::into)
v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)
})))
}
fn range_rev<'a, 'r>(
@ -119,10 +141,7 @@ impl IDb for SledDb {
) -> Result<ValueIter<'a>> {
let tree = self.get_tree(tree)?;
Ok(Box::new(tree.range::<&'r [u8], _>((low, high)).rev().map(
|v| {
v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into()))
.map_err(Into::into)
},
|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into),
)))
}
@ -170,10 +189,10 @@ impl IDb for SledDb {
trees.push((name, tree));
}
let trees_exporter: Exporter<'a> = Box::new(trees.into_iter().map(|(name, tree)| {
let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| {
v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into()))
.map_err(Into::into)
}));
let iter: ValueIter<'a> = Box::new(
tree.iter()
.map(|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)),
);
Ok((name.to_string(), iter))
}));
Ok(trees_exporter)
@ -192,7 +211,7 @@ impl IDb for SledDb {
let mut i = 0;
for item in data {
let (k, v) = item?;
tree.insert(k.as_ref(), v.as_ref())?;
tree.insert(k, v)?;
i += 1;
if i % 1000 == 0 {
println!("{}: imported {}", name, i);
@ -236,7 +255,7 @@ impl<'a> ITx<'a> for SledTx<'a> {
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'a>>> {
let tree = self.get_tree(tree)?;
let tmp = self.save_error(tree.get(key))?;
Ok(tmp.map(|v| v.to_vec().into()))
Ok(tmp.map(From::from))
}
fn len(&self, _tree: usize) -> Result<usize> {
unimplemented!(".len() in transaction not supported with Sled backend")

View file

@ -13,6 +13,8 @@ use crate::{
pub use rusqlite;
// --- err
impl From<rusqlite::Error> for Error {
fn from(e: rusqlite::Error) -> Error {
Error(format!("{}", e).into())
@ -25,6 +27,8 @@ impl<T> From<rusqlite::Error> for TxError<T> {
}
}
// -- db
pub struct SqliteDb {
db: Mutex<Connection>,
trees: RwLock<Vec<String>>,

View file

@ -14,59 +14,66 @@ fn test_suite(db: Db) {
let vc: &[u8] = &b"plup"[..];
tree.insert(ka, va).unwrap();
assert_eq!(tree.get(ka).unwrap(), Some(va.into()));
assert_eq!(tree.get(ka).unwrap().unwrap(), va);
let res = db.transaction::<_, (), _>(|tx| {
assert_eq!(tx.get(&tree, ka).unwrap(), Some(va.into()));
assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), va);
tx.insert(&tree, ka, vb).unwrap();
assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into()));
assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb);
tx.commit(12)
});
assert!(matches!(res, Ok(12)));
assert_eq!(tree.get(ka).unwrap(), Some(vb.into()));
assert_eq!(tree.get(ka).unwrap().unwrap(), vb);
let res = db.transaction::<(), _, _>(|tx| {
assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into()));
assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vb);
tx.insert(&tree, ka, vc).unwrap();
assert_eq!(tx.get(&tree, ka).unwrap(), Some(vc.into()));
assert_eq!(tx.get(&tree, ka).unwrap().unwrap(), vc);
tx.abort(42)
});
assert!(matches!(res, Err(TxError::Abort(42))));
assert_eq!(tree.get(ka).unwrap(), Some(vb.into()));
assert_eq!(tree.get(ka).unwrap().unwrap(), vb);
let mut iter = tree.iter().unwrap();
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
let next = iter.next().unwrap().unwrap();
assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb));
assert!(iter.next().is_none());
drop(iter);
tree.insert(kb, vc).unwrap();
assert_eq!(tree.get(kb).unwrap(), Some(vc.into()));
assert_eq!(tree.get(kb).unwrap().unwrap(), vc);
let mut iter = tree.iter().unwrap();
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into()));
let next = iter.next().unwrap().unwrap();
assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb));
let next = iter.next().unwrap().unwrap();
assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc));
assert!(iter.next().is_none());
drop(iter);
let mut iter = tree.range(kint..).unwrap();
assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into()));
let next = iter.next().unwrap().unwrap();
assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc));
assert!(iter.next().is_none());
drop(iter);
let mut iter = tree.range_rev(..kint).unwrap();
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
let next = iter.next().unwrap().unwrap();
assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb));
assert!(iter.next().is_none());
drop(iter);
let mut iter = tree.iter_rev().unwrap();
assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into()));
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
let next = iter.next().unwrap().unwrap();
assert_eq!((next.0.as_ref(), next.1.as_ref()), (kb, vc));
let next = iter.next().unwrap().unwrap();
assert_eq!((next.0.as_ref(), next.1.as_ref()), (ka, vb));
assert!(iter.next().is_none());
drop(iter);
}

View file

@ -117,7 +117,7 @@ impl Repair {
None => Ok(None),
Some(item) => {
let (item_key, item_bytes) = item?;
Ok(Some((item_key.into_owned(), item_bytes.into_owned())))
Ok(Some((item_key.into_vec(), item_bytes.into_vec())))
}
}
}
@ -173,7 +173,7 @@ impl Repair {
None => Ok(None),
Some(item) => {
let (item_key, item_bytes) = item?;
Ok(Some((item_key.into_owned(), item_bytes.into_owned())))
Ok(Some((item_key.into_vec(), item_bytes.into_vec())))
}
}
}

View file

@ -276,7 +276,7 @@ where
if blake2sum(&cur_v[..]) == vhash {
tx.remove(&self.store, k)?;
tx.insert(&self.merkle_todo, k, vec![])?;
return Ok(Some(cur_v.into_owned()));
return Ok(Some(cur_v.into_vec()));
}
}
Ok(None)

View file

@ -378,9 +378,12 @@ impl GcTodoEntry {
let key = self.todo_table_key();
gc_todo_tree.db().transaction(|tx| {
let old_val = tx.get(gc_todo_tree, &key)?;
if old_val == Some(self.value_hash.as_slice().into()) {
match old_val {
Some(ov) if ov == self.value_hash.as_slice() => {
tx.remove(gc_todo_tree, &key)?;
}
_ => (),
}
tx.commit(())
})?;
Ok(())

View file

@ -142,11 +142,12 @@ where
let deleted = self.data.merkle_todo.db().transaction(|tx| {
let old_val = tx.get(&self.data.merkle_todo, k)?;
if old_val == Some(vhash_by.into()) {
match old_val {
Some(ov) if ov == vhash_by => {
tx.remove(&self.data.merkle_todo, k)?;
tx.commit(true)
} else {
tx.commit(false)
}
_ => tx.commit(false),
}
})?;