Abstract database behind generic interface and implement alternative drivers #322
6 changed files with 498 additions and 102 deletions
104
Cargo.lock
generated
104
Cargo.lock
generated
|
@ -2,6 +2,23 @@
|
||||||
# It is not intended for manual editing.
|
# It is not intended for manual editing.
|
||||||
version = 3
|
version = 3
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "Inflector"
|
||||||
|
version = "0.11.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ahash"
|
||||||
|
version = "0.7.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom",
|
||||||
|
"once_cell",
|
||||||
|
"version_check",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aho-corasick"
|
name = "aho-corasick"
|
||||||
version = "0.7.18"
|
version = "0.7.18"
|
||||||
|
@ -11,6 +28,12 @@ dependencies = [
|
||||||
"memchr",
|
"memchr",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aliasable"
|
||||||
|
version = "0.1.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "anyhow"
|
name = "anyhow"
|
||||||
version = "1.0.56"
|
version = "1.0.56"
|
||||||
|
@ -716,6 +739,18 @@ dependencies = [
|
||||||
"synstructure",
|
"synstructure",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fallible-iterator"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fallible-streaming-iterator"
|
||||||
|
version = "0.1.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fastrand"
|
name = "fastrand"
|
||||||
version = "1.7.0"
|
version = "1.7.0"
|
||||||
|
@ -993,6 +1028,8 @@ version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"err-derive 0.3.1",
|
"err-derive 0.3.1",
|
||||||
"mktemp",
|
"mktemp",
|
||||||
|
"ouroboros",
|
||||||
|
"rusqlite",
|
||||||
"sled",
|
"sled",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -1297,6 +1334,18 @@ name = "hashbrown"
|
||||||
version = "0.11.2"
|
version = "0.11.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
|
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
|
||||||
|
dependencies = [
|
||||||
|
"ahash",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hashlink"
|
||||||
|
version = "0.7.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
|
||||||
|
dependencies = [
|
||||||
|
"hashbrown",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "heck"
|
name = "heck"
|
||||||
|
@ -1767,6 +1816,16 @@ dependencies = [
|
||||||
"walkdir",
|
"walkdir",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libsqlite3-sys"
|
||||||
|
version = "0.24.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"
|
||||||
|
dependencies = [
|
||||||
|
"pkg-config",
|
||||||
|
"vcpkg",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "linked-hash-map"
|
name = "linked-hash-map"
|
||||||
version = "0.5.4"
|
version = "0.5.4"
|
||||||
|
@ -2152,6 +2211,30 @@ version = "6.0.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"
|
checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ouroboros"
|
||||||
|
version = "0.15.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9f31a3b678685b150cba82b702dcdc5e155893f63610cf388d30cd988d4ca2bf"
|
||||||
|
dependencies = [
|
||||||
|
"aliasable",
|
||||||
|
"ouroboros_macro",
|
||||||
|
"stable_deref_trait",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ouroboros_macro"
|
||||||
|
version = "0.15.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "084fd65d5dd8b3772edccb5ffd1e4b7eba43897ecd0f9401e330e8c542959408"
|
||||||
|
dependencies = [
|
||||||
|
"Inflector",
|
||||||
|
"proc-macro-error",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parking_lot"
|
name = "parking_lot"
|
||||||
version = "0.11.2"
|
version = "0.11.2"
|
||||||
|
@ -2697,6 +2780,21 @@ dependencies = [
|
||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rusqlite"
|
||||||
|
version = "0.27.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"fallible-iterator",
|
||||||
|
"fallible-streaming-iterator",
|
||||||
|
"hashlink",
|
||||||
|
"libsqlite3-sys",
|
||||||
|
"memchr",
|
||||||
|
"smallvec",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc_version"
|
name = "rustc_version"
|
||||||
version = "0.4.0"
|
version = "0.4.0"
|
||||||
|
@ -3006,6 +3104,12 @@ version = "0.9.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"
|
checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "stable_deref_trait"
|
||||||
|
version = "1.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "static_init"
|
name = "static_init"
|
||||||
version = "1.0.2"
|
version = "1.0.2"
|
||||||
|
|
|
@ -15,8 +15,10 @@ path = "lib.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
err-derive = "0.3"
|
err-derive = "0.3"
|
||||||
|
ouroboros = "0.15"
|
||||||
|
|
||||||
sled = "0.34"
|
sled = "0.34"
|
||||||
|
rusqlite = "0.27"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
mktemp = "0.4"
|
mktemp = "0.4"
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
pub mod sled_adapter;
|
pub mod sled_adapter;
|
||||||
|
pub mod sqlite_adapter;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub mod test;
|
pub mod test;
|
||||||
|
@ -22,10 +23,10 @@ pub struct Tree(pub(crate) Arc<dyn IDb>, pub(crate) usize);
|
||||||
|
|
||||||
pub type Value<'a> = Cow<'a, [u8]>;
|
pub type Value<'a> = Cow<'a, [u8]>;
|
||||||
pub type ValueIter<'a> =
|
pub type ValueIter<'a> =
|
||||||
Box<dyn std::iter::Iterator<Item = Result<(Value<'a>, Value<'a>)>> + Send + Sync + 'a>;
|
Box<dyn std::iter::Iterator<Item = Result<(Value<'a>, Value<'a>)>> + 'a>;
|
||||||
|
|
||||||
pub type Exporter<'a> =
|
pub type Exporter<'a> =
|
||||||
Box<dyn std::iter::Iterator<Item = Result<(String, ValueIter<'a>)>> + Send + Sync + 'a>;
|
Box<dyn std::iter::Iterator<Item = Result<(String, ValueIter<'a>)>> + 'a>;
|
||||||
|
|
||||||
// ----
|
// ----
|
||||||
|
|
||||||
|
@ -64,26 +65,40 @@ impl Db {
|
||||||
function: fun,
|
function: fun,
|
||||||
result: Cell::new(None),
|
result: Cell::new(None),
|
||||||
};
|
};
|
||||||
match self.0.transaction(&f) {
|
let tx_res = self.0.transaction(&f);
|
||||||
Err(TxError::Db(e)) => Err(TxError::Db(e)),
|
let ret = f
|
||||||
Err(TxError::Abort(())) => {
|
.result
|
||||||
let r = f
|
.into_inner()
|
||||||
.result
|
.expect("Transaction did not store result");
|
||||||
.into_inner()
|
|
||||||
.expect("Transaction did not store result");
|
match tx_res {
|
||||||
assert!(matches!(r, Err(TxError::Abort(_))));
|
|
||||||
r
|
|
||||||
}
|
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
let r = f
|
assert!(matches!(ret, Ok(_)));
|
||||||
.result
|
ret
|
||||||
.into_inner()
|
|
||||||
.expect("Transaction did not store result");
|
|
||||||
assert!(matches!(r, Ok(_)));
|
|
||||||
r
|
|
||||||
}
|
}
|
||||||
|
Err(TxError::Abort(())) => {
|
||||||
|
assert!(matches!(ret, Err(TxError::Abort(_))));
|
||||||
|
ret
|
||||||
|
}
|
||||||
|
Err(TxError::Db(e2)) => match ret {
|
||||||
|
// Ok was stored -> the error occured when finalizing
|
||||||
|
// transaction
|
||||||
|
Ok(_) => Err(TxError::Db(e2)),
|
||||||
|
// An error was already stored: that's the one we want to
|
||||||
|
// return
|
||||||
|
Err(TxError::Db(e)) => Err(TxError::Db(e)),
|
||||||
|
_ => unreachable!(),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn export<'a>(&'a self) -> Result<Exporter<'a>> {
|
||||||
|
self.0.export()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> {
|
||||||
|
self.0.import(ex)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Tree {
|
impl Tree {
|
||||||
|
@ -181,14 +196,12 @@ impl<'a> Transaction<'a> {
|
||||||
// ----
|
// ----
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn abort<R, E>(self, e: E) -> TxResult<R, E>
|
pub fn abort<R, E>(self, e: E) -> TxResult<R, E> {
|
||||||
{
|
|
||||||
Err(TxError::Abort(e))
|
Err(TxError::Abort(e))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn commit<R, E>(self, r: R) -> TxResult<R, E>
|
pub fn commit<R, E>(self, r: R) -> TxResult<R, E> {
|
||||||
{
|
|
||||||
Ok(r)
|
Ok(r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -221,6 +234,9 @@ pub(crate) trait IDb: Send + Sync {
|
||||||
) -> Result<ValueIter<'a>>;
|
) -> Result<ValueIter<'a>>;
|
||||||
|
|
||||||
fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>;
|
fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>;
|
||||||
|
|
||||||
|
fn export<'a>(&'a self) -> Result<Exporter<'a>>;
|
||||||
|
fn import<'a>(&self, ex: Exporter<'a>) -> Result<()>;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) trait ITx<'a> {
|
pub(crate) trait ITx<'a> {
|
||||||
|
@ -251,10 +267,10 @@ pub(crate) trait ITxFn {
|
||||||
fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult;
|
fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
enum TxFnResult {
|
pub(crate) enum TxFnResult {
|
||||||
Abort,
|
|
||||||
Ok,
|
Ok,
|
||||||
Err,
|
Abort,
|
||||||
|
DbErr,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct TxFn<F, R, E>
|
struct TxFn<F, R, E>
|
||||||
|
@ -271,13 +287,13 @@ where
|
||||||
{
|
{
|
||||||
fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult {
|
fn try_on<'a>(&'a self, tx: &'a dyn ITx<'a>) -> TxFnResult {
|
||||||
let res = (self.function)(Transaction(tx));
|
let res = (self.function)(Transaction(tx));
|
||||||
let retval = match &res {
|
let res2 = match &res {
|
||||||
Ok(_) => TxFnResult::Ok,
|
Ok(_) => TxFnResult::Ok,
|
||||||
Err(TxError::Abort(_)) => TxFnResult::Abort,
|
Err(TxError::Abort(_)) => TxFnResult::Abort,
|
||||||
Err(TxError::Db(_)) => TxFnResult::Err,
|
Err(TxError::Db(_)) => TxFnResult::DbErr,
|
||||||
};
|
};
|
||||||
self.result.set(Some(res));
|
self.result.set(Some(res));
|
||||||
retval
|
res2
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -44,50 +44,6 @@ impl SledDb {
|
||||||
.cloned()
|
.cloned()
|
||||||
.ok_or(Error("invalid tree id".into()))
|
.ok_or(Error("invalid tree id".into()))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn export<'a>(&'a self) -> Result<Exporter<'a>> {
|
|
||||||
let mut trees = vec![];
|
|
||||||
for name in self.db.tree_names() {
|
|
||||||
let name = std::str::from_utf8(&name)
|
|
||||||
.map_err(|e| Error(format!("{}", e).into()))?
|
|
||||||
.to_string();
|
|
||||||
let tree = self.open_tree(&name)?;
|
|
||||||
let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
|
|
||||||
trees.push((name, tree));
|
|
||||||
}
|
|
||||||
let trees_exporter: Exporter<'a> = Box::new(trees.into_iter().map(|(name, tree)| {
|
|
||||||
let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| {
|
|
||||||
v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into()))
|
|
||||||
.map_err(Into::into)
|
|
||||||
}));
|
|
||||||
Ok((name.to_string(), iter))
|
|
||||||
}));
|
|
||||||
Ok(trees_exporter)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> {
|
|
||||||
for ex_tree in ex {
|
|
||||||
let (name, data) = ex_tree?;
|
|
||||||
|
|
||||||
let tree = self.open_tree(&name)?;
|
|
||||||
let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
|
|
||||||
if !tree.is_empty() {
|
|
||||||
return Err(Error(format!("tree {} already contains data", name).into()));
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut i = 0;
|
|
||||||
for item in data {
|
|
||||||
let (k, v) = item?;
|
|
||||||
tree.insert(k.as_ref(), v.as_ref())?;
|
|
||||||
i += 1;
|
|
||||||
if i % 1000 == 0 {
|
|
||||||
println!("{}: imported {}", name, i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
println!("{}: finished importing, {} items", name, i);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IDb for SledDb {
|
impl IDb for SledDb {
|
||||||
|
@ -104,6 +60,8 @@ impl IDb for SledDb {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result<Option<Value<'a>>> {
|
fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result<Option<Value<'a>>> {
|
||||||
let tree = self.get_tree(tree)?;
|
let tree = self.get_tree(tree)?;
|
||||||
Ok(tree.get(key)?.map(|v| v.to_vec().into()))
|
Ok(tree.get(key)?.map(|v| v.to_vec().into()))
|
||||||
|
@ -168,6 +126,8 @@ impl IDb for SledDb {
|
||||||
)))
|
)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> {
|
fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> {
|
||||||
let trees = self.trees.read().unwrap();
|
let trees = self.trees.read().unwrap();
|
||||||
let res = trees.0.transaction(|txtrees| {
|
let res = trees.0.transaction(|txtrees| {
|
||||||
|
@ -184,12 +144,9 @@ impl IDb for SledDb {
|
||||||
assert!(tx.err.into_inner().is_none());
|
assert!(tx.err.into_inner().is_none());
|
||||||
Err(ConflictableTransactionError::Abort(()))
|
Err(ConflictableTransactionError::Abort(()))
|
||||||
}
|
}
|
||||||
TxFnResult::Err => {
|
TxFnResult::DbErr => {
|
||||||
let err = tx
|
let e = tx.err.into_inner().expect("No DB error");
|
||||||
.err
|
Err(e.into())
|
||||||
.into_inner()
|
|
||||||
.expect("Transaction did not store error");
|
|
||||||
Err(err.into())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -199,6 +156,54 @@ impl IDb for SledDb {
|
||||||
Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())),
|
Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
fn export<'a>(&'a self) -> Result<Exporter<'a>> {
|
||||||
|
let mut trees = vec![];
|
||||||
|
for name in self.db.tree_names() {
|
||||||
|
let name = std::str::from_utf8(&name)
|
||||||
|
.map_err(|e| Error(format!("{}", e).into()))?
|
||||||
|
.to_string();
|
||||||
|
let tree = self.open_tree(&name)?;
|
||||||
|
let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
|
||||||
|
trees.push((name, tree));
|
||||||
|
}
|
||||||
|
let trees_exporter: Exporter<'a> = Box::new(trees.into_iter().map(|(name, tree)| {
|
||||||
|
let iter: ValueIter<'a> = Box::new(tree.iter().map(|v| {
|
||||||
|
v.map(|(x, y)| (x.to_vec().into(), y.to_vec().into()))
|
||||||
|
.map_err(Into::into)
|
||||||
|
}));
|
||||||
|
Ok((name.to_string(), iter))
|
||||||
|
}));
|
||||||
|
Ok(trees_exporter)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> {
|
||||||
|
for ex_tree in ex {
|
||||||
|
let (name, data) = ex_tree?;
|
||||||
|
|
||||||
|
let tree = self.open_tree(&name)?;
|
||||||
|
let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
|
||||||
|
if !tree.is_empty() {
|
||||||
|
return Err(Error(format!("tree {} already contains data", name).into()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut i = 0;
|
||||||
|
for item in data {
|
||||||
|
let (k, v) = item?;
|
||||||
|
tree.insert(k.as_ref(), v.as_ref())?;
|
||||||
|
i += 1;
|
||||||
|
if i % 1000 == 0 {
|
||||||
|
println!("{}: imported {}", name, i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!("{}: finished importing, {} items", name, i);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
}
|
}
|
||||||
|
|
||||||
// ----
|
// ----
|
||||||
|
|
265
src/db/sqlite_adapter.rs
Normal file
265
src/db/sqlite_adapter.rs
Normal file
|
@ -0,0 +1,265 @@
|
||||||
|
use core::ops::Bound;
|
||||||
|
|
||||||
|
use std::cell::Cell;
|
||||||
|
use std::sync::{Arc, Mutex, RwLock, MutexGuard};
|
||||||
|
|
||||||
|
use ouroboros::self_referencing;
|
||||||
|
|
||||||
|
use rusqlite::{params, Connection, Transaction};
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub use rusqlite;
|
||||||
|
|
||||||
|
impl From<rusqlite::Error> for Error {
|
||||||
|
fn from(e: rusqlite::Error) -> Error {
|
||||||
|
Error(format!("{}", e).into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> From<rusqlite::Error> for TxError<T> {
|
||||||
|
fn from(e: rusqlite::Error) -> TxError<T> {
|
||||||
|
TxError::Db(e.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct SqliteDb {
|
||||||
|
db: Mutex<Connection>,
|
||||||
|
trees: RwLock<Vec<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SqliteDb {
|
||||||
|
pub fn new(db: rusqlite::Connection) -> Db {
|
||||||
|
let s = Self {
|
||||||
|
db: Mutex::new(db),
|
||||||
|
trees: RwLock::new(Vec::new()),
|
||||||
|
};
|
||||||
|
Db(Arc::new(s))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_tree(&self, i: usize) -> Result<String> {
|
||||||
|
self.trees
|
||||||
|
.read()
|
||||||
|
.unwrap()
|
||||||
|
.get(i)
|
||||||
|
.cloned()
|
||||||
|
.ok_or(Error("invalid tree id".into()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl IDb for SqliteDb {
|
||||||
|
fn open_tree(&self, name: &str) -> Result<usize> {
|
||||||
|
let mut trees = self.trees.write().unwrap();
|
||||||
|
if let Some(i) = trees.iter().position(|x| x == name) {
|
||||||
|
Ok(i)
|
||||||
|
} else {
|
||||||
|
self.db.lock().unwrap().execute(
|
||||||
|
&format!(
|
||||||
|
"CREATE TABLE IF NOT EXISTS {} (
|
||||||
|
k BLOB PRIMARY KEY,
|
||||||
|
v BLOB
|
||||||
|
)",
|
||||||
|
name
|
||||||
|
),
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
let i = trees.len();
|
||||||
|
trees.push(name.to_string());
|
||||||
|
Ok(i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
fn get<'a>(&'a self, tree: usize, key: &[u8]) -> Result<Option<Value<'a>>> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?;
|
||||||
|
let mut res_iter = stmt.query([key])?;
|
||||||
|
match res_iter.next()? {
|
||||||
|
None => Ok(None),
|
||||||
|
Some(v) => Ok(Some(v.get::<_, Vec<u8>>(0)?.into())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove(&self, tree: usize, key: &[u8]) -> Result<bool> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let res = db.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?;
|
||||||
|
Ok(res > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn len(&self, tree: usize) -> Result<usize> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let mut stmt = db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?;
|
||||||
|
let mut res_iter = stmt.query([])?;
|
||||||
|
match res_iter.next()? {
|
||||||
|
None => Ok(0),
|
||||||
|
Some(v) => Ok(v.get::<_, usize>(0)?.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
db.execute(
|
||||||
|
&format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree),
|
||||||
|
params![key, value],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iter<'a>(&'a self, tree: usize) -> Result<ValueIter<'a>> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree);
|
||||||
|
let mut stmt = db.prepare(&sql)?;
|
||||||
|
let res = stmt.query([])?;
|
||||||
|
unimplemented!();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iter_rev<'a>(&'a self, tree: usize) -> Result<ValueIter<'a>> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
unimplemented!();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn range<'a, 'r>(
|
||||||
|
&'a self,
|
||||||
|
tree: usize,
|
||||||
|
low: Bound<&'r [u8]>,
|
||||||
|
high: Bound<&'r [u8]>,
|
||||||
|
) -> Result<ValueIter<'a>> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
unimplemented!();
|
||||||
|
}
|
||||||
|
fn range_rev<'a, 'r>(
|
||||||
|
&'a self,
|
||||||
|
tree: usize,
|
||||||
|
low: Bound<&'r [u8]>,
|
||||||
|
high: Bound<&'r [u8]>,
|
||||||
|
) -> Result<ValueIter<'a>> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
unimplemented!();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()> {
|
||||||
|
let trees = self.trees.read().unwrap();
|
||||||
|
let mut db = self.db.lock().unwrap();
|
||||||
|
let tx = SqliteTx {
|
||||||
|
tx: db.transaction()?,
|
||||||
|
trees: trees.as_ref(),
|
||||||
|
};
|
||||||
|
match f.try_on(&tx) {
|
||||||
|
TxFnResult::Ok => {
|
||||||
|
tx.tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
TxFnResult::Abort => {
|
||||||
|
tx.tx.rollback()?;
|
||||||
|
Err(TxError::Abort(()))
|
||||||
|
}
|
||||||
|
TxFnResult::DbErr => {
|
||||||
|
tx.tx.rollback()?;
|
||||||
|
Err(TxError::Db(Error(
|
||||||
|
"(this message will be discarded)".into(),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
fn export<'a>(&'a self) -> Result<Exporter<'a>> {
|
||||||
|
unimplemented!()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn import<'a>(&self, ex: Exporter<'a>) -> Result<()> {
|
||||||
|
unimplemented!()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
struct SqliteTx<'a> {
|
||||||
|
tx: Transaction<'a>,
|
||||||
|
trees: &'a [String],
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> SqliteTx<'a> {
|
||||||
|
fn get_tree(&self, i: usize) -> Result<String> {
|
||||||
|
self.trees.get(i).cloned().ok_or(Error(
|
||||||
|
"invalid tree id (it might have been openned after the transaction started)".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> ITx<'a> for SqliteTx<'a> {
|
||||||
|
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'a>>> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
let mut stmt = self
|
||||||
|
.tx
|
||||||
|
.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?;
|
||||||
|
let mut res_iter = stmt.query([key])?;
|
||||||
|
match res_iter.next()? {
|
||||||
|
None => Ok(None),
|
||||||
|
Some(v) => Ok(Some(v.get::<_, Vec<u8>>(0)?.into())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn len(&self, tree: usize) -> Result<usize> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
let mut stmt = self.tx.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?;
|
||||||
|
let mut res_iter = stmt.query([])?;
|
||||||
|
match res_iter.next()? {
|
||||||
|
None => Ok(0),
|
||||||
|
Some(v) => Ok(v.get::<_, usize>(0)?.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
self.tx.execute(
|
||||||
|
&format!("INSERT OR REPLACE INTO {} (k, v) VALUES (?1, ?2)", tree),
|
||||||
|
params![key, value],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
fn remove(&self, tree: usize, key: &[u8]) -> Result<bool> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
let res = self
|
||||||
|
.tx
|
||||||
|
.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?;
|
||||||
|
Ok(res > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iter(&self, _tree: usize) -> Result<ValueIter<'a>> {
|
||||||
|
unimplemented!();
|
||||||
|
}
|
||||||
|
fn iter_rev(&self, _tree: usize) -> Result<ValueIter<'a>> {
|
||||||
|
unimplemented!();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn range<'r>(
|
||||||
|
&self,
|
||||||
|
_tree: usize,
|
||||||
|
_low: Bound<&'r [u8]>,
|
||||||
|
_high: Bound<&'r [u8]>,
|
||||||
|
) -> Result<ValueIter<'a>> {
|
||||||
|
unimplemented!();
|
||||||
|
}
|
||||||
|
fn range_rev<'r>(
|
||||||
|
&self,
|
||||||
|
_tree: usize,
|
||||||
|
_low: Bound<&'r [u8]>,
|
||||||
|
_high: Bound<&'r [u8]>,
|
||||||
|
) -> Result<ValueIter<'a>> {
|
||||||
|
unimplemented!();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -1,9 +1,10 @@
|
||||||
use crate::*;
|
use crate::*;
|
||||||
|
|
||||||
use crate::sled_adapter::SledDb;
|
use crate::sled_adapter::SledDb;
|
||||||
|
use crate::sqlite_adapter::SqliteDb;
|
||||||
|
|
||||||
fn test_suite(db: Db) -> Result<()> {
|
fn test_suite(db: Db) {
|
||||||
let tree = db.open_tree("tree")?;
|
let tree = db.open_tree("tree").unwrap();
|
||||||
|
|
||||||
let ka: &[u8] = &b"test"[..];
|
let ka: &[u8] = &b"test"[..];
|
||||||
let kb: &[u8] = &b"zwello"[..];
|
let kb: &[u8] = &b"zwello"[..];
|
||||||
|
@ -12,66 +13,69 @@ fn test_suite(db: Db) -> Result<()> {
|
||||||
let vb: &[u8] = &b"plip"[..];
|
let vb: &[u8] = &b"plip"[..];
|
||||||
let vc: &[u8] = &b"plup"[..];
|
let vc: &[u8] = &b"plup"[..];
|
||||||
|
|
||||||
tree.insert(ka, va)?;
|
tree.insert(ka, va).unwrap();
|
||||||
assert_eq!(tree.get(ka)?, Some(va.into()));
|
assert_eq!(tree.get(ka).unwrap(), Some(va.into()));
|
||||||
|
|
||||||
let res = db.transaction::<_, (), _>(|tx| {
|
let res = db.transaction::<_, (), _>(|tx| {
|
||||||
assert_eq!(tx.get(&tree, ka)?, Some(va.into()));
|
assert_eq!(tx.get(&tree, ka).unwrap(), Some(va.into()));
|
||||||
|
|
||||||
tx.insert(&tree, ka, vb)?;
|
tx.insert(&tree, ka, vb).unwrap();
|
||||||
|
|
||||||
assert_eq!(tx.get(&tree, ka)?, Some(vb.into()));
|
assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into()));
|
||||||
|
|
||||||
tx.commit(12)
|
tx.commit(12)
|
||||||
});
|
});
|
||||||
assert!(matches!(res, Ok(12)));
|
assert!(matches!(res, Ok(12)));
|
||||||
assert_eq!(tree.get(ka)?, Some(vb.into()));
|
assert_eq!(tree.get(ka).unwrap(), Some(vb.into()));
|
||||||
|
|
||||||
let res = db.transaction::<(), _, _>(|tx| {
|
let res = db.transaction::<(), _, _>(|tx| {
|
||||||
assert_eq!(tx.get(&tree, ka)?, Some(vb.into()));
|
assert_eq!(tx.get(&tree, ka).unwrap(), Some(vb.into()));
|
||||||
|
|
||||||
tx.insert(&tree, ka, vc)?;
|
tx.insert(&tree, ka, vc).unwrap();
|
||||||
|
|
||||||
assert_eq!(tx.get(&tree, ka)?, Some(vc.into()));
|
assert_eq!(tx.get(&tree, ka).unwrap(), Some(vc.into()));
|
||||||
|
|
||||||
tx.abort(42)
|
tx.abort(42)
|
||||||
});
|
});
|
||||||
assert!(matches!(res, Err(TxError::Abort(42))));
|
assert!(matches!(res, Err(TxError::Abort(42))));
|
||||||
assert_eq!(tree.get(ka)?, Some(vb.into()));
|
assert_eq!(tree.get(ka).unwrap(), Some(vb.into()));
|
||||||
|
|
||||||
let mut iter = tree.iter()?;
|
let mut iter = tree.iter().unwrap();
|
||||||
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
|
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
|
||||||
assert!(iter.next().is_none());
|
assert!(iter.next().is_none());
|
||||||
|
|
||||||
tree.insert(kb, vc)?;
|
tree.insert(kb, vc).unwrap();
|
||||||
assert_eq!(tree.get(kb)?, Some(vc.into()));
|
assert_eq!(tree.get(kb).unwrap(), Some(vc.into()));
|
||||||
|
|
||||||
let mut iter = tree.iter()?;
|
let mut iter = tree.iter().unwrap();
|
||||||
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
|
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
|
||||||
assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into()));
|
assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into()));
|
||||||
assert!(iter.next().is_none());
|
assert!(iter.next().is_none());
|
||||||
|
|
||||||
let mut iter = tree.range(kint..)?;
|
let mut iter = tree.range(kint..).unwrap();
|
||||||
assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into()));
|
assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into()));
|
||||||
assert!(iter.next().is_none());
|
assert!(iter.next().is_none());
|
||||||
|
|
||||||
let mut iter = tree.range_rev(..kint)?;
|
let mut iter = tree.range_rev(..kint).unwrap();
|
||||||
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
|
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
|
||||||
assert!(iter.next().is_none());
|
assert!(iter.next().is_none());
|
||||||
|
|
||||||
let mut iter = tree.iter_rev()?;
|
let mut iter = tree.iter_rev().unwrap();
|
||||||
assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into()));
|
assert_eq!(iter.next().unwrap().unwrap(), (kb.into(), vc.into()));
|
||||||
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
|
assert_eq!(iter.next().unwrap().unwrap(), (ka.into(), vb.into()));
|
||||||
assert!(iter.next().is_none());
|
assert!(iter.next().is_none());
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_sled_db() -> Result<()> {
|
fn test_sled_db() {
|
||||||
let path = mktemp::Temp::new_dir().unwrap();
|
let path = mktemp::Temp::new_dir().unwrap();
|
||||||
let db = SledDb::new(sled::open(path.to_path_buf()).unwrap());
|
let db = SledDb::new(sled::open(path.to_path_buf()).unwrap());
|
||||||
test_suite(db)?;
|
test_suite(db);
|
||||||
drop(path);
|
drop(path);
|
||||||
Ok(())
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sqlite_db() {
|
||||||
|
let db = SqliteDb::new(rusqlite::Connection::open_in_memory().unwrap());
|
||||||
|
test_suite(db);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue