Abstract database behind generic interface and implement alternative drivers #322

Merged
lx merged 64 commits from db-abstraction into main 2022-06-08 08:01:56 +00:00
4 changed files with 157 additions and 76 deletions
Showing only changes of commit bd2997a453 - Show all commits

161
Cargo.lock generated
View file

@ -312,6 +312,15 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@ -344,6 +353,12 @@ version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
[[package]]
name = "bytemuck"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc"
[[package]]
name = "byteorder"
version = "1.4.3"
@ -381,6 +396,12 @@ dependencies = [
"jobserver",
]
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -497,7 +518,7 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -506,8 +527,8 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
dependencies = [
"cfg-if",
"crossbeam-utils",
"cfg-if 1.0.0",
"crossbeam-utils 0.8.8",
]
[[package]]
@ -517,20 +538,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"cfg-if 1.0.0",
"crossbeam-utils 0.8.8",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-queue"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
dependencies = [
"crossbeam-utils 0.6.6",
]
[[package]]
name = "crossbeam-utils"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
dependencies = [
"cfg-if 0.1.10",
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"lazy_static",
]
@ -614,7 +654,7 @@ version = "4.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"num_cpus",
]
@ -644,7 +684,7 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"dirs-sys-next",
]
@ -683,7 +723,7 @@ version = "0.8.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -1016,8 +1056,8 @@ version = "0.8.0"
dependencies = [
"clap 3.1.18",
"err-derive 0.3.1",
"heed",
"hexdump",
"lmdb",
"log",
"mktemp",
"rusqlite",
@ -1274,7 +1314,7 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
]
@ -1353,6 +1393,45 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
[[package]]
name = "heed"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392"
dependencies = [
"bytemuck",
"byteorder",
"heed-traits",
"heed-types",
"libc",
"lmdb-rkv-sys",
"once_cell",
"page_size",
"serde",
"synchronoise",
"url",
]
[[package]]
name = "heed-traits"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2"
[[package]]
name = "heed-types"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd"
dependencies = [
"bincode",
"bytemuck",
"byteorder",
"heed-traits",
"serde",
"serde_json",
]
[[package]]
name = "hermit-abi"
version = "0.1.19"
@ -1552,7 +1631,7 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -1824,21 +1903,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
[[package]]
name = "lmdb"
version = "0.8.0"
name = "lmdb-rkv-sys"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b0908efb5d6496aa977d96f91413da2635a902e5e31dbef0bfb88986c248539"
dependencies = [
"bitflags",
"libc",
"lmdb-sys",
]
[[package]]
name = "lmdb-sys"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5b392838cfe8858e86fac37cf97a0e8c55cc60ba0a18365cadc33092f128ce9"
checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe"
dependencies = [
"cc",
"libc",
@ -1860,7 +1928,7 @@ version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -2018,7 +2086,7 @@ dependencies = [
"arc-swap",
"async-trait",
"bytes 0.6.0",
"cfg-if",
"cfg-if 1.0.0",
"err-derive 0.2.4",
"futures",
"hex",
@ -2111,7 +2179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95"
dependencies = [
"bitflags",
"cfg-if",
"cfg-if 1.0.0",
"foreign-types",
"libc",
"once_cell",
@ -2224,6 +2292,16 @@ version = "6.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"
[[package]]
name = "page_size"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
@ -2251,7 +2329,7 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"instant",
"libc",
"redox_syscall",
@ -2265,7 +2343,7 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"libc",
"redox_syscall",
"smallvec",
@ -2453,7 +2531,7 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"fnv",
"lazy_static",
"memchr",
@ -3017,7 +3095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
dependencies = [
"block-buffer",
"cfg-if",
"cfg-if 1.0.0",
"cpufeatures",
"digest",
"opaque-debug",
@ -3052,7 +3130,7 @@ checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
dependencies = [
"crc32fast",
"crossbeam-epoch",
"crossbeam-utils",
"crossbeam-utils 0.8.8",
"fs2",
"fxhash",
"libc",
@ -3186,6 +3264,15 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "synchronoise"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb"
dependencies = [
"crossbeam-queue",
]
[[package]]
name = "synstructure"
version = "0.12.6"
@ -3204,7 +3291,7 @@ version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"fastrand",
"libc",
"redox_syscall",
@ -3503,7 +3590,7 @@ version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"log",
"pin-project-lite",
"tracing-attributes",
@ -3672,7 +3759,7 @@ version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"wasm-bindgen-macro",
]

View file

@ -21,7 +21,7 @@ err-derive = "0.3"
hexdump = "0.1"
log = "0.4"
lmdb = "0.8"
heed = "0.11"
rusqlite = "0.27"
sled = "0.34"

View file

@ -7,26 +7,24 @@ use std::cell::RefCell;
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use lmdb::{
Database, DatabaseFlags, Environment, RoTransaction, RwTransaction, Transaction, WriteFlags,
};
use heed::{Env, RoTxn, RwTxn, UntypedDatabase as Database};
use crate::{
Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
};
pub use lmdb;
pub use heed;
// -- err
impl From<lmdb::Error> for Error {
fn from(e: lmdb::Error) -> Error {
impl From<heed::Error> for Error {
fn from(e: heed::Error) -> Error {
Error(format!("LMDB: {}", e).into())
}
}
impl<T> From<lmdb::Error> for TxError<T> {
fn from(e: lmdb::Error) -> TxError<T> {
impl<T> From<heed::Error> for TxError<T> {
fn from(e: heed::Error) -> TxError<T> {
TxError::Db(e.into())
}
}
@ -34,12 +32,12 @@ impl<T> From<lmdb::Error> for TxError<T> {
// -- db
pub struct LmdbDb {
db: lmdb::Environment,
trees: RwLock<(Vec<lmdb::Database>, HashMap<String, usize>)>,
db: heed::Env,
trees: RwLock<(Vec<Database>, HashMap<String, usize>)>,
}
impl LmdbDb {
pub fn init(db: lmdb::Environment) -> Db {
pub fn init(db: Env) -> Db {
let s = Self {
db,
trees: RwLock::new((Vec::new(), HashMap::new())),
@ -47,7 +45,7 @@ impl LmdbDb {
Db(Arc::new(s))
}
fn get_tree(&self, i: usize) -> Result<lmdb::Database> {
fn get_tree(&self, i: usize) -> Result<Database> {
self.trees
.read()
.unwrap()
@ -64,7 +62,7 @@ impl IDb for LmdbDb {
if let Some(i) = trees.1.get(name) {
Ok(*i)
} else {
let tree = self.db.create_db(Some(name), DatabaseFlags::empty())?;
let tree = self.db.create_database(Some(name))?;
let i = trees.0.len();
trees.0.push(tree);
trees.1.insert(name.to_string(), i);
@ -82,7 +80,7 @@ impl IDb for LmdbDb {
let tree = self.get_tree(tree)?;
let res = TxAndValue {
tx: self.db.begin_ro_txn()?,
tx: self.db.read_txn()?,
value: NonNull::dangling(),
_pin: PhantomPinned,
};
@ -90,9 +88,9 @@ impl IDb for LmdbDb {
unsafe {
let tx = NonNull::from(&boxed.tx);
let val = match tx.as_ref().get(tree, &key) {
Err(lmdb::Error::NotFound) => return Ok(None),
v => v?,
let val = match tree.get(tx.as_ref(), &key)? {
None => return Ok(None),
Some(v) => v,
};
let mut_ref: Pin<&mut TxAndValue<'_>> = Pin::as_mut(&mut boxed);
@ -112,8 +110,8 @@ impl IDb for LmdbDb {
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?;
let mut tx = self.db.begin_rw_txn()?;
tx.put(tree, &key, &value, WriteFlags::empty())?;
let mut tx = self.db.write_txn()?;
tree.put(&mut tx, &key, &value)?;
tx.commit()?;
Ok(())
}
@ -149,7 +147,7 @@ impl IDb for LmdbDb {
let trees = self.trees.read().unwrap();
let mut tx = LmdbTx {
trees: &trees.0[..],
tx: self.db.begin_rw_txn()?,
tx: self.db.write_txn()?,
};
let res = f.try_on(&mut tx);
@ -159,11 +157,11 @@ impl IDb for LmdbDb {
Ok(())
}
TxFnResult::Abort => {
tx.tx.abort();
tx.tx.abort()?;
Err(TxError::Abort(()))
}
TxFnResult::DbErr => {
tx.tx.abort();
tx.tx.abort()?;
Err(TxError::Db(Error(
"(this message will be discarded)".into(),
)))
@ -176,7 +174,7 @@ impl IDb for LmdbDb {
struct LmdbTx<'a, 'db> {
trees: &'db [Database],
tx: RwTransaction<'a>,
tx: RwTxn<'a, 'a>,
}
impl<'a, 'db> LmdbTx<'a, 'db> {
@ -192,10 +190,9 @@ impl<'a, 'db> LmdbTx<'a, 'db> {
impl<'a, 'db> ITx for LmdbTx<'a, 'db> {
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> {
let tree = self.get_tree(tree)?;
match self.tx.get::<'a, _>(*tree, &key) {
Err(lmdb::Error::NotFound) => Ok(None),
Err(e) => Err(e.into()),
Ok(v) => Ok(Some(Value(Box::new(v)))),
match tree.get(&self.tx, &key)? {
Some(v) => Ok(Some(Value(Box::new(v)))),
None => Ok(None),
}
}
fn len(&self, _tree: usize) -> Result<usize> {
@ -203,17 +200,14 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> {
}
fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?;
self.tx.put(*tree, &key, &value, WriteFlags::empty())?;
let tree = *self.get_tree(tree)?;
tree.put(&mut self.tx, &key, &value)?;
Ok(())
}
fn remove(&mut self, tree: usize, key: &[u8]) -> Result<bool> {
let tree = self.get_tree(tree)?;
match self.tx.del::<'a, _>(*tree, &key, None) {
Ok(()) => Ok(true),
Err(lmdb::Error::NotFound) => Ok(false),
Err(e) => Err(e.into()),
}
let tree = *self.get_tree(tree)?;
let deleted = tree.delete(&mut self.tx, &key)?;
Ok(deleted)
}
fn iter(&self, _tree: usize) -> Result<ValueIter<'_>> {
@ -244,7 +238,7 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> {
// ----
struct TxAndValue<'a> {
tx: RoTransaction<'a>,
tx: RoTxn<'a>,
value: NonNull<&'a [u8]>,
_pin: PhantomPinned,
}

View file

@ -82,8 +82,8 @@ fn test_suite(db: Db) {
#[test]
fn test_lmdb_db() {
let path = mktemp::Temp::new_dir().unwrap();
let db = lmdb::Environment::new()
.set_max_dbs(100)
let db = heed::EnvOpenOptions::new()
.max_dbs(100)
.open(&path)
.unwrap();
let db = LmdbDb::init(db);