From bd2997a4534a4d06bc0cc636a2c0b50fe793057a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 15:53:20 +0200 Subject: [PATCH] LMDB: use Heed wrapper (much more complete) --- Cargo.lock | 161 +++++++++++++++++++++++++++++++---------- src/db/Cargo.toml | 2 +- src/db/lmdb_adapter.rs | 66 ++++++++--------- src/db/test.rs | 4 +- 4 files changed, 157 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b051e72e..38e990fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -312,6 +312,15 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -344,6 +353,12 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +[[package]] +name = "bytemuck" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc" + [[package]] name = "byteorder" version = "1.4.3" @@ -381,6 +396,12 @@ dependencies = [ "jobserver", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -497,7 +518,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -506,8 +527,8 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.8", ] [[package]] @@ -517,20 +538,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" dependencies = [ "autocfg", - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.8", "lazy_static", "memoffset", "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +dependencies = [ + "crossbeam-utils 0.6.6", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", +] + [[package]] name = "crossbeam-utils" version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "lazy_static", ] @@ -614,7 +654,7 @@ version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "num_cpus", ] @@ -644,7 +684,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dirs-sys-next", ] @@ -683,7 +723,7 @@ version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1016,8 +1056,8 @@ version = "0.8.0" dependencies = [ "clap 3.1.18", "err-derive 0.3.1", + "heed", "hexdump", - "lmdb", "log", "mktemp", "rusqlite", @@ -1274,7 +1314,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.10.0+wasi-snapshot-preview1", ] @@ -1353,6 +1393,45 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +[[package]] +name = "heed" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392" +dependencies = [ + "bytemuck", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-rkv-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2" + +[[package]] +name = "heed-types" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd" +dependencies = [ + "bincode", + "bytemuck", + "byteorder", + "heed-traits", + "serde", + "serde_json", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1552,7 +1631,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1824,21 +1903,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] -name = "lmdb" -version = "0.8.0" +name = "lmdb-rkv-sys" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0908efb5d6496aa977d96f91413da2635a902e5e31dbef0bfb88986c248539" -dependencies = [ - "bitflags", - "libc", - "lmdb-sys", -] - -[[package]] -name = "lmdb-sys" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5b392838cfe8858e86fac37cf97a0e8c55cc60ba0a18365cadc33092f128ce9" +checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe" dependencies = [ "cc", "libc", @@ -1860,7 +1928,7 @@ version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -2018,7 +2086,7 @@ dependencies = [ "arc-swap", "async-trait", "bytes 0.6.0", - "cfg-if", + "cfg-if 1.0.0", "err-derive 0.2.4", "futures", "hex", @@ -2111,7 +2179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -2224,6 +2292,16 @@ version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435" +[[package]] +name = "page_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -2251,7 +2329,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", @@ -2265,7 +2343,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", @@ -2453,7 +2531,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fnv", "lazy_static", "memchr", @@ -3017,7 +3095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", "opaque-debug", @@ -3052,7 +3130,7 @@ checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" dependencies = [ "crc32fast", "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-utils 0.8.8", "fs2", "fxhash", "libc", @@ -3186,6 +3264,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synchronoise" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb" +dependencies = [ + "crossbeam-queue", +] + [[package]] name = "synstructure" version = "0.12.6" @@ -3204,7 +3291,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "libc", "redox_syscall", @@ -3503,7 +3590,7 @@ version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -3672,7 +3759,7 @@ version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index b4601ff7..ce9c9373 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -21,7 +21,7 @@ err-derive = "0.3" hexdump = "0.1" log = "0.4" -lmdb = "0.8" +heed = "0.11" rusqlite = "0.27" sled = "0.34" diff --git a/src/db/lmdb_adapter.rs b/src/db/lmdb_adapter.rs index caf21517..ec1a5444 100644 --- a/src/db/lmdb_adapter.rs +++ b/src/db/lmdb_adapter.rs @@ -7,26 +7,24 @@ use std::cell::RefCell; use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use lmdb::{ - Database, DatabaseFlags, Environment, RoTransaction, RwTransaction, Transaction, WriteFlags, -}; +use heed::{Env, RoTxn, RwTxn, UntypedDatabase as Database}; use crate::{ Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter, }; -pub use lmdb; +pub use heed; // -- err -impl From for Error { - fn from(e: lmdb::Error) -> Error { +impl From for Error { + fn from(e: heed::Error) -> Error { Error(format!("LMDB: {}", e).into()) } } -impl From for TxError { - fn from(e: lmdb::Error) -> TxError { +impl From for TxError { + fn from(e: heed::Error) -> TxError { TxError::Db(e.into()) } } @@ -34,12 +32,12 @@ impl From for TxError { // -- db pub struct LmdbDb { - db: lmdb::Environment, - trees: RwLock<(Vec, HashMap)>, + db: heed::Env, + trees: RwLock<(Vec, HashMap)>, } impl LmdbDb { - pub fn init(db: lmdb::Environment) -> Db { + pub fn init(db: Env) -> Db { let s = Self { db, trees: RwLock::new((Vec::new(), HashMap::new())), @@ -47,7 +45,7 @@ impl LmdbDb { Db(Arc::new(s)) } - fn get_tree(&self, i: usize) -> Result { + fn get_tree(&self, i: usize) -> Result { self.trees .read() .unwrap() @@ -64,7 +62,7 @@ impl IDb for LmdbDb { if let Some(i) = trees.1.get(name) { Ok(*i) } else { - let tree = self.db.create_db(Some(name), DatabaseFlags::empty())?; + let tree = self.db.create_database(Some(name))?; let i = trees.0.len(); trees.0.push(tree); trees.1.insert(name.to_string(), i); @@ -82,7 +80,7 @@ impl IDb for LmdbDb { let tree = self.get_tree(tree)?; let res = TxAndValue { - tx: self.db.begin_ro_txn()?, + tx: self.db.read_txn()?, value: NonNull::dangling(), _pin: PhantomPinned, }; @@ -90,9 +88,9 @@ impl IDb for LmdbDb { unsafe { let tx = NonNull::from(&boxed.tx); - let val = match tx.as_ref().get(tree, &key) { - Err(lmdb::Error::NotFound) => return Ok(None), - v => v?, + let val = match tree.get(tx.as_ref(), &key)? { + None => return Ok(None), + Some(v) => v, }; let mut_ref: Pin<&mut TxAndValue<'_>> = Pin::as_mut(&mut boxed); @@ -112,8 +110,8 @@ impl IDb for LmdbDb { fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { let tree = self.get_tree(tree)?; - let mut tx = self.db.begin_rw_txn()?; - tx.put(tree, &key, &value, WriteFlags::empty())?; + let mut tx = self.db.write_txn()?; + tree.put(&mut tx, &key, &value)?; tx.commit()?; Ok(()) } @@ -149,7 +147,7 @@ impl IDb for LmdbDb { let trees = self.trees.read().unwrap(); let mut tx = LmdbTx { trees: &trees.0[..], - tx: self.db.begin_rw_txn()?, + tx: self.db.write_txn()?, }; let res = f.try_on(&mut tx); @@ -159,11 +157,11 @@ impl IDb for LmdbDb { Ok(()) } TxFnResult::Abort => { - tx.tx.abort(); + tx.tx.abort()?; Err(TxError::Abort(())) } TxFnResult::DbErr => { - tx.tx.abort(); + tx.tx.abort()?; Err(TxError::Db(Error( "(this message will be discarded)".into(), ))) @@ -176,7 +174,7 @@ impl IDb for LmdbDb { struct LmdbTx<'a, 'db> { trees: &'db [Database], - tx: RwTransaction<'a>, + tx: RwTxn<'a, 'a>, } impl<'a, 'db> LmdbTx<'a, 'db> { @@ -192,10 +190,9 @@ impl<'a, 'db> LmdbTx<'a, 'db> { impl<'a, 'db> ITx for LmdbTx<'a, 'db> { fn get(&self, tree: usize, key: &[u8]) -> Result>> { let tree = self.get_tree(tree)?; - match self.tx.get::<'a, _>(*tree, &key) { - Err(lmdb::Error::NotFound) => Ok(None), - Err(e) => Err(e.into()), - Ok(v) => Ok(Some(Value(Box::new(v)))), + match tree.get(&self.tx, &key)? { + Some(v) => Ok(Some(Value(Box::new(v)))), + None => Ok(None), } } fn len(&self, _tree: usize) -> Result { @@ -203,17 +200,14 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { } fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { - let tree = self.get_tree(tree)?; - self.tx.put(*tree, &key, &value, WriteFlags::empty())?; + let tree = *self.get_tree(tree)?; + tree.put(&mut self.tx, &key, &value)?; Ok(()) } fn remove(&mut self, tree: usize, key: &[u8]) -> Result { - let tree = self.get_tree(tree)?; - match self.tx.del::<'a, _>(*tree, &key, None) { - Ok(()) => Ok(true), - Err(lmdb::Error::NotFound) => Ok(false), - Err(e) => Err(e.into()), - } + let tree = *self.get_tree(tree)?; + let deleted = tree.delete(&mut self.tx, &key)?; + Ok(deleted) } fn iter(&self, _tree: usize) -> Result> { @@ -244,7 +238,7 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> { // ---- struct TxAndValue<'a> { - tx: RoTransaction<'a>, + tx: RoTxn<'a>, value: NonNull<&'a [u8]>, _pin: PhantomPinned, } diff --git a/src/db/test.rs b/src/db/test.rs index e5b83ab5..31fea1f0 100644 --- a/src/db/test.rs +++ b/src/db/test.rs @@ -82,8 +82,8 @@ fn test_suite(db: Db) { #[test] fn test_lmdb_db() { let path = mktemp::Temp::new_dir().unwrap(); - let db = lmdb::Environment::new() - .set_max_dbs(100) + let db = heed::EnvOpenOptions::new() + .max_dbs(100) .open(&path) .unwrap(); let db = LmdbDb::init(db);