LMDB: use Heed wrapper (much more complete)

Cargo.lock

@@ -312,6 +312,15 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -344,6 +353,12 @@ version = "3.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
 
+[[package]]
+name = "bytemuck"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc"
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -381,6 +396,12 @@ dependencies = [
  "jobserver",
 ]
 
+[[package]]
+name = "cfg-if"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -497,7 +518,7 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -506,8 +527,8 @@ version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.8",
 ]
 
 [[package]]
@@ -517,20 +538,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
 dependencies = [
  "autocfg",
- "cfg-if",
+ "cfg-if 1.0.0",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.8",
  "lazy_static",
  "memoffset",
  "scopeguard",
 ]
 
+[[package]]
+name = "crossbeam-queue"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
+dependencies = [
+ "crossbeam-utils 0.6.6",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
+dependencies = [
+ "cfg-if 0.1.10",
+ "lazy_static",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "lazy_static",
 ]
 
@@ -614,7 +654,7 @@ version = "4.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "num_cpus",
 ]
 
@@ -644,7 +684,7 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "dirs-sys-next",
 ]
 
@@ -683,7 +723,7 @@ version = "0.8.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -1016,8 +1056,8 @@ version = "0.8.0"
 dependencies = [
  "clap 3.1.18",
  "err-derive 0.3.1",
+ "heed",
  "hexdump",
- "lmdb",
  "log",
  "mktemp",
  "rusqlite",
@@ -1274,7 +1314,7 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "libc",
  "wasi 0.10.0+wasi-snapshot-preview1",
 ]
@@ -1353,6 +1393,45 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
 
+[[package]]
+name = "heed"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392"
+dependencies = [
+ "bytemuck",
+ "byteorder",
+ "heed-traits",
+ "heed-types",
+ "libc",
+ "lmdb-rkv-sys",
+ "once_cell",
+ "page_size",
+ "serde",
+ "synchronoise",
+ "url",
+]
+
+[[package]]
+name = "heed-traits"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2"
+
+[[package]]
+name = "heed-types"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd"
+dependencies = [
+ "bincode",
+ "bytemuck",
+ "byteorder",
+ "heed-traits",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -1552,7 +1631,7 @@ version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -1824,21 +1903,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
 
 [[package]]
-name = "lmdb"
+name = "lmdb-rkv-sys"
-version = "0.8.0"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b0908efb5d6496aa977d96f91413da2635a902e5e31dbef0bfb88986c248539"
+checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe"
-dependencies = [
- "bitflags",
- "libc",
- "lmdb-sys",
-]
-
-[[package]]
-name = "lmdb-sys"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5b392838cfe8858e86fac37cf97a0e8c55cc60ba0a18365cadc33092f128ce9"
 dependencies = [
  "cc",
  "libc",
@@ -1860,7 +1928,7 @@ version = "0.4.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -2018,7 +2086,7 @@ dependencies = [
  "arc-swap",
  "async-trait",
  "bytes 0.6.0",
- "cfg-if",
+ "cfg-if 1.0.0",
  "err-derive 0.2.4",
  "futures",
  "hex",
@@ -2111,7 +2179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95"
 dependencies = [
  "bitflags",
- "cfg-if",
+ "cfg-if 1.0.0",
  "foreign-types",
  "libc",
  "once_cell",
@@ -2224,6 +2292,16 @@ version = "6.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"
 
+[[package]]
+name = "page_size"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -2251,7 +2329,7 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "instant",
  "libc",
  "redox_syscall",
@@ -2265,7 +2343,7 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "libc",
  "redox_syscall",
  "smallvec",
@@ -2453,7 +2531,7 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "fnv",
  "lazy_static",
  "memchr",
@@ -3017,7 +3095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
 dependencies = [
  "block-buffer",
- "cfg-if",
+ "cfg-if 1.0.0",
  "cpufeatures",
  "digest",
  "opaque-debug",
@@ -3052,7 +3130,7 @@ checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
 dependencies = [
  "crc32fast",
  "crossbeam-epoch",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.8",
  "fs2",
  "fxhash",
  "libc",
@@ -3186,6 +3264,15 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "synchronoise"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb"
+dependencies = [
+ "crossbeam-queue",
+]
+
 [[package]]
 name = "synstructure"
 version = "0.12.6"
@@ -3204,7 +3291,7 @@ version = "3.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "fastrand",
  "libc",
  "redox_syscall",
@@ -3503,7 +3590,7 @@ version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "log",
  "pin-project-lite",
  "tracing-attributes",
@@ -3672,7 +3759,7 @@ version = "0.2.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "wasm-bindgen-macro",
 ]
 

src/db/Cargo.toml

@@ -21,7 +21,7 @@ err-derive = "0.3"
 hexdump = "0.1"
 log = "0.4"
 
-lmdb = "0.8"
+heed = "0.11"
 rusqlite = "0.27"
 sled = "0.34"
 

src/db/lmdb_adapter.rs

@@ -7,26 +7,24 @@ use std::cell::RefCell;
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 
-use lmdb::{
+use heed::{Env, RoTxn, RwTxn, UntypedDatabase as Database};
-	Database, DatabaseFlags, Environment, RoTransaction, RwTransaction, Transaction, WriteFlags,
-};
 
 use crate::{
 	Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
 };
 
-pub use lmdb;
+pub use heed;
 
 // -- err
 
-impl From<lmdb::Error> for Error {
+impl From<heed::Error> for Error {
-	fn from(e: lmdb::Error) -> Error {
+	fn from(e: heed::Error) -> Error {
 		Error(format!("LMDB: {}", e).into())
 	}
 }
 
-impl<T> From<lmdb::Error> for TxError<T> {
+impl<T> From<heed::Error> for TxError<T> {
-	fn from(e: lmdb::Error) -> TxError<T> {
+	fn from(e: heed::Error) -> TxError<T> {
 		TxError::Db(e.into())
 	}
 }
@@ -34,12 +32,12 @@ impl<T> From<lmdb::Error> for TxError<T> {
 // -- db
 
 pub struct LmdbDb {
-	db: lmdb::Environment,
+	db: heed::Env,
-	trees: RwLock<(Vec<lmdb::Database>, HashMap<String, usize>)>,
+	trees: RwLock<(Vec<Database>, HashMap<String, usize>)>,
 }
 
 impl LmdbDb {
-	pub fn init(db: lmdb::Environment) -> Db {
+	pub fn init(db: Env) -> Db {
 		let s = Self {
 			db,
 			trees: RwLock::new((Vec::new(), HashMap::new())),
@@ -47,7 +45,7 @@ impl LmdbDb {
 		Db(Arc::new(s))
 	}
 
-	fn get_tree(&self, i: usize) -> Result<lmdb::Database> {
+	fn get_tree(&self, i: usize) -> Result<Database> {
 		self.trees
 			.read()
 			.unwrap()
@@ -64,7 +62,7 @@ impl IDb for LmdbDb {
 		if let Some(i) = trees.1.get(name) {
 			Ok(*i)
 		} else {
-			let tree = self.db.create_db(Some(name), DatabaseFlags::empty())?;
+			let tree = self.db.create_database(Some(name))?;
 			let i = trees.0.len();
 			trees.0.push(tree);
 			trees.1.insert(name.to_string(), i);
@@ -82,7 +80,7 @@ impl IDb for LmdbDb {
 		let tree = self.get_tree(tree)?;
 
 		let res = TxAndValue {
-			tx: self.db.begin_ro_txn()?,
+			tx: self.db.read_txn()?,
 			value: NonNull::dangling(),
 			_pin: PhantomPinned,
 		};
@@ -90,9 +88,9 @@ impl IDb for LmdbDb {
 
 		unsafe {
 			let tx = NonNull::from(&boxed.tx);
-			let val = match tx.as_ref().get(tree, &key) {
+			let val = match tree.get(tx.as_ref(), &key)? {
-				Err(lmdb::Error::NotFound) => return Ok(None),
+				None => return Ok(None),
-				v => v?,
+				Some(v) => v,
 			};
 
 			let mut_ref: Pin<&mut TxAndValue<'_>> = Pin::as_mut(&mut boxed);
@@ -112,8 +110,8 @@ impl IDb for LmdbDb {
 
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
 		let tree = self.get_tree(tree)?;
-		let mut tx = self.db.begin_rw_txn()?;
+		let mut tx = self.db.write_txn()?;
-		tx.put(tree, &key, &value, WriteFlags::empty())?;
+		tree.put(&mut tx, &key, &value)?;
 		tx.commit()?;
 		Ok(())
 	}
@@ -149,7 +147,7 @@ impl IDb for LmdbDb {
 		let trees = self.trees.read().unwrap();
 		let mut tx = LmdbTx {
 			trees: &trees.0[..],
-			tx: self.db.begin_rw_txn()?,
+			tx: self.db.write_txn()?,
 		};
 
 		let res = f.try_on(&mut tx);
@@ -159,11 +157,11 @@ impl IDb for LmdbDb {
 				Ok(())
 			}
 			TxFnResult::Abort => {
-				tx.tx.abort();
+				tx.tx.abort()?;
 				Err(TxError::Abort(()))
 			}
 			TxFnResult::DbErr => {
-				tx.tx.abort();
+				tx.tx.abort()?;
 				Err(TxError::Db(Error(
 					"(this message will be discarded)".into(),
 				)))
@@ -176,7 +174,7 @@ impl IDb for LmdbDb {
 
 struct LmdbTx<'a, 'db> {
 	trees: &'db [Database],
-	tx: RwTransaction<'a>,
+	tx: RwTxn<'a, 'a>,
 }
 
 impl<'a, 'db> LmdbTx<'a, 'db> {
@@ -192,10 +190,9 @@ impl<'a, 'db> LmdbTx<'a, 'db> {
 impl<'a, 'db> ITx for LmdbTx<'a, 'db> {
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> {
 		let tree = self.get_tree(tree)?;
-		match self.tx.get::<'a, _>(*tree, &key) {
+		match tree.get(&self.tx, &key)? {
-			Err(lmdb::Error::NotFound) => Ok(None),
+			Some(v) => Ok(Some(Value(Box::new(v)))),
-			Err(e) => Err(e.into()),
+			None => Ok(None),
-			Ok(v) => Ok(Some(Value(Box::new(v)))),
 		}
 	}
 	fn len(&self, _tree: usize) -> Result<usize> {
@@ -203,17 +200,14 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> {
 	}
 
 	fn insert(&mut self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
-		let tree = self.get_tree(tree)?;
+		let tree = *self.get_tree(tree)?;
-		self.tx.put(*tree, &key, &value, WriteFlags::empty())?;
+		tree.put(&mut self.tx, &key, &value)?;
 		Ok(())
 	}
 	fn remove(&mut self, tree: usize, key: &[u8]) -> Result<bool> {
-		let tree = self.get_tree(tree)?;
+		let tree = *self.get_tree(tree)?;
-		match self.tx.del::<'a, _>(*tree, &key, None) {
+		let deleted = tree.delete(&mut self.tx, &key)?;
-			Ok(()) => Ok(true),
+		Ok(deleted)
-			Err(lmdb::Error::NotFound) => Ok(false),
-			Err(e) => Err(e.into()),
-		}
 	}
 
 	fn iter(&self, _tree: usize) -> Result<ValueIter<'_>> {
@@ -244,7 +238,7 @@ impl<'a, 'db> ITx for LmdbTx<'a, 'db> {
 // ----
 
 struct TxAndValue<'a> {
-	tx: RoTransaction<'a>,
+	tx: RoTxn<'a>,
 	value: NonNull<&'a [u8]>,
 	_pin: PhantomPinned,
 }

src/db/test.rs

@@ -82,8 +82,8 @@ fn test_suite(db: Db) {
 #[test]
 fn test_lmdb_db() {
 	let path = mktemp::Temp::new_dir().unwrap();
-	let db = lmdb::Environment::new()
+	let db = heed::EnvOpenOptions::new()
-		.set_max_dbs(100)
+		.max_dbs(100)
 		.open(&path)
 		.unwrap();
 	let db = LmdbDb::init(db);