Merge pull request 'convert_db: allow LMDB map size override' (#691) from zdenek.crha/garage:convert_db_lmdb_map_size into main

Reviewed-on: Deuxfleurs/garage#691

src/db/lib.rs

@@ -171,6 +171,48 @@ impl Db {
 	}
 }
 
+/// List of supported database engine types
+///
+/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
+/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
+/// and valid engine, whose support is not enabled via feature flag.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Engine {
+	Lmdb,
+	Sqlite,
+	Sled,
+}
+
+impl Engine {
+	/// Return variant name as static `&str`
+	pub fn as_str(&self) -> &'static str {
+		match self {
+			Self::Lmdb => "lmdb",
+			Self::Sqlite => "sqlite",
+			Self::Sled => "sled",
+		}
+	}
+}
+
+impl std::fmt::Display for Engine {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		self.as_str().fmt(fmt)
+	}
+}
+
+impl std::str::FromStr for Engine {
+	type Err = Error;
+
+	fn from_str(text: &str) -> Result<Engine> {
+		match text {
+			"lmdb" | "heed" => Ok(Self::Lmdb),
+			"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
+			"sled" => Ok(Self::Sled),
+			kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
+		}
+	}
+}
+
 #[allow(clippy::len_without_is_empty)]
 impl Tree {
 	#[inline]

src/garage/cli/convert_db.rs

@@ -14,44 +14,73 @@ pub struct ConvertDbOpt {
 	/// Input database engine (sled, lmdb or sqlite; limited by db engines
 	/// enabled in this build)
 	#[structopt(short = "a")]
-	input_engine: String,
+	input_engine: Engine,
 
 	/// Output database path
 	#[structopt(short = "o")]
 	output_path: PathBuf,
 	/// Output database engine
 	#[structopt(short = "b")]
-	output_engine: String,
+	output_engine: Engine,
+
+	#[structopt(flatten)]
+	db_open: OpenDbOpt,
+}
+
+/// Overrides for database open operation
+#[derive(StructOpt, Debug, Default)]
+pub struct OpenDbOpt {
+	#[cfg(feature = "lmdb")]
+	#[structopt(flatten)]
+	lmdb: OpenLmdbOpt,
+}
+
+/// Overrides for LMDB database open operation
+#[cfg(feature = "lmdb")]
+#[derive(StructOpt, Debug, Default)]
+pub struct OpenLmdbOpt {
+	/// LMDB map size override
+	/// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB)
+	#[cfg(feature = "lmdb")]
+	#[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)]
+	map_size: Option<bytesize::ByteSize>,
 }
 
 pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
-	let input = open_db(args.input_path, args.input_engine)?;
+	if args.input_engine == args.output_engine {
-	let output = open_db(args.output_path, args.output_engine)?;
+		return Err(Error("input and output database engine must differ".into()));
+	}
+
+	let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
+	let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
 	output.import(&input)?;
 	Ok(())
 }
 
-fn open_db(path: PathBuf, engine: String) -> Result<Db> {
+fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
-	match engine.as_str() {
+	match engine {
 		#[cfg(feature = "sled")]
-		"sled" => {
+		Engine::Sled => {
 			let db = sled_adapter::sled::Config::default().path(&path).open()?;
 			Ok(sled_adapter::SledDb::init(db))
 		}
 		#[cfg(feature = "sqlite")]
-		"sqlite" | "sqlite3" | "rusqlite" => {
+		Engine::Sqlite => {
 			let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
 			db.pragma_update(None, "journal_mode", &"WAL")?;
 			db.pragma_update(None, "synchronous", &"NORMAL")?;
 			Ok(sqlite_adapter::SqliteDb::init(db))
 		}
 		#[cfg(feature = "lmdb")]
-		"lmdb" | "heed" => {
+		Engine::Lmdb => {
 			std::fs::create_dir_all(&path).map_err(|e| {
 				Error(format!("Unable to create LMDB data directory: {}", e).into())
 			})?;
 
-			let map_size = lmdb_adapter::recommended_map_size();
+			let map_size = match open.lmdb.map_size {
+				Some(c) => c.as_u64() as usize,
+				None => lmdb_adapter::recommended_map_size(),
+			};
 
 			let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
 			env_builder.max_dbs(100);
@@ -62,8 +91,13 @@ fn open_db(path: PathBuf, engine: String) -> Result<Db> {
 			let db = env_builder.open(&path)?;
 			Ok(lmdb_adapter::LmdbDb::init(db))
 		}
-		e => Err(Error(
+
-			format!("Invalid or unsupported DB engine: {}", e).into(),
+		// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
+		// attribute is added so that we won't have to change this match in case stop building
+		// support for one or more engines by default.
+		#[allow(unreachable_patterns)]
+		engine => Err(Error(
+			format!("Engine support not available in this build: {}", engine).into(),
 		)),
 	}
 }