Merge pull request 'convert_db: allow LMDB map size override' (#691) from zdenek.crha/garage:convert_db_lmdb_map_size into main

Reviewed-on: Deuxfleurs/garage#691
This commit is contained in:
Alex 2024-01-24 08:19:45 +00:00
commit 08a871390e
2 changed files with 88 additions and 12 deletions

View file

@ -171,6 +171,48 @@ impl Db {
} }
} }
/// List of supported database engine types
///
/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
/// and valid engine, whose support is not enabled via feature flag.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Engine {
Lmdb,
Sqlite,
Sled,
}
impl Engine {
/// Return variant name as static `&str`
pub fn as_str(&self) -> &'static str {
match self {
Self::Lmdb => "lmdb",
Self::Sqlite => "sqlite",
Self::Sled => "sled",
}
}
}
impl std::fmt::Display for Engine {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
self.as_str().fmt(fmt)
}
}
impl std::str::FromStr for Engine {
type Err = Error;
fn from_str(text: &str) -> Result<Engine> {
match text {
"lmdb" | "heed" => Ok(Self::Lmdb),
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
"sled" => Ok(Self::Sled),
kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
}
}
}
#[allow(clippy::len_without_is_empty)] #[allow(clippy::len_without_is_empty)]
impl Tree { impl Tree {
#[inline] #[inline]

View file

@ -14,44 +14,73 @@ pub struct ConvertDbOpt {
/// Input database engine (sled, lmdb or sqlite; limited by db engines /// Input database engine (sled, lmdb or sqlite; limited by db engines
/// enabled in this build) /// enabled in this build)
#[structopt(short = "a")] #[structopt(short = "a")]
input_engine: String, input_engine: Engine,
/// Output database path /// Output database path
#[structopt(short = "o")] #[structopt(short = "o")]
output_path: PathBuf, output_path: PathBuf,
/// Output database engine /// Output database engine
#[structopt(short = "b")] #[structopt(short = "b")]
output_engine: String, output_engine: Engine,
#[structopt(flatten)]
db_open: OpenDbOpt,
}
/// Overrides for database open operation
#[derive(StructOpt, Debug, Default)]
pub struct OpenDbOpt {
#[cfg(feature = "lmdb")]
#[structopt(flatten)]
lmdb: OpenLmdbOpt,
}
/// Overrides for LMDB database open operation
#[cfg(feature = "lmdb")]
#[derive(StructOpt, Debug, Default)]
pub struct OpenLmdbOpt {
/// LMDB map size override
/// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB)
#[cfg(feature = "lmdb")]
#[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)]
map_size: Option<bytesize::ByteSize>,
} }
pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
let input = open_db(args.input_path, args.input_engine)?; if args.input_engine == args.output_engine {
let output = open_db(args.output_path, args.output_engine)?; return Err(Error("input and output database engine must differ".into()));
}
let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
output.import(&input)?; output.import(&input)?;
Ok(()) Ok(())
} }
fn open_db(path: PathBuf, engine: String) -> Result<Db> { fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
match engine.as_str() { match engine {
#[cfg(feature = "sled")] #[cfg(feature = "sled")]
"sled" => { Engine::Sled => {
let db = sled_adapter::sled::Config::default().path(&path).open()?; let db = sled_adapter::sled::Config::default().path(&path).open()?;
Ok(sled_adapter::SledDb::init(db)) Ok(sled_adapter::SledDb::init(db))
} }
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
"sqlite" | "sqlite3" | "rusqlite" => { Engine::Sqlite => {
let db = sqlite_adapter::rusqlite::Connection::open(&path)?; let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
db.pragma_update(None, "journal_mode", &"WAL")?; db.pragma_update(None, "journal_mode", &"WAL")?;
db.pragma_update(None, "synchronous", &"NORMAL")?; db.pragma_update(None, "synchronous", &"NORMAL")?;
Ok(sqlite_adapter::SqliteDb::init(db)) Ok(sqlite_adapter::SqliteDb::init(db))
} }
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
"lmdb" | "heed" => { Engine::Lmdb => {
std::fs::create_dir_all(&path).map_err(|e| { std::fs::create_dir_all(&path).map_err(|e| {
Error(format!("Unable to create LMDB data directory: {}", e).into()) Error(format!("Unable to create LMDB data directory: {}", e).into())
})?; })?;
let map_size = lmdb_adapter::recommended_map_size(); let map_size = match open.lmdb.map_size {
Some(c) => c.as_u64() as usize,
None => lmdb_adapter::recommended_map_size(),
};
let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new(); let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
env_builder.max_dbs(100); env_builder.max_dbs(100);
@ -62,8 +91,13 @@ fn open_db(path: PathBuf, engine: String) -> Result<Db> {
let db = env_builder.open(&path)?; let db = env_builder.open(&path)?;
Ok(lmdb_adapter::LmdbDb::init(db)) Ok(lmdb_adapter::LmdbDb::init(db))
} }
e => Err(Error(
format!("Invalid or unsupported DB engine: {}", e).into(), // Pattern is unreachable when all supported DB engines are compiled into binary. The allow
// attribute is added so that we won't have to change this match in case stop building
// support for one or more engines by default.
#[allow(unreachable_patterns)]
engine => Err(Error(
format!("Engine support not available in this build: {}", engine).into(),
)), )),
} }
} }