convert_db: allow LMDB map size override #691

Merged
lx merged 4 commits from zdenek.crha/garage:convert_db_lmdb_map_size into main 2024-01-24 08:19:45 +00:00
2 changed files with 88 additions and 12 deletions

View file

@ -171,6 +171,48 @@ impl Db {
} }
} }
/// List of supported database engine types
///
/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
/// and valid engine, whose support is not enabled via feature flag.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Engine {
Lmdb,
Sqlite,
Sled,
}
impl Engine {
/// Return variant name as static `&str`
pub fn as_str(&self) -> &'static str {
match self {
Self::Lmdb => "lmdb",
Self::Sqlite => "sqlite",
Self::Sled => "sled",
}
}
}
impl std::fmt::Display for Engine {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
self.as_str().fmt(fmt)
}
}
impl std::str::FromStr for Engine {
type Err = Error;
fn from_str(text: &str) -> Result<Engine> {
match text {
"lmdb" | "heed" => Ok(Self::Lmdb),
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
"sled" => Ok(Self::Sled),
kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
}
}
}
#[allow(clippy::len_without_is_empty)] #[allow(clippy::len_without_is_empty)]
impl Tree { impl Tree {
#[inline] #[inline]

View file

@ -14,44 +14,73 @@ pub struct ConvertDbOpt {
/// Input database engine (sled, lmdb or sqlite; limited by db engines /// Input database engine (sled, lmdb or sqlite; limited by db engines
/// enabled in this build) /// enabled in this build)
#[structopt(short = "a")] #[structopt(short = "a")]
input_engine: String, input_engine: Engine,
/// Output database path /// Output database path
#[structopt(short = "o")] #[structopt(short = "o")]
output_path: PathBuf, output_path: PathBuf,
/// Output database engine /// Output database engine
#[structopt(short = "b")] #[structopt(short = "b")]
output_engine: String, output_engine: Engine,
#[structopt(flatten)]
db_open: OpenDbOpt,
zdenek.crha marked this conversation as resolved Outdated
Outdated
Review

This is no longer specific to the output db, so maybe rename it db_open ?

This is no longer specific to the output db, so maybe rename it `db_open` ?
}
/// Overrides for database open operation
#[derive(StructOpt, Debug, Default)]
pub struct OpenDbOpt {
#[cfg(feature = "lmdb")]
#[structopt(flatten)]
lmdb: OpenLmdbOpt,
}
/// Overrides for LMDB database open operation
#[cfg(feature = "lmdb")]
#[derive(StructOpt, Debug, Default)]
pub struct OpenLmdbOpt {
/// LMDB map size override
zdenek.crha marked this conversation as resolved Outdated
Outdated
Review

Similarly, comment should be updated to say that it applies to either input or output db

Similarly, comment should be updated to say that it applies to either input or output db
/// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB)
#[cfg(feature = "lmdb")]
#[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)]
map_size: Option<bytesize::ByteSize>,
} }
pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
let input = open_db(args.input_path, args.input_engine)?; if args.input_engine == args.output_engine {
let output = open_db(args.output_path, args.output_engine)?; return Err(Error("input and output database engine must differ".into()));
}
let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
output.import(&input)?; output.import(&input)?;
Ok(()) Ok(())
} }
fn open_db(path: PathBuf, engine: String) -> Result<Db> { fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
match engine.as_str() { match engine {
#[cfg(feature = "sled")] #[cfg(feature = "sled")]
"sled" => { Engine::Sled => {
let db = sled_adapter::sled::Config::default().path(&path).open()?; let db = sled_adapter::sled::Config::default().path(&path).open()?;
Ok(sled_adapter::SledDb::init(db)) Ok(sled_adapter::SledDb::init(db))
} }
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
"sqlite" | "sqlite3" | "rusqlite" => { Engine::Sqlite => {
let db = sqlite_adapter::rusqlite::Connection::open(&path)?; let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
db.pragma_update(None, "journal_mode", &"WAL")?; db.pragma_update(None, "journal_mode", &"WAL")?;
db.pragma_update(None, "synchronous", &"NORMAL")?; db.pragma_update(None, "synchronous", &"NORMAL")?;
Ok(sqlite_adapter::SqliteDb::init(db)) Ok(sqlite_adapter::SqliteDb::init(db))
} }
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
"lmdb" | "heed" => { Engine::Lmdb => {
std::fs::create_dir_all(&path).map_err(|e| { std::fs::create_dir_all(&path).map_err(|e| {
Error(format!("Unable to create LMDB data directory: {}", e).into()) Error(format!("Unable to create LMDB data directory: {}", e).into())
})?; })?;
let map_size = lmdb_adapter::recommended_map_size(); let map_size = match open.lmdb.map_size {
Some(c) => c.as_u64() as usize,
None => lmdb_adapter::recommended_map_size(),
};
let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new(); let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
env_builder.max_dbs(100); env_builder.max_dbs(100);
@ -62,8 +91,13 @@ fn open_db(path: PathBuf, engine: String) -> Result<Db> {
let db = env_builder.open(&path)?; let db = env_builder.open(&path)?;
Ok(lmdb_adapter::LmdbDb::init(db)) Ok(lmdb_adapter::LmdbDb::init(db))
} }
e => Err(Error(
format!("Invalid or unsupported DB engine: {}", e).into(), // Pattern is unreachable when all supported DB engines are compiled into binary. The allow
// attribute is added so that we won't have to change this match in case stop building
// support for one or more engines by default.
#[allow(unreachable_patterns)]
engine => Err(Error(
format!("Engine support not available in this build: {}", engine).into(),
)), )),
} }
} }