convert_db: allow LMDB map size override #691

Merged
lx merged 4 commits from zdenek.crha/garage:convert_db_lmdb_map_size into main 2024-01-24 08:19:45 +00:00
2 changed files with 60 additions and 12 deletions
Showing only changes of commit 4b54e053df - Show all commits

View file

@ -171,6 +171,53 @@ impl Db {
} }
} }
/// List of supported database engine types
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Engine {
#[cfg(feature = "lmdb")]
Lmdb,
#[cfg(feature = "sqlite")]
Sqlite,
#[cfg(feature = "sled")]
Sled,
}
impl std::fmt::Display for Engine {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
#[cfg(feature = "lmdb")]
Self::Lmdb => fmt.write_str("lmdb"),
#[cfg(feature = "sqlite")]
Self::Sqlite => fmt.write_str("sqlite"),
#[cfg(feature = "sled")]
Self::Sled => fmt.write_str("sled"),
}
}
}
impl std::str::FromStr for Engine {
type Err = Error;
fn from_str(text: &str) -> Result<Engine> {
match text {
#[cfg(feature = "lmdb")]
"lmdb" | "heed" => Ok(Self::Lmdb),
#[cfg(feature = "sqlite")]
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
#[cfg(feature = "sled")]
"sled" => Ok(Self::Sled),
kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
}
}
}
#[allow(clippy::len_without_is_empty)] #[allow(clippy::len_without_is_empty)]
impl Tree { impl Tree {
#[inline] #[inline]

View file

@ -14,14 +14,14 @@ pub struct ConvertDbOpt {
/// Input database engine (sled, lmdb or sqlite; limited by db engines /// Input database engine (sled, lmdb or sqlite; limited by db engines
/// enabled in this build) /// enabled in this build)
#[structopt(short = "a")] #[structopt(short = "a")]
input_engine: String, input_engine: Engine,
/// Output database path /// Output database path
#[structopt(short = "o")] #[structopt(short = "o")]
output_path: PathBuf, output_path: PathBuf,
/// Output database engine /// Output database engine
#[structopt(short = "b")] #[structopt(short = "b")]
output_engine: String, output_engine: Engine,
#[structopt(flatten)] #[structopt(flatten)]
output_open: OpenDbOpt, output_open: OpenDbOpt,
zdenek.crha marked this conversation as resolved Outdated
Outdated
Review

This is no longer specific to the output db, so maybe rename it db_open ?

This is no longer specific to the output db, so maybe rename it `db_open` ?
@ -47,28 +47,32 @@ pub struct OpenLmdbOpt {
} }
pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
let input = open_db(args.input_path, args.input_engine, OpenDbOpt::default())?; if args.input_engine == args.output_engine {
let output = open_db(args.output_path, args.output_engine, args.output_open)?; return Err(Error("input and output database engine must differ".into()));
}
let input = open_db(args.input_path, args.input_engine, &args.output_open)?;
let output = open_db(args.output_path, args.output_engine, &args.output_open)?;
output.import(&input)?; output.import(&input)?;
Ok(()) Ok(())
} }
fn open_db(path: PathBuf, engine: String, open: OpenDbOpt) -> Result<Db> { fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
match engine.as_str() { match engine {
#[cfg(feature = "sled")] #[cfg(feature = "sled")]
"sled" => { Engine::Sled => {
let db = sled_adapter::sled::Config::default().path(&path).open()?; let db = sled_adapter::sled::Config::default().path(&path).open()?;
Ok(sled_adapter::SledDb::init(db)) Ok(sled_adapter::SledDb::init(db))
} }
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
"sqlite" | "sqlite3" | "rusqlite" => { Engine::Sqlite => {
let db = sqlite_adapter::rusqlite::Connection::open(&path)?; let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
db.pragma_update(None, "journal_mode", &"WAL")?; db.pragma_update(None, "journal_mode", &"WAL")?;
db.pragma_update(None, "synchronous", &"NORMAL")?; db.pragma_update(None, "synchronous", &"NORMAL")?;
Ok(sqlite_adapter::SqliteDb::init(db)) Ok(sqlite_adapter::SqliteDb::init(db))
} }
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
"lmdb" | "heed" => { Engine::Lmdb => {
std::fs::create_dir_all(&path).map_err(|e| { std::fs::create_dir_all(&path).map_err(|e| {
Error(format!("Unable to create LMDB data directory: {}", e).into()) Error(format!("Unable to create LMDB data directory: {}", e).into())
})?; })?;
@ -87,8 +91,5 @@ fn open_db(path: PathBuf, engine: String, open: OpenDbOpt) -> Result<Db> {
let db = env_builder.open(&path)?; let db = env_builder.open(&path)?;
Ok(lmdb_adapter::LmdbDb::init(db)) Ok(lmdb_adapter::LmdbDb::init(db))
} }
e => Err(Error(
format!("Invalid or unsupported DB engine: {}", e).into(),
)),
} }
} }