convert_db: allow LMDB map size override #691

Merged
lx merged 4 commits from zdenek.crha/garage:convert_db_lmdb_map_size into main 2024-01-24 08:19:45 +00:00
Showing only changes of commit 8527dd87cc - Show all commits

View file

@ -22,16 +22,38 @@ pub struct ConvertDbOpt {
/// Output database engine /// Output database engine
#[structopt(short = "b")] #[structopt(short = "b")]
output_engine: String, output_engine: String,
#[structopt(flatten)]
output_open: OpenDbOpt,
zdenek.crha marked this conversation as resolved Outdated
Outdated
Review

This is no longer specific to the output db, so maybe rename it db_open ?

This is no longer specific to the output db, so maybe rename it `db_open` ?
}
/// Open database config overrides
#[derive(StructOpt, Debug, Default)]
pub struct OpenDbOpt {
#[cfg(feature = "lmdb")]
#[structopt(flatten)]
lmdb: OpenLmdbOpt,
}
/// Output LMDB database config overrides
#[cfg(feature = "lmdb")]
#[derive(StructOpt, Debug, Default)]
pub struct OpenLmdbOpt {
/// Output LMDB map size override
zdenek.crha marked this conversation as resolved Outdated
Outdated
Review

Similarly, comment should be updated to say that it applies to either input or output db

Similarly, comment should be updated to say that it applies to either input or output db
/// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB)
#[cfg(feature = "lmdb")]
#[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)]
map_size: Option<bytesize::ByteSize>,
} }
pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
let input = open_db(args.input_path, args.input_engine)?; let input = open_db(args.input_path, args.input_engine, OpenDbOpt::default())?;
let output = open_db(args.output_path, args.output_engine)?; let output = open_db(args.output_path, args.output_engine, args.output_open)?;
output.import(&input)?; output.import(&input)?;
Ok(()) Ok(())
} }
fn open_db(path: PathBuf, engine: String) -> Result<Db> { fn open_db(path: PathBuf, engine: String, open: OpenDbOpt) -> Result<Db> {
match engine.as_str() { match engine.as_str() {
#[cfg(feature = "sled")] #[cfg(feature = "sled")]
"sled" => { "sled" => {
@ -51,7 +73,10 @@ fn open_db(path: PathBuf, engine: String) -> Result<Db> {
Error(format!("Unable to create LMDB data directory: {}", e).into()) Error(format!("Unable to create LMDB data directory: {}", e).into())
})?; })?;
let map_size = lmdb_adapter::recommended_map_size(); let map_size = match open.lmdb.map_size {
Some(c) => c.as_u64() as usize,
None => lmdb_adapter::recommended_map_size(),
};
let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new(); let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
env_builder.max_dbs(100); env_builder.max_dbs(100);