convert_db: allow LMDB map size override #691
2 changed files with 88 additions and 12 deletions
|
@ -171,6 +171,48 @@ impl Db {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// List of supported database engine types
|
||||||
|
///
|
||||||
|
/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
|
||||||
|
/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
|
||||||
|
/// and valid engine, whose support is not enabled via feature flag.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum Engine {
|
||||||
|
Lmdb,
|
||||||
|
Sqlite,
|
||||||
|
Sled,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Engine {
|
||||||
|
/// Return variant name as static `&str`
|
||||||
|
pub fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Lmdb => "lmdb",
|
||||||
|
Self::Sqlite => "sqlite",
|
||||||
|
Self::Sled => "sled",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Engine {
|
||||||
|
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||||
|
self.as_str().fmt(fmt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::str::FromStr for Engine {
|
||||||
|
type Err = Error;
|
||||||
|
|
||||||
|
fn from_str(text: &str) -> Result<Engine> {
|
||||||
|
match text {
|
||||||
|
"lmdb" | "heed" => Ok(Self::Lmdb),
|
||||||
|
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
|
||||||
|
"sled" => Ok(Self::Sled),
|
||||||
|
kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::len_without_is_empty)]
|
#[allow(clippy::len_without_is_empty)]
|
||||||
impl Tree {
|
impl Tree {
|
||||||
#[inline]
|
#[inline]
|
||||||
|
|
|
@ -14,44 +14,73 @@ pub struct ConvertDbOpt {
|
||||||
/// Input database engine (sled, lmdb or sqlite; limited by db engines
|
/// Input database engine (sled, lmdb or sqlite; limited by db engines
|
||||||
/// enabled in this build)
|
/// enabled in this build)
|
||||||
#[structopt(short = "a")]
|
#[structopt(short = "a")]
|
||||||
input_engine: String,
|
input_engine: Engine,
|
||||||
|
|
||||||
/// Output database path
|
/// Output database path
|
||||||
#[structopt(short = "o")]
|
#[structopt(short = "o")]
|
||||||
output_path: PathBuf,
|
output_path: PathBuf,
|
||||||
/// Output database engine
|
/// Output database engine
|
||||||
#[structopt(short = "b")]
|
#[structopt(short = "b")]
|
||||||
output_engine: String,
|
output_engine: Engine,
|
||||||
|
|
||||||
|
#[structopt(flatten)]
|
||||||
|
db_open: OpenDbOpt,
|
||||||
zdenek.crha marked this conversation as resolved
Outdated
|
|||||||
|
}
|
||||||
|
|
||||||
|
/// Overrides for database open operation
|
||||||
|
#[derive(StructOpt, Debug, Default)]
|
||||||
|
pub struct OpenDbOpt {
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
|
#[structopt(flatten)]
|
||||||
|
lmdb: OpenLmdbOpt,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Overrides for LMDB database open operation
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
|
#[derive(StructOpt, Debug, Default)]
|
||||||
|
pub struct OpenLmdbOpt {
|
||||||
|
/// LMDB map size override
|
||||||
zdenek.crha marked this conversation as resolved
Outdated
lx
commented
Similarly, comment should be updated to say that it applies to either input or output db Similarly, comment should be updated to say that it applies to either input or output db
|
|||||||
|
/// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB)
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
|
#[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)]
|
||||||
|
map_size: Option<bytesize::ByteSize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
|
pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
|
||||||
let input = open_db(args.input_path, args.input_engine)?;
|
if args.input_engine == args.output_engine {
|
||||||
let output = open_db(args.output_path, args.output_engine)?;
|
return Err(Error("input and output database engine must differ".into()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
|
||||||
|
let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
|
||||||
output.import(&input)?;
|
output.import(&input)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn open_db(path: PathBuf, engine: String) -> Result<Db> {
|
fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
|
||||||
match engine.as_str() {
|
match engine {
|
||||||
#[cfg(feature = "sled")]
|
#[cfg(feature = "sled")]
|
||||||
"sled" => {
|
Engine::Sled => {
|
||||||
let db = sled_adapter::sled::Config::default().path(&path).open()?;
|
let db = sled_adapter::sled::Config::default().path(&path).open()?;
|
||||||
Ok(sled_adapter::SledDb::init(db))
|
Ok(sled_adapter::SledDb::init(db))
|
||||||
}
|
}
|
||||||
#[cfg(feature = "sqlite")]
|
#[cfg(feature = "sqlite")]
|
||||||
"sqlite" | "sqlite3" | "rusqlite" => {
|
Engine::Sqlite => {
|
||||||
let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
|
let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
|
||||||
db.pragma_update(None, "journal_mode", &"WAL")?;
|
db.pragma_update(None, "journal_mode", &"WAL")?;
|
||||||
db.pragma_update(None, "synchronous", &"NORMAL")?;
|
db.pragma_update(None, "synchronous", &"NORMAL")?;
|
||||||
Ok(sqlite_adapter::SqliteDb::init(db))
|
Ok(sqlite_adapter::SqliteDb::init(db))
|
||||||
}
|
}
|
||||||
#[cfg(feature = "lmdb")]
|
#[cfg(feature = "lmdb")]
|
||||||
"lmdb" | "heed" => {
|
Engine::Lmdb => {
|
||||||
std::fs::create_dir_all(&path).map_err(|e| {
|
std::fs::create_dir_all(&path).map_err(|e| {
|
||||||
Error(format!("Unable to create LMDB data directory: {}", e).into())
|
Error(format!("Unable to create LMDB data directory: {}", e).into())
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let map_size = lmdb_adapter::recommended_map_size();
|
let map_size = match open.lmdb.map_size {
|
||||||
|
Some(c) => c.as_u64() as usize,
|
||||||
|
None => lmdb_adapter::recommended_map_size(),
|
||||||
|
};
|
||||||
|
|
||||||
let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
|
let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
|
||||||
env_builder.max_dbs(100);
|
env_builder.max_dbs(100);
|
||||||
|
@ -62,8 +91,13 @@ fn open_db(path: PathBuf, engine: String) -> Result<Db> {
|
||||||
let db = env_builder.open(&path)?;
|
let db = env_builder.open(&path)?;
|
||||||
Ok(lmdb_adapter::LmdbDb::init(db))
|
Ok(lmdb_adapter::LmdbDb::init(db))
|
||||||
}
|
}
|
||||||
e => Err(Error(
|
|
||||||
format!("Invalid or unsupported DB engine: {}", e).into(),
|
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
|
||||||
|
// attribute is added so that we won't have to change this match in case stop building
|
||||||
|
// support for one or more engines by default.
|
||||||
|
#[allow(unreachable_patterns)]
|
||||||
|
engine => Err(Error(
|
||||||
|
format!("Engine support not available in this build: {}", engine).into(),
|
||||||
)),
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue
This is no longer specific to the output db, so maybe rename it
db_open
?