convert_db: allow LMDB map size override #691
2 changed files with 88 additions and 12 deletions
|
@ -171,6 +171,48 @@ impl Db {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// List of supported database engine types
|
||||||
|
///
|
||||||
|
/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
|
||||||
|
/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
|
||||||
|
/// and valid engine, whose support is not enabled via feature flag.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum Engine {
|
||||||
|
Lmdb,
|
||||||
|
Sqlite,
|
||||||
|
Sled,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Engine {
|
||||||
|
/// Return variant name as static `&str`
|
||||||
|
pub fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Lmdb => "lmdb",
|
||||||
|
Self::Sqlite => "sqlite",
|
||||||
|
Self::Sled => "sled",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Engine {
|
||||||
|
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||||
|
self.as_str().fmt(fmt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::str::FromStr for Engine {
|
||||||
|
type Err = Error;
|
||||||
|
|
||||||
|
fn from_str(text: &str) -> Result<Engine> {
|
||||||
|
match text {
|
||||||
|
"lmdb" | "heed" => Ok(Self::Lmdb),
|
||||||
|
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
|
||||||
|
"sled" => Ok(Self::Sled),
|
||||||
|
kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::len_without_is_empty)]
|
#[allow(clippy::len_without_is_empty)]
|
||||||
impl Tree {
|
impl Tree {
|
||||||
#[inline]
|
#[inline]
|
||||||
|
|
|
@ -14,44 +14,73 @@ pub struct ConvertDbOpt {
|
||||||
/// Input database engine (sled, lmdb or sqlite; limited by db engines
|
/// Input database engine (sled, lmdb or sqlite; limited by db engines
|
||||||
/// enabled in this build)
|
/// enabled in this build)
|
||||||
#[structopt(short = "a")]
|
#[structopt(short = "a")]
|
||||||
input_engine: String,
|
input_engine: Engine,
|
||||||
|
|
||||||
/// Output database path
|
/// Output database path
|
||||||
#[structopt(short = "o")]
|
#[structopt(short = "o")]
|
||||||
output_path: PathBuf,
|
output_path: PathBuf,
|
||||||
/// Output database engine
|
/// Output database engine
|
||||||
#[structopt(short = "b")]
|
#[structopt(short = "b")]
|
||||||
output_engine: String,
|
output_engine: Engine,
|
||||||
|
|
||||||
|
#[structopt(flatten)]
|
||||||
|
db_open: OpenDbOpt,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Overrides for database open operation
|
||||||
|
#[derive(StructOpt, Debug, Default)]
|
||||||
|
pub struct OpenDbOpt {
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
|
#[structopt(flatten)]
|
||||||
|
lmdb: OpenLmdbOpt,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Overrides for LMDB database open operation
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
|
#[derive(StructOpt, Debug, Default)]
|
||||||
|
pub struct OpenLmdbOpt {
|
||||||
|
/// LMDB map size override
|
||||||
|
/// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB)
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
|
#[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)]
|
||||||
|
map_size: Option<bytesize::ByteSize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
|
pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
|
||||||
let input = open_db(args.input_path, args.input_engine)?;
|
if args.input_engine == args.output_engine {
|
||||||
let output = open_db(args.output_path, args.output_engine)?;
|
return Err(Error("input and output database engine must differ".into()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
|
||||||
|
let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
|
||||||
output.import(&input)?;
|
output.import(&input)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn open_db(path: PathBuf, engine: String) -> Result<Db> {
|
fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
|
||||||
match engine.as_str() {
|
match engine {
|
||||||
#[cfg(feature = "sled")]
|
#[cfg(feature = "sled")]
|
||||||
"sled" => {
|
Engine::Sled => {
|
||||||
let db = sled_adapter::sled::Config::default().path(&path).open()?;
|
let db = sled_adapter::sled::Config::default().path(&path).open()?;
|
||||||
Ok(sled_adapter::SledDb::init(db))
|
Ok(sled_adapter::SledDb::init(db))
|
||||||
}
|
}
|
||||||
#[cfg(feature = "sqlite")]
|
#[cfg(feature = "sqlite")]
|
||||||
"sqlite" | "sqlite3" | "rusqlite" => {
|
Engine::Sqlite => {
|
||||||
let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
|
let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
|
||||||
db.pragma_update(None, "journal_mode", &"WAL")?;
|
db.pragma_update(None, "journal_mode", &"WAL")?;
|
||||||
db.pragma_update(None, "synchronous", &"NORMAL")?;
|
db.pragma_update(None, "synchronous", &"NORMAL")?;
|
||||||
Ok(sqlite_adapter::SqliteDb::init(db))
|
Ok(sqlite_adapter::SqliteDb::init(db))
|
||||||
}
|
}
|
||||||
#[cfg(feature = "lmdb")]
|
#[cfg(feature = "lmdb")]
|
||||||
"lmdb" | "heed" => {
|
Engine::Lmdb => {
|
||||||
std::fs::create_dir_all(&path).map_err(|e| {
|
std::fs::create_dir_all(&path).map_err(|e| {
|
||||||
Error(format!("Unable to create LMDB data directory: {}", e).into())
|
Error(format!("Unable to create LMDB data directory: {}", e).into())
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let map_size = lmdb_adapter::recommended_map_size();
|
let map_size = match open.lmdb.map_size {
|
||||||
|
Some(c) => c.as_u64() as usize,
|
||||||
|
None => lmdb_adapter::recommended_map_size(),
|
||||||
|
};
|
||||||
|
|
||||||
let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
|
let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
|
||||||
env_builder.max_dbs(100);
|
env_builder.max_dbs(100);
|
||||||
|
@ -62,8 +91,13 @@ fn open_db(path: PathBuf, engine: String) -> Result<Db> {
|
||||||
let db = env_builder.open(&path)?;
|
let db = env_builder.open(&path)?;
|
||||||
Ok(lmdb_adapter::LmdbDb::init(db))
|
Ok(lmdb_adapter::LmdbDb::init(db))
|
||||||
}
|
}
|
||||||
e => Err(Error(
|
|
||||||
format!("Invalid or unsupported DB engine: {}", e).into(),
|
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
|
||||||
|
// attribute is added so that we won't have to change this match in case stop building
|
||||||
|
// support for one or more engines by default.
|
||||||
|
#[allow(unreachable_patterns)]
|
||||||
|
engine => Err(Error(
|
||||||
|
format!("Engine support not available in this build: {}", engine).into(),
|
||||||
)),
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue