forked from Deuxfleurs/garage
[factor-db-open] Combine logic for opening db engines
This commit is contained in:
parent
afee8c2207
commit
ec34728b27
4 changed files with 183 additions and 183 deletions
|
@ -1,5 +1,4 @@
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
#[cfg(feature = "sqlite")]
|
|
||||||
extern crate tracing;
|
extern crate tracing;
|
||||||
|
|
||||||
#[cfg(feature = "lmdb")]
|
#[cfg(feature = "lmdb")]
|
||||||
|
@ -11,6 +10,8 @@ pub mod sqlite_adapter;
|
||||||
|
|
||||||
pub mod counted_tree_hack;
|
pub mod counted_tree_hack;
|
||||||
|
|
||||||
|
pub mod open;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub mod test;
|
pub mod test;
|
||||||
|
|
||||||
|
@ -22,6 +23,8 @@ use std::sync::Arc;
|
||||||
|
|
||||||
use err_derive::Error;
|
use err_derive::Error;
|
||||||
|
|
||||||
|
pub use open::*;
|
||||||
|
|
||||||
pub(crate) type OnCommit = Vec<Box<dyn FnOnce()>>;
|
pub(crate) type OnCommit = Vec<Box<dyn FnOnce()>>;
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
|
@ -171,48 +174,6 @@ impl Db {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// List of supported database engine types
|
|
||||||
///
|
|
||||||
/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
|
|
||||||
/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
|
|
||||||
/// and valid engine, whose support is not enabled via feature flag.
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
||||||
pub enum Engine {
|
|
||||||
Lmdb,
|
|
||||||
Sqlite,
|
|
||||||
Sled,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Engine {
|
|
||||||
/// Return variant name as static `&str`
|
|
||||||
pub fn as_str(&self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Lmdb => "lmdb",
|
|
||||||
Self::Sqlite => "sqlite",
|
|
||||||
Self::Sled => "sled",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::fmt::Display for Engine {
|
|
||||||
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
||||||
self.as_str().fmt(fmt)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::str::FromStr for Engine {
|
|
||||||
type Err = Error;
|
|
||||||
|
|
||||||
fn from_str(text: &str) -> Result<Engine> {
|
|
||||||
match text {
|
|
||||||
"lmdb" | "heed" => Ok(Self::Lmdb),
|
|
||||||
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
|
|
||||||
"sled" => Ok(Self::Sled),
|
|
||||||
kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::len_without_is_empty)]
|
#[allow(clippy::len_without_is_empty)]
|
||||||
impl Tree {
|
impl Tree {
|
||||||
#[inline]
|
#[inline]
|
||||||
|
|
153
src/db/open.rs
Normal file
153
src/db/open.rs
Normal file
|
@ -0,0 +1,153 @@
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use crate::{Db, Error, Result};
|
||||||
|
|
||||||
|
/// List of supported database engine types
|
||||||
|
///
|
||||||
|
/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
|
||||||
|
/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
|
||||||
|
/// and valid engine, whose support is not enabled via feature flag.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum Engine {
|
||||||
|
Lmdb,
|
||||||
|
Sqlite,
|
||||||
|
Sled,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Engine {
|
||||||
|
/// Return variant name as static `&str`
|
||||||
|
pub fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Lmdb => "lmdb",
|
||||||
|
Self::Sqlite => "sqlite",
|
||||||
|
Self::Sled => "sled",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Engine {
|
||||||
|
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||||
|
self.as_str().fmt(fmt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::str::FromStr for Engine {
|
||||||
|
type Err = Error;
|
||||||
|
|
||||||
|
fn from_str(text: &str) -> Result<Engine> {
|
||||||
|
match text {
|
||||||
|
"lmdb" | "heed" => Ok(Self::Lmdb),
|
||||||
|
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
|
||||||
|
"sled" => Ok(Self::Sled),
|
||||||
|
kind => Err(Error(
|
||||||
|
format!(
|
||||||
|
"Invalid DB engine: {} (options are: lmdb, sled, sqlite)",
|
||||||
|
kind
|
||||||
|
)
|
||||||
|
.into(),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct OpenOpt {
|
||||||
|
pub fsync: bool,
|
||||||
|
pub lmdb_map_size: Option<usize>,
|
||||||
|
pub sled_cache_capacity: usize,
|
||||||
|
pub sled_flush_every_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for OpenOpt {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
fsync: false,
|
||||||
|
lmdb_map_size: None,
|
||||||
|
sled_cache_capacity: 1024 * 1024 * 1024,
|
||||||
|
sled_flush_every_ms: 2000,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
|
||||||
|
match engine {
|
||||||
|
// ---- Sled DB ----
|
||||||
|
#[cfg(feature = "sled")]
|
||||||
|
Engine::Sled => {
|
||||||
|
if opt.fsync {
|
||||||
|
return Err(Error(
|
||||||
|
"`metadata_fsync = true` is not supported with the Sled database engine".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
info!("Opening Sled database at: {}", path.display());
|
||||||
|
let db = crate::sled_adapter::sled::Config::default()
|
||||||
|
.path(&path)
|
||||||
|
.cache_capacity(opt.sled_cache_capacity as u64)
|
||||||
|
.flush_every_ms(Some(opt.sled_flush_every_ms))
|
||||||
|
.open()?;
|
||||||
|
Ok(crate::sled_adapter::SledDb::init(db))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Sqlite DB ----
|
||||||
|
#[cfg(feature = "sqlite")]
|
||||||
|
Engine::Sqlite => {
|
||||||
|
info!("Opening Sqlite database at: {}", path.display());
|
||||||
|
let db = crate::sqlite_adapter::rusqlite::Connection::open(&path)?;
|
||||||
|
db.pragma_update(None, "journal_mode", "WAL")?;
|
||||||
|
if opt.fsync {
|
||||||
|
db.pragma_update(None, "synchronous", "NORMAL")?;
|
||||||
|
} else {
|
||||||
|
db.pragma_update(None, "synchronous", "OFF")?;
|
||||||
|
}
|
||||||
|
Ok(crate::sqlite_adapter::SqliteDb::init(db))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- LMDB DB ----
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
|
Engine::Lmdb => {
|
||||||
|
info!("Opening LMDB database at: {}", path.display());
|
||||||
|
if let Err(e) = std::fs::create_dir_all(&path) {
|
||||||
|
return Err(Error(
|
||||||
|
format!("Unable to create LMDB data directory: {}", e).into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let map_size = match opt.lmdb_map_size {
|
||||||
|
None => crate::lmdb_adapter::recommended_map_size(),
|
||||||
|
Some(v) => v - (v % 4096),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut env_builder = heed::EnvOpenOptions::new();
|
||||||
|
env_builder.max_dbs(100);
|
||||||
|
env_builder.map_size(map_size);
|
||||||
|
env_builder.max_readers(2048);
|
||||||
|
unsafe {
|
||||||
|
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
|
||||||
|
if !opt.fsync {
|
||||||
|
env_builder.flag(heed::flags::Flags::MdbNoSync);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match env_builder.open(&path) {
|
||||||
|
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
|
||||||
|
return Err(Error(
|
||||||
|
"OutOfMemory error while trying to open LMDB database. This can happen \
|
||||||
|
if your operating system is not allowing you to use sufficient virtual \
|
||||||
|
memory address space. Please check that no limit is set (ulimit -v). \
|
||||||
|
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
|
||||||
|
On 32-bit machines, you should probably switch to another database engine."
|
||||||
|
.into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
|
||||||
|
Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
|
||||||
|
// attribute is added so that we won't have to change this match in case stop building
|
||||||
|
// support for one or more engines by default.
|
||||||
|
#[allow(unreachable_patterns)]
|
||||||
|
engine => Err(Error(
|
||||||
|
format!("DB engine support not available in this build: {}", engine).into(),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
|
@ -51,53 +51,13 @@ pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
|
||||||
return Err(Error("input and output database engine must differ".into()));
|
return Err(Error("input and output database engine must differ".into()));
|
||||||
}
|
}
|
||||||
|
|
||||||
let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
|
let opt = OpenOpt {
|
||||||
let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
|
lmdb_map_size: args.db_open.lmdb.map_size.map(|x| x.as_u64() as usize),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let input = open_db(&args.input_path, args.input_engine, &opt)?;
|
||||||
|
let output = open_db(&args.output_path, args.output_engine, &opt)?;
|
||||||
output.import(&input)?;
|
output.import(&input)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
|
|
||||||
match engine {
|
|
||||||
#[cfg(feature = "sled")]
|
|
||||||
Engine::Sled => {
|
|
||||||
let db = sled_adapter::sled::Config::default().path(&path).open()?;
|
|
||||||
Ok(sled_adapter::SledDb::init(db))
|
|
||||||
}
|
|
||||||
#[cfg(feature = "sqlite")]
|
|
||||||
Engine::Sqlite => {
|
|
||||||
let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
|
|
||||||
db.pragma_update(None, "journal_mode", "WAL")?;
|
|
||||||
db.pragma_update(None, "synchronous", "NORMAL")?;
|
|
||||||
Ok(sqlite_adapter::SqliteDb::init(db))
|
|
||||||
}
|
|
||||||
#[cfg(feature = "lmdb")]
|
|
||||||
Engine::Lmdb => {
|
|
||||||
std::fs::create_dir_all(&path).map_err(|e| {
|
|
||||||
Error(format!("Unable to create LMDB data directory: {}", e).into())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let map_size = match open.lmdb.map_size {
|
|
||||||
Some(c) => c.as_u64() as usize,
|
|
||||||
None => lmdb_adapter::recommended_map_size(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
|
|
||||||
env_builder.max_dbs(100);
|
|
||||||
env_builder.map_size(map_size);
|
|
||||||
unsafe {
|
|
||||||
env_builder.flag(lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
|
|
||||||
}
|
|
||||||
let db = env_builder.open(&path)?;
|
|
||||||
Ok(lmdb_adapter::LmdbDb::init(db))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
|
|
||||||
// attribute is added so that we won't have to change this match in case stop building
|
|
||||||
// support for one or more engines by default.
|
|
||||||
#[allow(unreachable_patterns)]
|
|
||||||
engine => Err(Error(
|
|
||||||
format!("Engine support not available in this build: {}", engine).into(),
|
|
||||||
)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
use std::str::FromStr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use garage_net::NetworkKey;
|
use garage_net::NetworkKey;
|
||||||
|
@ -113,108 +114,33 @@ impl Garage {
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Opening database...");
|
info!("Opening database...");
|
||||||
|
let db_engine = db::Engine::from_str(&config.db_engine)
|
||||||
|
.ok_or_message("Invalid `db_engine` value in configuration file")?;
|
||||||
let mut db_path = config.metadata_dir.clone();
|
let mut db_path = config.metadata_dir.clone();
|
||||||
let db = match config.db_engine.as_str() {
|
match db_engine {
|
||||||
// ---- Sled DB ----
|
db::Engine::Sled => {
|
||||||
#[cfg(feature = "sled")]
|
|
||||||
"sled" => {
|
|
||||||
if config.metadata_fsync {
|
|
||||||
return Err(Error::Message(format!(
|
|
||||||
"`metadata_fsync = true` is not supported with the Sled database engine"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
db_path.push("db");
|
db_path.push("db");
|
||||||
info!("Opening Sled database at: {}", db_path.display());
|
|
||||||
let db = db::sled_adapter::sled::Config::default()
|
|
||||||
.path(&db_path)
|
|
||||||
.cache_capacity(config.sled_cache_capacity as u64)
|
|
||||||
.flush_every_ms(Some(config.sled_flush_every_ms))
|
|
||||||
.open()
|
|
||||||
.ok_or_message("Unable to open sled DB")?;
|
|
||||||
db::sled_adapter::SledDb::init(db)
|
|
||||||
}
|
}
|
||||||
#[cfg(not(feature = "sled"))]
|
db::Engine::Sqlite => {
|
||||||
"sled" => return Err(Error::Message("sled db not available in this build".into())),
|
|
||||||
// ---- Sqlite DB ----
|
|
||||||
#[cfg(feature = "sqlite")]
|
|
||||||
"sqlite" | "sqlite3" | "rusqlite" => {
|
|
||||||
db_path.push("db.sqlite");
|
db_path.push("db.sqlite");
|
||||||
info!("Opening Sqlite database at: {}", db_path.display());
|
|
||||||
let db = db::sqlite_adapter::rusqlite::Connection::open(db_path)
|
|
||||||
.and_then(|db| {
|
|
||||||
db.pragma_update(None, "journal_mode", &"WAL")?;
|
|
||||||
if config.metadata_fsync {
|
|
||||||
db.pragma_update(None, "synchronous", &"NORMAL")?;
|
|
||||||
} else {
|
|
||||||
db.pragma_update(None, "synchronous", &"OFF")?;
|
|
||||||
}
|
|
||||||
Ok(db)
|
|
||||||
})
|
|
||||||
.ok_or_message("Unable to open sqlite DB")?;
|
|
||||||
db::sqlite_adapter::SqliteDb::init(db)
|
|
||||||
}
|
}
|
||||||
#[cfg(not(feature = "sqlite"))]
|
db::Engine::Lmdb => {
|
||||||
"sqlite" | "sqlite3" | "rusqlite" => {
|
|
||||||
return Err(Error::Message(
|
|
||||||
"sqlite db not available in this build".into(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
// ---- LMDB DB ----
|
|
||||||
#[cfg(feature = "lmdb")]
|
|
||||||
"lmdb" | "heed" => {
|
|
||||||
db_path.push("db.lmdb");
|
db_path.push("db.lmdb");
|
||||||
info!("Opening LMDB database at: {}", db_path.display());
|
|
||||||
std::fs::create_dir_all(&db_path)
|
|
||||||
.ok_or_message("Unable to create LMDB data directory")?;
|
|
||||||
let map_size = match config.lmdb_map_size {
|
|
||||||
v if v == usize::default() => garage_db::lmdb_adapter::recommended_map_size(),
|
|
||||||
v => v - (v % 4096),
|
|
||||||
};
|
|
||||||
|
|
||||||
use db::lmdb_adapter::heed;
|
|
||||||
let mut env_builder = heed::EnvOpenOptions::new();
|
|
||||||
env_builder.max_dbs(100);
|
|
||||||
env_builder.max_readers(500);
|
|
||||||
env_builder.map_size(map_size);
|
|
||||||
unsafe {
|
|
||||||
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
|
|
||||||
if !config.metadata_fsync {
|
|
||||||
env_builder.flag(heed::flags::Flags::MdbNoSync);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let db = match env_builder.open(&db_path) {
|
|
||||||
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
|
|
||||||
return Err(Error::Message(
|
|
||||||
"OutOfMemory error while trying to open LMDB database. This can happen \
|
|
||||||
if your operating system is not allowing you to use sufficient virtual \
|
|
||||||
memory address space. Please check that no limit is set (ulimit -v). \
|
|
||||||
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
|
|
||||||
On 32-bit machines, you should probably switch to another database engine.".into()))
|
|
||||||
}
|
|
||||||
x => x.ok_or_message("Unable to open LMDB DB")?,
|
|
||||||
};
|
|
||||||
db::lmdb_adapter::LmdbDb::init(db)
|
|
||||||
}
|
|
||||||
#[cfg(not(feature = "lmdb"))]
|
|
||||||
"lmdb" | "heed" => return Err(Error::Message("lmdb db not available in this build".into())),
|
|
||||||
// ---- Unavailable DB engine ----
|
|
||||||
e => {
|
|
||||||
return Err(Error::Message(format!(
|
|
||||||
"Unsupported DB engine: {} (options: {})",
|
|
||||||
e,
|
|
||||||
vec![
|
|
||||||
#[cfg(feature = "sled")]
|
|
||||||
"sled",
|
|
||||||
#[cfg(feature = "sqlite")]
|
|
||||||
"sqlite",
|
|
||||||
#[cfg(feature = "lmdb")]
|
|
||||||
"lmdb",
|
|
||||||
]
|
|
||||||
.join(", ")
|
|
||||||
)));
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
let db_opt = db::OpenOpt {
|
||||||
|
fsync: config.metadata_fsync,
|
||||||
|
lmdb_map_size: match config.lmdb_map_size {
|
||||||
|
v if v == usize::default() => None,
|
||||||
|
v => Some(v),
|
||||||
|
},
|
||||||
|
sled_cache_capacity: config.sled_cache_capacity,
|
||||||
|
sled_flush_every_ms: config.sled_flush_every_ms,
|
||||||
};
|
};
|
||||||
|
let db = db::open_db(&db_path, db_engine, &db_opt)
|
||||||
|
.ok_or_message("Unable to open metadata db")?;
|
||||||
|
|
||||||
|
info!("Initializing RPC...");
|
||||||
let network_key = hex::decode(config.rpc_secret.as_ref().ok_or_message(
|
let network_key = hex::decode(config.rpc_secret.as_ref().ok_or_message(
|
||||||
"rpc_secret value is missing, not present in config file or in environment",
|
"rpc_secret value is missing, not present in config file or in environment",
|
||||||
)?)
|
)?)
|
||||||
|
|
Loading…
Reference in a new issue