From df0877bbba91d210fa8a91bd095ca13e0ea2176f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Jun 2022 11:44:41 +0200 Subject: [PATCH] Conversion utility --- Cargo.lock | 1 + src/db/Cargo.toml | 11 ++++++- src/db/bin/convert.rs | 55 ++++++++++++++++++++++++++++++++++ src/db/lib.rs | 43 ++++++++++++++++++++------- src/db/sled_adapter.rs | 64 +++++++++------------------------------- src/db/sqlite_adapter.rs | 35 ++++++++++++---------- 6 files changed, 131 insertions(+), 78 deletions(-) create mode 100644 src/db/bin/convert.rs diff --git a/Cargo.lock b/Cargo.lock index 9ff2c02d..73879369 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1014,6 +1014,7 @@ dependencies = [ name = "garage_db" version = "0.8.0" dependencies = [ + "clap 3.1.18", "err-derive 0.3.1", "hexdump", "mktemp", diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index ca189a67..22abc0b9 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -11,7 +11,10 @@ readme = "../../README.md" [lib] path = "lib.rs" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[[bin]] +name = "convert" +path = "bin/convert.rs" +required-features = ["cli"] [dependencies] err-derive = "0.3" @@ -20,5 +23,11 @@ hexdump = "0.1" sled = "0.34" rusqlite = "0.27" +# cli deps +clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } + [dev-dependencies] mktemp = "0.4" + +[features] +cli = ["clap"] diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs new file mode 100644 index 00000000..8c4f0ddc --- /dev/null +++ b/src/db/bin/convert.rs @@ -0,0 +1,55 @@ +use std::path::PathBuf; + +use garage_db::*; + +use clap::{Parser}; + +/// K2V command line interface +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Args { + /// Input DB path + #[clap(short = 'i')] + input_path: PathBuf, + /// Input DB engine + #[clap(short = 'a')] + input_engine: String, + + /// Output DB path + #[clap(short = 'o')] + output_path: PathBuf, + /// Output DB engine + #[clap(short = 'b')] + output_engine: String, +} + +fn main() { + let args = Args::parse(); + match do_conversion(args) { + Ok(()) => println!("Success!"), + Err(e) => eprintln!("Error: {}", e), + } +} + +fn do_conversion(args: Args) -> Result<()> { + let input = open_db(args.input_path, args.input_engine)?; + let output = open_db(args.output_path, args.output_engine)?; + output.import(&input)?; + Ok(()) +} + +fn open_db(path: PathBuf, engine: String) -> Result { + match engine.as_str() { + "sled" => { + let db = sled_adapter::sled::Config::default() + .path(&path) + .open()?; + Ok(sled_adapter::SledDb::init(db)) + } + "sqlite" | "rusqlite" => { + let db = sqlite_adapter::rusqlite::Connection::open(&path)?; + Ok(sqlite_adapter::SqliteDb::init(db)) + } + e => Err(Error(format!("Invalid DB engine: {}", e).into())), + } +} diff --git a/src/db/lib.rs b/src/db/lib.rs index 95d2c16b..49ec0765 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -23,8 +23,6 @@ pub struct Tree(pub(crate) Arc, pub(crate) usize); pub type ValueIter<'a> = Box, Value<'a>)>> + 'a>; -pub type Exporter<'a> = Box)>> + 'a>; - // ---- pub struct Value<'a>(pub(crate) Box + 'a>); @@ -115,7 +113,7 @@ impl<'a> IValue<'a> for &'a [u8] { #[derive(Debug, Error)] #[error(display = "{}", _0)] -pub struct Error(Cow<'static, str>); +pub struct Error(pub Cow<'static, str>); pub type Result = std::result::Result; @@ -140,6 +138,10 @@ impl Db { Ok(Tree(self.0.clone(), tree_id)) } + pub fn list_trees(&self) -> Result> { + self.0.list_trees() + } + pub fn transaction(&self, fun: F) -> TxResult where F: Fn(Transaction<'_>) -> TxResult, @@ -175,12 +177,33 @@ impl Db { } } - pub fn export(&self) -> Result> { - self.0.export() - } + pub fn import(&self, other: &Db) -> Result<()> { + let existing_trees = self.list_trees()?; + if !existing_trees.is_empty() { + return Err(Error(format!("destination database already contains data: {:?}", existing_trees).into())); + } - pub fn import(&self, ex: Exporter<'_>) -> Result<()> { - self.0.import(ex) + let tree_names = other.list_trees()?; + for name in tree_names { + let tree = self.open_tree(&name)?; + if tree.len()? > 0 { + return Err(Error(format!("tree {} already contains data", name).into())); + } + + let ex_tree = other.open_tree(&name)?; + + let mut i = 0; + for item in ex_tree.iter()? { + let (k, v) = item?; + tree.insert(k, v)?; + i += 1; + if i % 1000 == 0 { + println!("{}: imported {}", name, i); + } + } + println!("{}: finished importing, {} items", name, i); + } + Ok(()) } } @@ -293,6 +316,7 @@ impl<'a> Transaction<'a> { pub(crate) trait IDb: Send + Sync { fn open_tree(&self, name: &str) -> Result; + fn list_trees(&self) -> Result>; fn get(&self, tree: usize, key: &[u8]) -> Result>>; fn len(&self, tree: usize) -> Result; @@ -317,9 +341,6 @@ pub(crate) trait IDb: Send + Sync { ) -> Result>; fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; - - fn export(&self) -> Result>; - fn import(&self, ex: Exporter<'_>) -> Result<()>; } pub(crate) trait ITx<'a> { diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs index 7382776f..3388b0ca 100644 --- a/src/db/sled_adapter.rs +++ b/src/db/sled_adapter.rs @@ -10,8 +10,7 @@ use sled::transaction::{ }; use crate::{ - Db, Error, Exporter, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, - ValueIter, + Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter, }; pub use sled; @@ -85,6 +84,19 @@ impl IDb for SledDb { } } + fn list_trees(&self) -> Result> { + let mut trees = vec![]; + for name in self.db.tree_names() { + let name = std::str::from_utf8(&name) + .map_err(|e| Error(format!("{}", e).into()))? + .to_string(); + if name != "__sled__default" { + trees.push(name); + } + } + Ok(trees) + } + // ---- fn get(&self, tree: usize, key: &[u8]) -> Result>> { @@ -175,54 +187,6 @@ impl IDb for SledDb { Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())), } } - - // ---- - - fn export(&self) -> Result> { - let mut trees = vec![]; - for name in self.db.tree_names() { - let name = std::str::from_utf8(&name) - .map_err(|e| Error(format!("{}", e).into()))? - .to_string(); - let tree = self.open_tree(&name)?; - let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); - trees.push((name, tree)); - } - let trees_exporter: Exporter<'_> = Box::new(trees.into_iter().map(|(name, tree)| { - let iter: ValueIter<'_> = Box::new( - tree.iter() - .map(|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)), - ); - Ok((name, iter)) - })); - Ok(trees_exporter) - } - - fn import(&self, ex: Exporter<'_>) -> Result<()> { - for ex_tree in ex { - let (name, data) = ex_tree?; - - let tree = self.open_tree(&name)?; - let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone(); - if !tree.is_empty() { - return Err(Error(format!("tree {} already contains data", name).into())); - } - - let mut i = 0; - for item in data { - let (k, v) = item?; - tree.insert(k, v)?; - i += 1; - if i % 1000 == 0 { - println!("{}: imported {}", name, i); - } - } - println!("{}: finished importing, {} items", name, i); - } - Ok(()) - } - - // ---- } // ---- diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs index 31d20553..386eb951 100644 --- a/src/db/sqlite_adapter.rs +++ b/src/db/sqlite_adapter.rs @@ -7,9 +7,7 @@ use std::sync::{Arc, Mutex, MutexGuard, RwLock}; use rusqlite::{params, Connection, Rows, Statement, Transaction}; -use crate::{ - Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter, -}; +use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter}; pub use rusqlite; @@ -55,8 +53,10 @@ impl SqliteDb { impl IDb for SqliteDb { fn open_tree(&self, name: &str) -> Result { + let name = format!("tree_{}", name.replace(":", "_COLON_")); + let mut trees = self.trees.write().unwrap(); - if let Some(i) = trees.iter().position(|x| x == name) { + if let Some(i) = trees.iter().position(|x| x == &name) { Ok(i) } else { self.db.lock().unwrap().execute( @@ -75,6 +75,21 @@ impl IDb for SqliteDb { } } + fn list_trees(&self) -> Result> { + let mut trees = vec![]; + let db = self.db.lock().unwrap(); + let mut stmt = db.prepare( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'", + )?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let name = row.get::<_, String>(0)?; + let name = name.replace("_COLON_", ":"); + trees.push(name); + } + Ok(trees) + } + // ---- fn get(&self, tree: usize, key: &[u8]) -> Result>> { @@ -197,18 +212,6 @@ impl IDb for SqliteDb { } } } - - // ---- - - fn export(&self) -> Result> { - unimplemented!() - } - - fn import(&self, ex: Exporter<'_>) -> Result<()> { - unimplemented!() - } - - // ---- } // ----