Conversion utility
Some checks reported errors
continuous-integration/drone/push Build was killed
continuous-integration/drone/pr Build was killed

This commit is contained in:
Alex 2022-06-03 11:44:41 +02:00
parent cc0d984118
commit df0877bbba
Signed by: lx
GPG key ID: 0E496D15096376BE
6 changed files with 131 additions and 78 deletions

1
Cargo.lock generated
View file

@ -1014,6 +1014,7 @@ dependencies = [
name = "garage_db" name = "garage_db"
version = "0.8.0" version = "0.8.0"
dependencies = [ dependencies = [
"clap 3.1.18",
"err-derive 0.3.1", "err-derive 0.3.1",
"hexdump", "hexdump",
"mktemp", "mktemp",

View file

@ -11,7 +11,10 @@ readme = "../../README.md"
[lib] [lib]
path = "lib.rs" path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [[bin]]
name = "convert"
path = "bin/convert.rs"
required-features = ["cli"]
[dependencies] [dependencies]
err-derive = "0.3" err-derive = "0.3"
@ -20,5 +23,11 @@ hexdump = "0.1"
sled = "0.34" sled = "0.34"
rusqlite = "0.27" rusqlite = "0.27"
# cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
[dev-dependencies] [dev-dependencies]
mktemp = "0.4" mktemp = "0.4"
[features]
cli = ["clap"]

55
src/db/bin/convert.rs Normal file
View file

@ -0,0 +1,55 @@
use std::path::PathBuf;
use garage_db::*;
use clap::{Parser};
/// K2V command line interface
#[derive(Parser, Debug)]
#[clap(author, version, about, long_about = None)]
struct Args {
/// Input DB path
#[clap(short = 'i')]
input_path: PathBuf,
/// Input DB engine
#[clap(short = 'a')]
input_engine: String,
/// Output DB path
#[clap(short = 'o')]
output_path: PathBuf,
/// Output DB engine
#[clap(short = 'b')]
output_engine: String,
}
fn main() {
let args = Args::parse();
match do_conversion(args) {
Ok(()) => println!("Success!"),
Err(e) => eprintln!("Error: {}", e),
}
}
fn do_conversion(args: Args) -> Result<()> {
let input = open_db(args.input_path, args.input_engine)?;
let output = open_db(args.output_path, args.output_engine)?;
output.import(&input)?;
Ok(())
}
fn open_db(path: PathBuf, engine: String) -> Result<Db> {
match engine.as_str() {
"sled" => {
let db = sled_adapter::sled::Config::default()
.path(&path)
.open()?;
Ok(sled_adapter::SledDb::init(db))
}
"sqlite" | "rusqlite" => {
let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
Ok(sqlite_adapter::SqliteDb::init(db))
}
e => Err(Error(format!("Invalid DB engine: {}", e).into())),
}
}

View file

@ -23,8 +23,6 @@ pub struct Tree(pub(crate) Arc<dyn IDb>, pub(crate) usize);
pub type ValueIter<'a> = Box<dyn std::iter::Iterator<Item = Result<(Value<'a>, Value<'a>)>> + 'a>; pub type ValueIter<'a> = Box<dyn std::iter::Iterator<Item = Result<(Value<'a>, Value<'a>)>> + 'a>;
pub type Exporter<'a> = Box<dyn std::iter::Iterator<Item = Result<(String, ValueIter<'a>)>> + 'a>;
// ---- // ----
pub struct Value<'a>(pub(crate) Box<dyn IValue<'a> + 'a>); pub struct Value<'a>(pub(crate) Box<dyn IValue<'a> + 'a>);
@ -115,7 +113,7 @@ impl<'a> IValue<'a> for &'a [u8] {
#[derive(Debug, Error)] #[derive(Debug, Error)]
#[error(display = "{}", _0)] #[error(display = "{}", _0)]
pub struct Error(Cow<'static, str>); pub struct Error(pub Cow<'static, str>);
pub type Result<T> = std::result::Result<T, Error>; pub type Result<T> = std::result::Result<T, Error>;
@ -140,6 +138,10 @@ impl Db {
Ok(Tree(self.0.clone(), tree_id)) Ok(Tree(self.0.clone(), tree_id))
} }
pub fn list_trees(&self) -> Result<Vec<String>> {
self.0.list_trees()
}
pub fn transaction<R, E, F>(&self, fun: F) -> TxResult<R, E> pub fn transaction<R, E, F>(&self, fun: F) -> TxResult<R, E>
where where
F: Fn(Transaction<'_>) -> TxResult<R, E>, F: Fn(Transaction<'_>) -> TxResult<R, E>,
@ -175,12 +177,33 @@ impl Db {
} }
} }
pub fn export(&self) -> Result<Exporter<'_>> { pub fn import(&self, other: &Db) -> Result<()> {
self.0.export() let existing_trees = self.list_trees()?;
if !existing_trees.is_empty() {
return Err(Error(format!("destination database already contains data: {:?}", existing_trees).into()));
} }
pub fn import(&self, ex: Exporter<'_>) -> Result<()> { let tree_names = other.list_trees()?;
self.0.import(ex) for name in tree_names {
let tree = self.open_tree(&name)?;
if tree.len()? > 0 {
return Err(Error(format!("tree {} already contains data", name).into()));
}
let ex_tree = other.open_tree(&name)?;
let mut i = 0;
for item in ex_tree.iter()? {
let (k, v) = item?;
tree.insert(k, v)?;
i += 1;
if i % 1000 == 0 {
println!("{}: imported {}", name, i);
}
}
println!("{}: finished importing, {} items", name, i);
}
Ok(())
} }
} }
@ -293,6 +316,7 @@ impl<'a> Transaction<'a> {
pub(crate) trait IDb: Send + Sync { pub(crate) trait IDb: Send + Sync {
fn open_tree(&self, name: &str) -> Result<usize>; fn open_tree(&self, name: &str) -> Result<usize>;
fn list_trees(&self) -> Result<Vec<String>>;
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>>; fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>>;
fn len(&self, tree: usize) -> Result<usize>; fn len(&self, tree: usize) -> Result<usize>;
@ -317,9 +341,6 @@ pub(crate) trait IDb: Send + Sync {
) -> Result<ValueIter<'_>>; ) -> Result<ValueIter<'_>>;
fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>; fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>;
fn export(&self) -> Result<Exporter<'_>>;
fn import(&self, ex: Exporter<'_>) -> Result<()>;
} }
pub(crate) trait ITx<'a> { pub(crate) trait ITx<'a> {

View file

@ -10,8 +10,7 @@ use sled::transaction::{
}; };
use crate::{ use crate::{
Db, Error, Exporter, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
ValueIter,
}; };
pub use sled; pub use sled;
@ -85,6 +84,19 @@ impl IDb for SledDb {
} }
} }
fn list_trees(&self) -> Result<Vec<String>> {
let mut trees = vec![];
for name in self.db.tree_names() {
let name = std::str::from_utf8(&name)
.map_err(|e| Error(format!("{}", e).into()))?
.to_string();
if name != "__sled__default" {
trees.push(name);
}
}
Ok(trees)
}
// ---- // ----
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> { fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> {
@ -175,54 +187,6 @@ impl IDb for SledDb {
Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())), Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())),
} }
} }
// ----
fn export(&self) -> Result<Exporter<'_>> {
let mut trees = vec![];
for name in self.db.tree_names() {
let name = std::str::from_utf8(&name)
.map_err(|e| Error(format!("{}", e).into()))?
.to_string();
let tree = self.open_tree(&name)?;
let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
trees.push((name, tree));
}
let trees_exporter: Exporter<'_> = Box::new(trees.into_iter().map(|(name, tree)| {
let iter: ValueIter<'_> = Box::new(
tree.iter()
.map(|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)),
);
Ok((name, iter))
}));
Ok(trees_exporter)
}
fn import(&self, ex: Exporter<'_>) -> Result<()> {
for ex_tree in ex {
let (name, data) = ex_tree?;
let tree = self.open_tree(&name)?;
let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
if !tree.is_empty() {
return Err(Error(format!("tree {} already contains data", name).into()));
}
let mut i = 0;
for item in data {
let (k, v) = item?;
tree.insert(k, v)?;
i += 1;
if i % 1000 == 0 {
println!("{}: imported {}", name, i);
}
}
println!("{}: finished importing, {} items", name, i);
}
Ok(())
}
// ----
} }
// ---- // ----

View file

@ -7,9 +7,7 @@ use std::sync::{Arc, Mutex, MutexGuard, RwLock};
use rusqlite::{params, Connection, Rows, Statement, Transaction}; use rusqlite::{params, Connection, Rows, Statement, Transaction};
use crate::{ use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter};
Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
};
pub use rusqlite; pub use rusqlite;
@ -55,8 +53,10 @@ impl SqliteDb {
impl IDb for SqliteDb { impl IDb for SqliteDb {
fn open_tree(&self, name: &str) -> Result<usize> { fn open_tree(&self, name: &str) -> Result<usize> {
let name = format!("tree_{}", name.replace(":", "_COLON_"));
let mut trees = self.trees.write().unwrap(); let mut trees = self.trees.write().unwrap();
if let Some(i) = trees.iter().position(|x| x == name) { if let Some(i) = trees.iter().position(|x| x == &name) {
Ok(i) Ok(i)
} else { } else {
self.db.lock().unwrap().execute( self.db.lock().unwrap().execute(
@ -75,6 +75,21 @@ impl IDb for SqliteDb {
} }
} }
fn list_trees(&self) -> Result<Vec<String>> {
let mut trees = vec![];
let db = self.db.lock().unwrap();
let mut stmt = db.prepare(
"SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'",
)?;
let mut rows = stmt.query([])?;
while let Some(row) = rows.next()? {
let name = row.get::<_, String>(0)?;
let name = name.replace("_COLON_", ":");
trees.push(name);
}
Ok(trees)
}
// ---- // ----
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> { fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> {
@ -197,18 +212,6 @@ impl IDb for SqliteDb {
} }
} }
} }
// ----
fn export(&self) -> Result<Exporter<'_>> {
unimplemented!()
}
fn import(&self, ex: Exporter<'_>) -> Result<()> {
unimplemented!()
}
// ----
} }
// ---- // ----