Conversion utility

Cargo.lock

@@ -1014,6 +1014,7 @@ dependencies = [
 name = "garage_db"
 version = "0.8.0"
 dependencies = [
+ "clap 3.1.18",
  "err-derive 0.3.1",
  "hexdump",
  "mktemp",

src/db/Cargo.toml

@@ -11,7 +11,10 @@ readme = "../../README.md"
 [lib]
 path = "lib.rs"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[[bin]]
+name = "convert"
+path = "bin/convert.rs"
+required-features = ["cli"]
 
 [dependencies]
 err-derive = "0.3"
@@ -20,5 +23,11 @@ hexdump = "0.1"
 sled = "0.34"
 rusqlite = "0.27"
 
+# cli deps
+clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
+
 [dev-dependencies]
 mktemp = "0.4"
+
+[features]
+cli = ["clap"]

src/db/bin/convert.rs

@@ -0,0 +1,55 @@
+use std::path::PathBuf;
+
+use garage_db::*;
+
+use clap::{Parser};
+
+/// K2V command line interface
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+	/// Input DB path
+	#[clap(short = 'i')]
+	input_path: PathBuf,
+	/// Input DB engine
+	#[clap(short = 'a')]
+	input_engine: String,
+
+	/// Output DB path
+	#[clap(short = 'o')]
+	output_path: PathBuf,
+	/// Output DB engine
+	#[clap(short = 'b')]
+	output_engine: String,
+}
+
+fn main() {
+	let args = Args::parse();
+	match do_conversion(args) {
+		Ok(()) => println!("Success!"),
+		Err(e) => eprintln!("Error: {}", e),
+	}
+}
+
+fn do_conversion(args: Args) -> Result<()> {
+	let input = open_db(args.input_path, args.input_engine)?;
+	let output = open_db(args.output_path, args.output_engine)?;
+	output.import(&input)?;
+	Ok(())
+}
+
+fn open_db(path: PathBuf, engine: String) -> Result<Db> {
+	match engine.as_str() {
+		"sled" => {
+			let db = sled_adapter::sled::Config::default()
+				.path(&path)
+				.open()?;
+			Ok(sled_adapter::SledDb::init(db))
+		}
+		"sqlite" | "rusqlite" => {
+			let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
+			Ok(sqlite_adapter::SqliteDb::init(db))
+		}
+		e => Err(Error(format!("Invalid DB engine: {}", e).into())),
+	}
+}

src/db/lib.rs

@@ -23,8 +23,6 @@ pub struct Tree(pub(crate) Arc<dyn IDb>, pub(crate) usize);
 
 pub type ValueIter<'a> = Box<dyn std::iter::Iterator<Item = Result<(Value<'a>, Value<'a>)>> + 'a>;
 
-pub type Exporter<'a> = Box<dyn std::iter::Iterator<Item = Result<(String, ValueIter<'a>)>> + 'a>;
-
 // ----
 
 pub struct Value<'a>(pub(crate) Box<dyn IValue<'a> + 'a>);
@@ -115,7 +113,7 @@ impl<'a> IValue<'a> for &'a [u8] {
 
 #[derive(Debug, Error)]
 #[error(display = "{}", _0)]
-pub struct Error(Cow<'static, str>);
+pub struct Error(pub Cow<'static, str>);
 
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -140,6 +138,10 @@ impl Db {
 		Ok(Tree(self.0.clone(), tree_id))
 	}
 
+	pub fn list_trees(&self) -> Result<Vec<String>> {
+		self.0.list_trees()
+	}
+
 	pub fn transaction<R, E, F>(&self, fun: F) -> TxResult<R, E>
 	where
 		F: Fn(Transaction<'_>) -> TxResult<R, E>,
@@ -175,12 +177,33 @@ impl Db {
 		}
 	}
 
-	pub fn export(&self) -> Result<Exporter<'_>> {
+	pub fn import(&self, other: &Db) -> Result<()> {
-		self.0.export()
+		let existing_trees = self.list_trees()?;
+		if !existing_trees.is_empty() {
+			return Err(Error(format!("destination database already contains data: {:?}", existing_trees).into()));
 		}
 
-	pub fn import(&self, ex: Exporter<'_>) -> Result<()> {
+		let tree_names = other.list_trees()?;
-		self.0.import(ex)
+		for name in tree_names {
+			let tree = self.open_tree(&name)?;
+			if tree.len()? > 0 {
+				return Err(Error(format!("tree {} already contains data", name).into()));
+			}
+
+			let ex_tree = other.open_tree(&name)?;
+
+			let mut i = 0;
+			for item in ex_tree.iter()? {
+				let (k, v) = item?;
+				tree.insert(k, v)?;
+				i += 1;
+				if i % 1000 == 0 {
+					println!("{}: imported {}", name, i);
+				}
+			}
+			println!("{}: finished importing, {} items", name, i);
+		}
+		Ok(())
 	}
 }
 
@@ -293,6 +316,7 @@ impl<'a> Transaction<'a> {
 
 pub(crate) trait IDb: Send + Sync {
 	fn open_tree(&self, name: &str) -> Result<usize>;
+	fn list_trees(&self) -> Result<Vec<String>>;
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>>;
 	fn len(&self, tree: usize) -> Result<usize>;
@@ -317,9 +341,6 @@ pub(crate) trait IDb: Send + Sync {
 	) -> Result<ValueIter<'_>>;
 
 	fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>;
-
-	fn export(&self) -> Result<Exporter<'_>>;
-	fn import(&self, ex: Exporter<'_>) -> Result<()>;
 }
 
 pub(crate) trait ITx<'a> {

src/db/sled_adapter.rs

@@ -10,8 +10,7 @@ use sled::transaction::{
 };
 
 use crate::{
-	Db, Error, Exporter, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value,
+	Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
-	ValueIter,
 };
 
 pub use sled;
@@ -85,6 +84,19 @@ impl IDb for SledDb {
 		}
 	}
 
+	fn list_trees(&self) -> Result<Vec<String>> {
+		let mut trees = vec![];
+		for name in self.db.tree_names() {
+			let name = std::str::from_utf8(&name)
+				.map_err(|e| Error(format!("{}", e).into()))?
+				.to_string();
+			if name != "__sled__default" {
+				trees.push(name);
+			}
+		}
+		Ok(trees)
+	}
+
 	// ----
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> {
@@ -175,54 +187,6 @@ impl IDb for SledDb {
 			Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())),
 		}
 	}
-
-	// ----
-
-	fn export(&self) -> Result<Exporter<'_>> {
-		let mut trees = vec![];
-		for name in self.db.tree_names() {
-			let name = std::str::from_utf8(&name)
-				.map_err(|e| Error(format!("{}", e).into()))?
-				.to_string();
-			let tree = self.open_tree(&name)?;
-			let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
-			trees.push((name, tree));
-		}
-		let trees_exporter: Exporter<'_> = Box::new(trees.into_iter().map(|(name, tree)| {
-			let iter: ValueIter<'_> = Box::new(
-				tree.iter()
-					.map(|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)),
-			);
-			Ok((name, iter))
-		}));
-		Ok(trees_exporter)
-	}
-
-	fn import(&self, ex: Exporter<'_>) -> Result<()> {
-		for ex_tree in ex {
-			let (name, data) = ex_tree?;
-
-			let tree = self.open_tree(&name)?;
-			let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
-			if !tree.is_empty() {
-				return Err(Error(format!("tree {} already contains data", name).into()));
-			}
-
-			let mut i = 0;
-			for item in data {
-				let (k, v) = item?;
-				tree.insert(k, v)?;
-				i += 1;
-				if i % 1000 == 0 {
-					println!("{}: imported {}", name, i);
-				}
-			}
-			println!("{}: finished importing, {} items", name, i);
-		}
-		Ok(())
-	}
-
-	// ----
 }
 
 // ----

src/db/sqlite_adapter.rs

@@ -7,9 +7,7 @@ use std::sync::{Arc, Mutex, MutexGuard, RwLock};
 
 use rusqlite::{params, Connection, Rows, Statement, Transaction};
 
-use crate::{
+use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter};
-	Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
-};
 
 pub use rusqlite;
 
@@ -55,8 +53,10 @@ impl SqliteDb {
 
 impl IDb for SqliteDb {
 	fn open_tree(&self, name: &str) -> Result<usize> {
+		let name = format!("tree_{}", name.replace(":", "_COLON_"));
+
 		let mut trees = self.trees.write().unwrap();
-		if let Some(i) = trees.iter().position(|x| x == name) {
+		if let Some(i) = trees.iter().position(|x| x == &name) {
 			Ok(i)
 		} else {
 			self.db.lock().unwrap().execute(
@@ -75,6 +75,21 @@ impl IDb for SqliteDb {
 		}
 	}
 
+	fn list_trees(&self) -> Result<Vec<String>> {
+		let mut trees = vec![];
+		let db = self.db.lock().unwrap();
+		let mut stmt = db.prepare(
+			"SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'",
+		)?;
+		let mut rows = stmt.query([])?;
+		while let Some(row) = rows.next()? {
+			let name = row.get::<_, String>(0)?;
+			let name = name.replace("_COLON_", ":");
+			trees.push(name);
+		}
+		Ok(trees)
+	}
+
 	// ----
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> {
@@ -197,18 +212,6 @@ impl IDb for SqliteDb {
 			}
 		}
 	}
-
-	// ----
-
-	fn export(&self) -> Result<Exporter<'_>> {
-		unimplemented!()
-	}
-
-	fn import(&self, ex: Exporter<'_>) -> Result<()> {
-		unimplemented!()
-	}
-
-	// ----
 }
 
 // ----