metadata db snapshotting #775
6 changed files with 44 additions and 1 deletions
|
@ -4769,6 +4769,7 @@ in
|
||||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
src = fetchCratesIo { inherit name version; sha256 = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"; };
|
src = fetchCratesIo { inherit name version; sha256 = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"; };
|
||||||
features = builtins.concatLists [
|
features = builtins.concatLists [
|
||||||
|
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "backup")
|
||||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled")
|
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled")
|
||||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "modern_sqlite")
|
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "modern_sqlite")
|
||||||
];
|
];
|
||||||
|
|
|
@ -17,7 +17,7 @@ hexdump.workspace = true
|
||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
|
|
||||||
heed = { workspace = true, optional = true }
|
heed = { workspace = true, optional = true }
|
||||||
rusqlite = { workspace = true, optional = true }
|
rusqlite = { workspace = true, optional = true, features = ["backup"] }
|
||||||
sled = { workspace = true, optional = true }
|
sled = { workspace = true, optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
|
|
@ -19,6 +19,7 @@ use core::ops::{Bound, RangeBounds};
|
||||||
|
|
||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
use std::cell::Cell;
|
use std::cell::Cell;
|
||||||
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use err_derive::Error;
|
use err_derive::Error;
|
||||||
|
@ -48,6 +49,12 @@ pub type TxValueIter<'a> = Box<dyn std::iter::Iterator<Item = TxOpResult<(Value,
|
||||||
#[error(display = "{}", _0)]
|
#[error(display = "{}", _0)]
|
||||||
pub struct Error(pub Cow<'static, str>);
|
pub struct Error(pub Cow<'static, str>);
|
||||||
|
|
||||||
|
impl From<std::io::Error> for Error {
|
||||||
|
fn from(e: std::io::Error) -> Error {
|
||||||
|
Error(format!("IO: {}", e).into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub type Result<T> = std::result::Result<T, Error>;
|
pub type Result<T> = std::result::Result<T, Error>;
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
|
@ -129,6 +136,10 @@ impl Db {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn snapshot(&self, path: &PathBuf) -> Result<()> {
|
||||||
|
self.0.snapshot(path)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn import(&self, other: &Db) -> Result<()> {
|
pub fn import(&self, other: &Db) -> Result<()> {
|
||||||
let existing_trees = self.list_trees()?;
|
let existing_trees = self.list_trees()?;
|
||||||
if !existing_trees.is_empty() {
|
if !existing_trees.is_empty() {
|
||||||
|
@ -325,6 +336,7 @@ pub(crate) trait IDb: Send + Sync {
|
||||||
fn engine(&self) -> String;
|
fn engine(&self) -> String;
|
||||||
fn open_tree(&self, name: &str) -> Result<usize>;
|
fn open_tree(&self, name: &str) -> Result<usize>;
|
||||||
fn list_trees(&self) -> Result<Vec<String>>;
|
fn list_trees(&self) -> Result<Vec<String>>;
|
||||||
|
fn snapshot(&self, path: &PathBuf) -> Result<()>;
|
||||||
|
|
||||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
|
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
|
||||||
fn len(&self, tree: usize) -> Result<usize>;
|
fn len(&self, tree: usize) -> Result<usize>;
|
||||||
|
|
|
@ -3,6 +3,7 @@ use core::ptr::NonNull;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::convert::TryInto;
|
use std::convert::TryInto;
|
||||||
|
use std::path::PathBuf;
|
||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
|
|
||||||
use heed::types::ByteSlice;
|
use heed::types::ByteSlice;
|
||||||
|
@ -102,6 +103,15 @@ impl IDb for LmdbDb {
|
||||||
Ok(ret2)
|
Ok(ret2)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn snapshot(&self, to: &PathBuf) -> Result<()> {
|
||||||
|
std::fs::create_dir_all(to)?;
|
||||||
|
let mut path = to.clone();
|
||||||
|
path.push("data.mdb");
|
||||||
|
self.db
|
||||||
|
.copy_to_path(path, heed::CompactionOption::Disabled)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// ----
|
// ----
|
||||||
|
|
||||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
||||||
|
|
|
@ -2,6 +2,7 @@ use core::ops::Bound;
|
||||||
|
|
||||||
use std::cell::Cell;
|
use std::cell::Cell;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
|
|
||||||
use sled::transaction::{
|
use sled::transaction::{
|
||||||
|
@ -96,6 +97,13 @@ impl IDb for SledDb {
|
||||||
Ok(trees)
|
Ok(trees)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn snapshot(&self, to: &PathBuf) -> Result<()> {
|
||||||
|
let to_db = sled::open(to)?;
|
||||||
|
let export = self.db.export();
|
||||||
|
to_db.import(export);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// ----
|
// ----
|
||||||
|
|
||||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
||||||
|
|
|
@ -2,6 +2,7 @@ use core::ops::Bound;
|
||||||
|
|
||||||
use std::borrow::BorrowMut;
|
use std::borrow::BorrowMut;
|
||||||
use std::marker::PhantomPinned;
|
use std::marker::PhantomPinned;
|
||||||
|
use std::path::PathBuf;
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::ptr::NonNull;
|
use std::ptr::NonNull;
|
||||||
use std::sync::{Arc, Mutex, MutexGuard};
|
use std::sync::{Arc, Mutex, MutexGuard};
|
||||||
|
@ -119,6 +120,17 @@ impl IDb for SqliteDb {
|
||||||
Ok(trees)
|
Ok(trees)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn snapshot(&self, to: &PathBuf) -> Result<()> {
|
||||||
|
fn progress(p: rusqlite::backup::Progress) {
|
||||||
|
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
|
||||||
|
info!("Sqlite snapshot progres: {}%", percent);
|
||||||
|
}
|
||||||
|
let this = self.0.lock().unwrap();
|
||||||
|
this.db
|
||||||
|
.backup(rusqlite::DatabaseName::Main, to, Some(progress))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// ----
|
// ----
|
||||||
|
|
||||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
||||||
|
|
Loading…
Reference in a new issue