[db-snapshot] Implement db snapshotting logic in garage_db

This commit is contained in:
Alex 2024-03-14 17:24:53 +01:00
parent a80ce6ab5a
commit 8dff278b72
Signed by: lx
GPG key ID: 0E496D15096376BE
6 changed files with 44 additions and 1 deletions

View file

@ -4769,6 +4769,7 @@ in
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"; }; src = fetchCratesIo { inherit name version; sha256 = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"; };
features = builtins.concatLists [ features = builtins.concatLists [
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "backup")
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled") (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled")
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "modern_sqlite") (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "modern_sqlite")
]; ];

View file

@ -17,7 +17,7 @@ hexdump.workspace = true
tracing.workspace = true tracing.workspace = true
heed = { workspace = true, optional = true } heed = { workspace = true, optional = true }
rusqlite = { workspace = true, optional = true } rusqlite = { workspace = true, optional = true, features = ["backup"] }
sled = { workspace = true, optional = true } sled = { workspace = true, optional = true }
[dev-dependencies] [dev-dependencies]

View file

@ -19,6 +19,7 @@ use core::ops::{Bound, RangeBounds};
use std::borrow::Cow; use std::borrow::Cow;
use std::cell::Cell; use std::cell::Cell;
use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use err_derive::Error; use err_derive::Error;
@ -48,6 +49,12 @@ pub type TxValueIter<'a> = Box<dyn std::iter::Iterator<Item = TxOpResult<(Value,
#[error(display = "{}", _0)] #[error(display = "{}", _0)]
pub struct Error(pub Cow<'static, str>); pub struct Error(pub Cow<'static, str>);
impl From<std::io::Error> for Error {
fn from(e: std::io::Error) -> Error {
Error(format!("IO: {}", e).into())
}
}
pub type Result<T> = std::result::Result<T, Error>; pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, Error)] #[derive(Debug, Error)]
@ -129,6 +136,10 @@ impl Db {
} }
} }
pub fn snapshot(&self, path: &PathBuf) -> Result<()> {
self.0.snapshot(path)
}
pub fn import(&self, other: &Db) -> Result<()> { pub fn import(&self, other: &Db) -> Result<()> {
let existing_trees = self.list_trees()?; let existing_trees = self.list_trees()?;
if !existing_trees.is_empty() { if !existing_trees.is_empty() {
@ -325,6 +336,7 @@ pub(crate) trait IDb: Send + Sync {
fn engine(&self) -> String; fn engine(&self) -> String;
fn open_tree(&self, name: &str) -> Result<usize>; fn open_tree(&self, name: &str) -> Result<usize>;
fn list_trees(&self) -> Result<Vec<String>>; fn list_trees(&self) -> Result<Vec<String>>;
fn snapshot(&self, path: &PathBuf) -> Result<()>;
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>; fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
fn len(&self, tree: usize) -> Result<usize>; fn len(&self, tree: usize) -> Result<usize>;

View file

@ -3,6 +3,7 @@ use core::ptr::NonNull;
use std::collections::HashMap; use std::collections::HashMap;
use std::convert::TryInto; use std::convert::TryInto;
use std::path::PathBuf;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use heed::types::ByteSlice; use heed::types::ByteSlice;
@ -102,6 +103,15 @@ impl IDb for LmdbDb {
Ok(ret2) Ok(ret2)
} }
fn snapshot(&self, to: &PathBuf) -> Result<()> {
std::fs::create_dir_all(to)?;
let mut path = to.clone();
path.push("data.mdb");
self.db
.copy_to_path(path, heed::CompactionOption::Disabled)?;
Ok(())
}
// ---- // ----
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> { fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {

View file

@ -2,6 +2,7 @@ use core::ops::Bound;
use std::cell::Cell; use std::cell::Cell;
use std::collections::HashMap; use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use sled::transaction::{ use sled::transaction::{
@ -96,6 +97,13 @@ impl IDb for SledDb {
Ok(trees) Ok(trees)
} }
fn snapshot(&self, to: &PathBuf) -> Result<()> {
let to_db = sled::open(to)?;
let export = self.db.export();
to_db.import(export);
Ok(())
}
// ---- // ----
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> { fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {

View file

@ -2,6 +2,7 @@ use core::ops::Bound;
use std::borrow::BorrowMut; use std::borrow::BorrowMut;
use std::marker::PhantomPinned; use std::marker::PhantomPinned;
use std::path::PathBuf;
use std::pin::Pin; use std::pin::Pin;
use std::ptr::NonNull; use std::ptr::NonNull;
use std::sync::{Arc, Mutex, MutexGuard}; use std::sync::{Arc, Mutex, MutexGuard};
@ -119,6 +120,17 @@ impl IDb for SqliteDb {
Ok(trees) Ok(trees)
} }
fn snapshot(&self, to: &PathBuf) -> Result<()> {
fn progress(p: rusqlite::backup::Progress) {
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
info!("Sqlite snapshot progres: {}%", percent);
}
let this = self.0.lock().unwrap();
this.db
.backup(rusqlite::DatabaseName::Main, to, Some(progress))?;
Ok(())
}
// ---- // ----
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> { fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {