Deuxfleurs/garage
44
56
Fork 73
Code Issues 100 Pull requests 14 Releases 30 Wiki Activity

WIP add content defined chunking #42

Closed
trinity-1686a wants to merge 42 commits from content-defined-chunking into master
pull from: content-defined-chunking
merge into: Deuxfleurs:master
Conversation 3 Commits 42 Files changed 60 +26 -379
7 changed files with 26 additions and 379 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 445912dc6a - Show all commits

260
Cargo.lock generated
View file

@ -194,15 +194,6 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "cloudabi"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
dependencies = [
"bitflags",
]
[[package]]
name = "crc32fast"
version = "1.2.1"
@ -212,21 +203,6 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crossbeam-epoch"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
dependencies = [
"autocfg",
"cfg-if 0.1.10",
"crossbeam-utils 0.7.2",
"lazy_static",
"maybe-uninit",
"memoffset 0.5.6",
"scopeguard",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.2"
@ -234,24 +210,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d60ab4a8dba064f2fbb5aa270c28da5cf4bbd0e72dae1140a6b0353a779dbe00"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.2",
"crossbeam-utils",
"lazy_static",
"loom",
"memoffset 0.6.1",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg",
"cfg-if 0.1.10",
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.2"
@ -518,9 +483,9 @@ dependencies = [
"futures-util",
"garage_api",
"garage_model",
"garage_rpc 0.1.1",
"garage_table 0.1.1",
"garage_util 0.1.1",
"garage_rpc",
"garage_table",
"garage_util",
"garage_web",
"hex",
"log",
@ -529,8 +494,7 @@ dependencies = [
"rmp-serde",
"serde",
"sha2",
"sled 0.31.0",
"sled 0.34.6",
"sled",
"structopt",
"tokio",
"toml",
@ -548,8 +512,8 @@ dependencies = [
"futures",
"futures-util",
"garage_model",
"garage_table 0.1.1",
"garage_util 0.1.1",
"garage_table",
"garage_util",
"hex",
"hmac",
"http",
@ -575,10 +539,9 @@ dependencies = [
"bytes 0.4.12",
"futures",
"futures-util",
"garage_model_010b",
"garage_rpc 0.1.1",
"garage_table 0.1.1",
"garage_util 0.1.1",
"garage_rpc",
"garage_table",
"garage_util",
"hex",
"log",
"rand 0.7.3",
@ -586,32 +549,7 @@ dependencies = [
"serde",
"serde_bytes",
"sha2",
"sled 0.34.6",
"tokio",
]
[[package]]
name = "garage_model_010b"
version = "0.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa7bd48aaf0f423a4d18e762c8c8c19a230f9fc237c63c1356ded0463a810428"
dependencies = [
"arc-swap",
"async-trait",
"bytes 0.4.12",
"futures",
"futures-util",
"garage_rpc 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"garage_table 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"garage_util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"hex",
"log",
"rand 0.7.3",
"rmp-serde",
"serde",
"serde_bytes",
"sha2",
"sled 0.31.0",
"sled",
"tokio",
]
@ -623,35 +561,7 @@ dependencies = [
"bytes 0.4.12",
"futures",
"futures-util",
"garage_util 0.1.1",
"gethostname",
"hex",
"http",
"hyper",
"hyper-rustls",
"log",
"rand 0.7.3",
"rmp-serde",
"rustls",
"serde",
"serde_json",
"sha2",
"tokio",
"tokio-rustls",
"webpki",
]
[[package]]
name = "garage_rpc"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d374863eb8eff7a9593dbbb73e4c0ea33cd8e88ffc6ac77076ae36721bda805c"
dependencies = [
"arc-swap",
"bytes 0.4.12",
"futures",
"futures-util",
"garage_util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"garage_util",
"gethostname",
"hex",
"http",
@ -678,8 +588,8 @@ dependencies = [
"bytes 0.4.12",
"futures",
"futures-util",
"garage_rpc 0.1.1",
"garage_util 0.1.1",
"garage_rpc",
"garage_util",
"hex",
"hexdump",
"log",
@ -687,31 +597,7 @@ dependencies = [
"rmp-serde",
"serde",
"serde_bytes",
"sled 0.34.6",
"tokio",
]
[[package]]
name = "garage_table"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65d14bc1a342b4806a824ece64d37cb849ea5b1c015be9946695b5c163807154"
dependencies = [
"arc-swap",
"async-trait",
"bytes 0.4.12",
"futures",
"futures-util",
"garage_rpc 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"garage_util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"hex",
"hexdump",
"log",
"rand 0.7.3",
"rmp-serde",
"serde",
"serde_bytes",
"sled 0.31.0",
"sled",
"tokio",
]
@ -735,33 +621,7 @@ dependencies = [
"serde",
"serde_json",
"sha2",
"sled 0.34.6",
"tokio",
"toml",
"webpki",
]
[[package]]
name = "garage_util"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "393cc63d44006449fc72c15760aa2962a0e47d95ed8d9d82713aac7decb7f988"
dependencies = [
"err-derive",
"futures",
"futures-util",
"hex",
"http",
"hyper",
"log",
"rand 0.7.3",
"rmp-serde",
"roxmltree",
"rustls",
"serde",
"serde_json",
"sha2",
"sled 0.31.0",
"sled",
"tokio",
"toml",
"webpki",
@ -775,8 +635,8 @@ dependencies = [
"futures",
"garage_api",
"garage_model",
"garage_table 0.1.1",
"garage_util 0.1.1",
"garage_table",
"garage_util",
"http",
"hyper",
"idna",
@ -1084,15 +944,6 @@ version = "0.2.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
[[package]]
name = "lock_api"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75"
dependencies = [
"scopeguard",
]
[[package]]
name = "lock_api"
version = "0.4.2"
@ -1128,12 +979,6 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
[[package]]
name = "maybe-uninit"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "md-5"
version = "0.9.1"
@ -1151,15 +996,6 @@ version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memoffset"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
dependencies = [
"autocfg",
]
[[package]]
name = "memoffset"
version = "0.6.1"
@ -1281,16 +1117,6 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "parking_lot"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
dependencies = [
"lock_api 0.3.4",
"parking_lot_core 0.7.2",
]
[[package]]
name = "parking_lot"
version = "0.11.1"
@ -1298,22 +1124,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
dependencies = [
"instant",
"lock_api 0.4.2",
"parking_lot_core 0.8.3",
]
[[package]]
name = "parking_lot_core"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
dependencies = [
"cfg-if 0.1.10",
"cloudabi",
"libc",
"redox_syscall 0.1.57",
"smallvec",
"winapi 0.3.9",
"lock_api",
"parking_lot_core",
]
[[package]]
@ -1325,7 +1137,7 @@ dependencies = [
"cfg-if 1.0.0",
"instant",
"libc",
"redox_syscall 0.2.5",
"redox_syscall",
"smallvec",
"winapi 0.3.9",
]
@ -1548,12 +1360,6 @@ dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "redox_syscall"
version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
[[package]]
name = "redox_syscall"
version = "0.2.5"
@ -1746,22 +1552,6 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
[[package]]
name = "sled"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fb6824dde66ad33bf20c6e8476f5b82b871bc8bc3c129a10ea2f7dae5060fa3"
dependencies = [
"crc32fast",
"crossbeam-epoch 0.8.2",
"crossbeam-utils 0.7.2",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot 0.10.2",
]
[[package]]
name = "sled"
version = "0.34.6"
@ -1769,13 +1559,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
dependencies = [
"crc32fast",
"crossbeam-epoch 0.9.2",
"crossbeam-utils 0.8.2",
"crossbeam-epoch",
"crossbeam-utils",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot 0.11.1",
"parking_lot",
]
[[package]]

1
src/garage/Cargo.toml
View file

@ -29,7 +29,6 @@ log = "0.4"
pretty_env_logger = "0.4"
sled = "0.34"
old_sled = { package = "sled", version = "0.31" }
structopt = { version = "0.3", default-features = false }
toml = "0.5"

23
src/garage/server.rs
View file

@ -40,28 +40,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
info!("Opening database...");
let mut db_path = config.metadata_dir.clone();
db_path.push("db");
let db = match sled::open(&db_path) {
Ok(db) => db,
Err(e) => {
warn!("Old DB could not be openned ({}), attempting migration.", e);
let old = old_sled::open(&db_path).expect("Unable to open old DB for migration");
let mut new_path = config.metadata_dir.clone();
new_path.push("db2");
let new = sled::open(&new_path).expect("Unable to open new DB for migration");
new.import(old.export());
if old.checksum().expect("unable to compute old db checksum")
!= new.checksum().expect("unable to compute new db checksum")
{
panic!("db checksums don't match after migration");
}
drop(new);
drop(old);
std::fs::remove_dir_all(&db_path).expect("Cannot remove old DB folder");
std::fs::rename(new_path, &db_path)
.expect("Cannot move new DB folder to correct place");
sled::open(db_path).expect("Unable to open new DB after migration")
}
};
let db = sled::open(&db_path).expect("Unable to open sled DB");
info!("Initialize RPC server...");
let mut rpc_server = RpcServer::new(config.rpc_bind_addr.clone(), config.rpc_tls.clone());

1
src/model/Cargo.toml
View file

@ -16,7 +16,6 @@ path = "lib.rs"
garage_util = { version = "0.1.1", path = "../util" }
garage_rpc = { version = "0.1.1", path = "../rpc" }
garage_table = { version = "0.1.1", path = "../table" }
model010 = { package = "garage_model_010b", version = "0.0.1" }
bytes = "0.4"
rand = "0.7"

40
src/model/bucket_table.rs
View file

@ -5,11 +5,6 @@ use garage_table::*;
use crate::key_table::PermissionSet;
// We import the same file but in its version 0.1.0.
// We can then access v0.1.0 data structures.
// We use them to perform migrations.
use model010::bucket_table as prev;
/// A bucket is a collection of objects
///
/// Its parameters are not directly accessible as:
@ -108,39 +103,4 @@ impl TableSchema for BucketTable {
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
filter.apply(entry.is_deleted())
}
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
let old = match rmp_serde::decode::from_read_ref::<_, prev::Bucket>(bytes) {
Ok(x) => x,
Err(_) => return None,
};
if old.deleted {
Some(Bucket {
name: old.name,
state: crdt::LWW::migrate_from_raw(old.timestamp, BucketState::Deleted),
})
} else {
let mut keys = crdt::LWWMap::new();
for ak in old.authorized_keys() {
keys.merge(&crdt::LWWMap::migrate_from_raw_item(
ak.key_id.clone(),
ak.timestamp,
PermissionSet {
allow_read: ak.allow_read,
allow_write: ak.allow_write,
},
));
}
let params = BucketParams {
authorized_keys: keys,
website: crdt::LWW::new(false),
};
Some(Bucket {
name: old.name,
state: crdt::LWW::migrate_from_raw(old.timestamp, BucketState::Present(params)),
})
}
}
}

28
src/model/key_table.rs
View file

@ -3,8 +3,6 @@ use serde::{Deserialize, Serialize};
use garage_table::crdt::*;
use garage_table::*;
use model010::key_table as prev;
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct Key {
// Primary key
@ -103,30 +101,4 @@ impl TableSchema for KeyTable {
fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
filter.apply(entry.deleted.get())
}
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
let old = match rmp_serde::decode::from_read_ref::<_, prev::Key>(bytes) {
Ok(x) => x,
Err(_) => return None,
};
let mut new = Self::E {
key_id: old.key_id.clone(),
secret_key: old.secret_key.clone(),
name: crdt::LWW::migrate_from_raw(old.name_timestamp, old.name.clone()),
deleted: crdt::Bool::new(old.deleted),
authorized_buckets: crdt::LWWMap::new(),
};
for ab in old.authorized_buckets() {
let it = crdt::LWWMap::migrate_from_raw_item(
ab.bucket.clone(),
ab.timestamp,
PermissionSet {
allow_read: ab.allow_read,
allow_write: ab.allow_write,
},
);
new.authorized_buckets.merge(&it);
}
Some(new)
}
}

52
src/model/object_table.rs
View file

@ -11,8 +11,6 @@ use garage_table::*;
use crate::version_table::*;
use model010::object_table as prev;
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct Object {
// Primary key
@ -226,55 +224,5 @@ impl TableSchema for ObjectTable {
let deleted = !entry.versions.iter().any(|v| v.is_data());
filter.apply(deleted)
}
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
let old = match rmp_serde::decode::from_read_ref::<_, prev::Object>(bytes) {
Ok(x) => x,
Err(_) => return None,
};
let new_v = old
.versions()
.iter()
.map(migrate_version)
.collect::<Vec<_>>();
let new = Object::new(old.bucket.clone(), old.key.clone(), new_v);
Some(new)
}
}
fn migrate_version(old: &prev::ObjectVersion) -> ObjectVersion {
let headers = ObjectVersionHeaders {
content_type: old.mime_type.clone(),
other: BTreeMap::new(),
};
let meta = ObjectVersionMeta {
headers: headers.clone(),
size: old.size,
etag: "".to_string(),
};
let state = match old.state {
prev::ObjectVersionState::Uploading => ObjectVersionState::Uploading(headers),
prev::ObjectVersionState::Aborted => ObjectVersionState::Aborted,
prev::ObjectVersionState::Complete => match &old.data {
prev::ObjectVersionData::Uploading => ObjectVersionState::Uploading(headers),
prev::ObjectVersionData::DeleteMarker => {
ObjectVersionState::Complete(ObjectVersionData::DeleteMarker)
}
prev::ObjectVersionData::Inline(x) => {
ObjectVersionState::Complete(ObjectVersionData::Inline(meta, x.clone()))
}
prev::ObjectVersionData::FirstBlock(h) => {
let mut hash = [0u8; 32];
hash.copy_from_slice(h.as_ref());
ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, Hash::from(hash)))
}
},
};
let mut uuid = [0u8; 32];
uuid.copy_from_slice(old.uuid.as_ref());
ObjectVersion {
uuid: UUID::from(uuid),
timestamp: old.timestamp,
state,
}
}