Deuxfleurs/garage
48
56
Fork 73
Code Issues 100 Pull requests 13 Releases 30 Wiki Activity

Abstract database behind generic interface and implement alternative drivers #322

Merged
lx merged 64 commits from db-abstraction into main 2022-06-08 08:01:56 +00:00
Conversation 0 Commits 64 Files changed 42 +46 -30
17 changed files with 46 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 16e0a655d0 - Show all commits

34
Cargo.lock generated
View file

@ -930,7 +930,7 @@ dependencies = [
"pretty_env_logger",
"prometheus",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde 1.1.0",
"serde",
"serde_bytes",
"serde_json",
@ -1002,7 +1002,7 @@ dependencies = [
"hex",
"opentelemetry",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde 1.1.0",
"serde",
"serde_bytes",
"tokio",
@ -1069,7 +1069,7 @@ dependencies = [
"netapp 0.4.4",
"opentelemetry",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde 1.1.0",
"serde",
"serde_bytes",
"tokio",
@ -1126,7 +1126,7 @@ dependencies = [
"opentelemetry",
"pnet_datalink",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde 1.1.0",
"schemars",
"serde",
"serde_bytes",
@ -1172,7 +1172,7 @@ dependencies = [
"hexdump",
"opentelemetry",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde 1.1.0",
"serde",
"serde_bytes",
"tokio",
@ -1220,7 +1220,7 @@ dependencies = [
"netapp 0.4.4",
"opentelemetry",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde 1.1.0",
"serde",
"serde_json",
"sha2",
@ -2249,6 +2249,12 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "paste"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
[[package]]
name = "pem"
version = "0.8.3"
@ -2638,12 +2644,13 @@ dependencies = [
[[package]]
name = "rmp"
version = "0.8.10"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f55e5fa1446c4d5dd1f5daeed2a4fe193071771a2636274d0d7a3b082aa7ad6"
checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f"
dependencies = [
"byteorder",
"num-traits",
"paste",
]
[[package]]
@ -2668,6 +2675,17 @@ dependencies = [
"serde",
]
[[package]]
name = "rmp-serde"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25786b0d276110195fa3d6f3f31299900cf71dfbd6c28450f3f58a0e7f7a347e"
dependencies = [
"byteorder",
"rmp",
"serde",
]
[[package]]
name = "roxmltree"
version = "0.14.1"

2
src/block/Cargo.toml
View file

@ -28,7 +28,7 @@ tracing = "0.1.30"
rand = "0.8"
zstd = { version = "0.9", default-features = false }
rmp-serde = "0.15"
rmp-serde = "1.1"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11"

2
src/garage/Cargo.toml
View file

@ -37,7 +37,7 @@ rand = "0.8"
async-trait = "0.1.7"
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
rmp-serde = "0.15"
rmp-serde = "1.1"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11"
structopt = { version = "0.3", default-features = false }

4
src/garage/repair.rs
View file

@ -81,7 +81,7 @@ impl Repair {
info!("repair_versions: {}", i);
}
let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
let version = rmp_serde::decode::from_slice::<Version>(&item_bytes)?;
if version.deleted.get() {
continue;
}
@ -133,7 +133,7 @@ impl Repair {
info!("repair_block_ref: {}", i);
}
let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
let block_ref = rmp_serde::decode::from_slice::<BlockRef>(&item_bytes)?;
if block_ref.deleted.get() {
continue;
}

2
src/model/Cargo.toml
View file

@ -31,7 +31,7 @@ tracing = "0.1.30"
rand = "0.8"
zstd = { version = "0.9", default-features = false }
rmp-serde = "0.15"
rmp-serde = "1.1"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11"

8
src/model/index_counter.rs
View file

@ -181,11 +181,9 @@ impl<T: CounterSchema> IndexCounter<T> {
let new_entry = self.local_counter.db().transaction(|tx| {
let mut entry = match tx.get(&self.local_counter, &tree_key[..])? {
Some(old_bytes) => {
rmp_serde::decode::from_read_ref::<_, LocalCounterEntry>(&old_bytes)
.map_err(Error::RmpDecode)
.map_err(db::TxError::Abort)?
}
Some(old_bytes) => rmp_serde::decode::from_slice::<LocalCounterEntry>(&old_bytes)
.map_err(Error::RmpDecode)
.map_err(db::TxError::Abort)?,
None => LocalCounterEntry {
values: BTreeMap::new(),
},

2
src/model/key_table.rs
View file

@ -175,7 +175,7 @@ impl TableSchema for KeyTable {
}
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
let old_k = rmp_serde::decode::from_read_ref::<_, old::Key>(bytes).ok()?;
let old_k = rmp_serde::decode::from_slice::<old::Key>(bytes).ok()?;
let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone());
let state = if old_k.deleted.get() {

2
src/model/migrate.rs
View file

@ -28,7 +28,7 @@ impl Migrate {
let mut old_buckets = vec![];
for res in tree.iter().map_err(GarageError::from)? {
let (_k, v) = res.map_err(GarageError::from)?;
let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..])
let bucket = rmp_serde::decode::from_slice::<old_bucket::Bucket>(&v[..])
.map_err(GarageError::from)?;
old_buckets.push(bucket);
}

2
src/model/s3/object_table.rs
View file

@ -270,7 +270,7 @@ impl TableSchema for ObjectTable {
}
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
let old_obj = rmp_serde::decode::from_read_ref::<_, old::Object>(bytes).ok()?;
let old_obj = rmp_serde::decode::from_slice::<old::Object>(bytes).ok()?;
Some(migrate_object(old_obj))
}
}

2
src/model/s3/version_table.rs
View file

@ -168,7 +168,7 @@ impl TableSchema for VersionTable {
}
fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
let old = rmp_serde::decode::from_read_ref::<_, old::Version>(bytes).ok()?;
let old = rmp_serde::decode::from_slice::<old::Version>(bytes).ok()?;
let blocks = old
.blocks

2
src/rpc/Cargo.toml
View file

@ -26,7 +26,7 @@ rand = "0.8"
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
async-trait = "0.1.7"
rmp-serde = "0.15"
rmp-serde = "1.1"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11"
serde_json = "1.0"

2
src/table/Cargo.toml
View file

@ -26,7 +26,7 @@ hexdump = "0.1"
tracing = "0.1.30"
rand = "0.8"
rmp-serde = "0.15"
rmp-serde = "1.1"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11"

2
src/table/data.rs
View file

@ -297,7 +297,7 @@ where
}
pub fn decode_entry(&self, bytes: &[u8]) -> Result<F::E, Error> {
match rmp_serde::decode::from_read_ref::<_, F::E>(bytes) {
match rmp_serde::decode::from_slice::<F::E>(bytes) {
Ok(x) => Ok(x),
Err(e) => match F::try_migrate(bytes) {
Some(x) => Ok(x),

2
src/table/merkle.rs
View file

@ -354,7 +354,7 @@ impl MerkleNode {
fn decode_opt(ent: Option<db::Value<'_>>) -> Result<Self, Error> {
match ent {
None => Ok(MerkleNode::Empty),
Some(v) => Ok(rmp_serde::decode::from_read_ref::<_, MerkleNode>(&v[..])?),
Some(v) => Ok(rmp_serde::decode::from_slice::<MerkleNode>(&v[..])?),
}
}

2
src/util/Cargo.toml
View file

@ -25,7 +25,7 @@ rand = "0.8"
sha2 = "0.9"
chrono = "0.4"
rmp-serde = "0.15"
rmp-serde = "1.1"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_json = "1.0"
toml = "0.5"

2
src/util/data.rs
View file

@ -151,7 +151,7 @@ where
let mut wr = Vec::with_capacity(128);
let mut se = rmp_serde::Serializer::new(&mut wr)
.with_struct_map()
.with_string_variants();
.with_binary();
val.serialize(&mut se)?;
Ok(wr)
}

4
src/util/persister.rs
View file

@ -33,7 +33,7 @@ where
let mut bytes = vec![];
file.read_to_end(&mut bytes)?;
let value = rmp_serde::decode::from_read_ref(&bytes[..])?;
let value = rmp_serde::decode::from_slice(&bytes[..])?;
Ok(value)
}
@ -57,7 +57,7 @@ where
let mut bytes = vec![];
file.read_to_end(&mut bytes).await?;
let value = rmp_serde::decode::from_read_ref(&bytes[..])?;
let value = rmp_serde::decode::from_slice(&bytes[..])?;
Ok(value)
}