Report available disk space in garage stats #487

Merged
lx merged 4 commits from report-disk-usage into main 2023-01-26 15:40:42 +00:00
23 changed files with 2754 additions and 2554 deletions
Showing only changes of commit 94d559ae00 - Show all commits

1527
Cargo.lock generated

File diff suppressed because it is too large Load diff

3523
Cargo.nix

File diff suppressed because it is too large Load diff

View file

@ -21,28 +21,28 @@ garage_util = { version = "0.8.1", path = "../util" }
garage_rpc = { version = "0.8.1", path = "../rpc" } garage_rpc = { version = "0.8.1", path = "../rpc" }
async-trait = "0.1.7" async-trait = "0.1.7"
base64 = "0.13" base64 = "0.21"
bytes = "1.0" bytes = "1.0"
chrono = "0.4" chrono = "0.4"
crypto-common = "0.1" crypto-common = "0.1"
err-derive = "0.3" err-derive = "0.3"
hex = "0.4" hex = "0.4"
hmac = "0.12" hmac = "0.12"
idna = "0.2" idna = "0.3"
tracing = "0.1.30" tracing = "0.1"
md-5 = "0.10" md-5 = "0.10"
nom = "7.1" nom = "7.1"
sha2 = "0.10" sha2 = "0.10"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
pin-project = "1.0.11" pin-project = "1.0.12"
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
tokio-stream = "0.1" tokio-stream = "0.1"
form_urlencoded = "1.0.0" form_urlencoded = "1.0.0"
http = "0.2" http = "0.2"
httpdate = "0.3" httpdate = "1.0"
http-range = "0.1" http-range = "0.1"
hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] }
multer = "2.0" multer = "2.0"
@ -52,7 +52,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_bytes = "0.11" serde_bytes = "0.11"
serde_json = "1.0" serde_json = "1.0"
quick-xml = { version = "0.21", features = [ "serialize" ] } quick-xml = { version = "0.21", features = [ "serialize" ] }
url = "2.1" url = "2.3"
opentelemetry = "0.17" opentelemetry = "0.17"
opentelemetry-prometheus = { version = "0.10", optional = true } opentelemetry-prometheus = { version = "0.10", optional = true }

View file

@ -1,5 +1,6 @@
use std::sync::Arc; use std::sync::Arc;
use base64::prelude::*;
use hyper::{Body, Request, Response, StatusCode}; use hyper::{Body, Request, Response, StatusCode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -31,9 +32,11 @@ pub async fn handle_insert_batch(
.transpose() .transpose()
.ok_or_bad_request("Invalid causality token")?; .ok_or_bad_request("Invalid causality token")?;
let v = match it.v { let v = match it.v {
Some(vs) => { Some(vs) => DvvsValue::Value(
DvvsValue::Value(base64::decode(vs).ok_or_bad_request("Invalid base64 value")?) BASE64_STANDARD
} .decode(vs)
.ok_or_bad_request("Invalid base64 value")?,
),
None => DvvsValue::Deleted, None => DvvsValue::Deleted,
}; };
items2.push((it.pk, it.sk, ct, v)); items2.push((it.pk, it.sk, ct, v));
@ -322,7 +325,7 @@ impl ReadBatchResponseItem {
.values() .values()
.iter() .iter()
.map(|v| match v { .map(|v| match v {
DvvsValue::Value(x) => Some(base64::encode(x)), DvvsValue::Value(x) => Some(BASE64_STANDARD.encode(x)),
DvvsValue::Deleted => None, DvvsValue::Deleted => None,
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();

View file

@ -1,5 +1,6 @@
use std::sync::Arc; use std::sync::Arc;
use base64::prelude::*;
use http::header; use http::header;
use hyper::{Body, Request, Response, StatusCode}; use hyper::{Body, Request, Response, StatusCode};
@ -81,7 +82,7 @@ impl ReturnFormat {
.iter() .iter()
.map(|v| match v { .map(|v| match v {
DvvsValue::Deleted => serde_json::Value::Null, DvvsValue::Deleted => serde_json::Value::Null,
DvvsValue::Value(v) => serde_json::Value::String(base64::encode(v)), DvvsValue::Value(v) => serde_json::Value::String(BASE64_STANDARD.encode(v)),
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let json_body = let json_body =

View file

@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::iter::{Iterator, Peekable}; use std::iter::{Iterator, Peekable};
use std::sync::Arc; use std::sync::Arc;
use base64::prelude::*;
use hyper::{Body, Response}; use hyper::{Body, Response};
use garage_util::data::*; use garage_util::data::*;
@ -129,11 +130,11 @@ pub async fn handle_list(
next_continuation_token: match (query.is_v2, &pagination) { next_continuation_token: match (query.is_v2, &pagination) {
(true, Some(RangeBegin::AfterKey { key })) => Some(s3_xml::Value(format!( (true, Some(RangeBegin::AfterKey { key })) => Some(s3_xml::Value(format!(
"]{}", "]{}",
base64::encode(key.as_bytes()) BASE64_STANDARD.encode(key.as_bytes())
))), ))),
(true, Some(RangeBegin::IncludingKey { key, .. })) => Some(s3_xml::Value(format!( (true, Some(RangeBegin::IncludingKey { key, .. })) => Some(s3_xml::Value(format!(
"[{}", "[{}",
base64::encode(key.as_bytes()) BASE64_STANDARD.encode(key.as_bytes())
))), ))),
_ => None, _ => None,
}, },
@ -583,14 +584,16 @@ impl ListObjectsQuery {
(Some(token), _) => match &token[..1] { (Some(token), _) => match &token[..1] {
"[" => Ok(RangeBegin::IncludingKey { "[" => Ok(RangeBegin::IncludingKey {
key: String::from_utf8( key: String::from_utf8(
base64::decode(token[1..].as_bytes()) BASE64_STANDARD
.decode(token[1..].as_bytes())
.ok_or_bad_request("Invalid continuation token")?, .ok_or_bad_request("Invalid continuation token")?,
)?, )?,
fallback_key: None, fallback_key: None,
}), }),
"]" => Ok(RangeBegin::AfterKey { "]" => Ok(RangeBegin::AfterKey {
key: String::from_utf8( key: String::from_utf8(
base64::decode(token[1..].as_bytes()) BASE64_STANDARD
.decode(token[1..].as_bytes())
.ok_or_bad_request("Invalid continuation token")?, .ok_or_bad_request("Invalid continuation token")?,
)?, )?,
}), }),

View file

@ -4,6 +4,7 @@ use std::ops::RangeInclusive;
use std::sync::Arc; use std::sync::Arc;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use base64::prelude::*;
use bytes::Bytes; use bytes::Bytes;
use chrono::{DateTime, Duration, Utc}; use chrono::{DateTime, Duration, Utc};
use futures::{Stream, StreamExt}; use futures::{Stream, StreamExt};
@ -138,7 +139,9 @@ pub async fn handle_post_object(
.get_existing_bucket(bucket_id) .get_existing_bucket(bucket_id)
.await?; .await?;
let decoded_policy = base64::decode(&policy).ok_or_bad_request("Invalid policy")?; let decoded_policy = BASE64_STANDARD
.decode(&policy)
.ok_or_bad_request("Invalid policy")?;
let decoded_policy: Policy = let decoded_policy: Policy =
serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?; serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?;

View file

@ -1,6 +1,7 @@
use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::Arc; use std::sync::Arc;
use base64::prelude::*;
use futures::prelude::*; use futures::prelude::*;
use hyper::body::{Body, Bytes}; use hyper::body::{Body, Bytes};
use hyper::header::{HeaderMap, HeaderValue}; use hyper::header::{HeaderMap, HeaderValue};
@ -207,7 +208,7 @@ fn ensure_checksum_matches(
} }
} }
if let Some(expected_md5) = content_md5 { if let Some(expected_md5) = content_md5 {
if expected_md5.trim_matches('"') != base64::encode(data_md5sum) { if expected_md5.trim_matches('"') != BASE64_STANDARD.encode(data_md5sum) {
return Err(Error::bad_request("Unable to validate content-md5")); return Err(Error::bad_request("Unable to validate content-md5"));
} else { } else {
trace!("Successfully validated content-md5"); trace!("Successfully validated content-md5");

View file

@ -25,11 +25,11 @@ arc-swap = "1.5"
async-trait = "0.1.7" async-trait = "0.1.7"
bytes = "1.0" bytes = "1.0"
hex = "0.4" hex = "0.4"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
async-compression = { version = "0.3", features = ["tokio", "zstd"] } async-compression = { version = "0.3", features = ["tokio", "zstd"] }
zstd = { version = "0.9", default-features = false } zstd = { version = "0.12", default-features = false }
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11" serde_bytes = "0.11"

View file

@ -19,18 +19,18 @@ required-features = ["cli"]
[dependencies] [dependencies]
err-derive = "0.3" err-derive = "0.3"
hexdump = "0.1" hexdump = "0.1"
tracing = "0.1.30" tracing = "0.1"
heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true } heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true }
rusqlite = { version = "0.27", optional = true } rusqlite = { version = "0.28", optional = true }
sled = { version = "0.34", optional = true } sled = { version = "0.34", optional = true }
# cli deps # cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } clap = { version = "4.1", optional = true, features = ["derive", "env"] }
pretty_env_logger = { version = "0.4", optional = true } pretty_env_logger = { version = "0.4", optional = true }
[dev-dependencies] [dev-dependencies]
mktemp = "0.4" mktemp = "0.5"
[features] [features]
default = [ "sled" ] default = [ "sled" ]

View file

@ -33,10 +33,10 @@ garage_web = { version = "0.8.1", path = "../web" }
backtrace = "0.3" backtrace = "0.3"
bytes = "1.0" bytes = "1.0"
bytesize = "1.1" bytesize = "1.1"
timeago = "0.3" timeago = "0.4"
parse_duration = "2.1" parse_duration = "2.1"
hex = "0.4" hex = "0.4"
tracing = { version = "0.1.30" } tracing = { version = "0.1" }
tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] }
rand = "0.8" rand = "0.8"
async-trait = "0.1.7" async-trait = "0.1.7"
@ -45,7 +45,7 @@ sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11" serde_bytes = "0.11"
structopt = { version = "0.3", default-features = false } structopt = { version = "0.3", default-features = false }
toml = "0.5" toml = "0.6"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
@ -69,7 +69,7 @@ sha2 = "0.10"
static_init = "1.0" static_init = "1.0"
assert-json-diff = "2.0" assert-json-diff = "2.0"
serde_json = "1.0" serde_json = "1.0"
base64 = "0.13" base64 = "0.21"
[features] [features]

View file

@ -3,6 +3,7 @@ use std::collections::HashMap;
use crate::common; use crate::common;
use assert_json_diff::assert_json_eq; use assert_json_diff::assert_json_eq;
use base64::prelude::*;
use serde_json::json; use serde_json::json;
use super::json_body; use super::json_body;
@ -36,12 +37,12 @@ async fn test_batch() {
{{"pk": "root", "sk": "d.2", "ct": null, "v": "{}"}}, {{"pk": "root", "sk": "d.2", "ct": null, "v": "{}"}},
{{"pk": "root", "sk": "e", "ct": null, "v": "{}"}} {{"pk": "root", "sk": "e", "ct": null, "v": "{}"}}
]"#, ]"#,
base64::encode(values.get(&"a").unwrap()), BASE64_STANDARD.encode(values.get(&"a").unwrap()),
base64::encode(values.get(&"b").unwrap()), BASE64_STANDARD.encode(values.get(&"b").unwrap()),
base64::encode(values.get(&"c").unwrap()), BASE64_STANDARD.encode(values.get(&"c").unwrap()),
base64::encode(values.get(&"d.1").unwrap()), BASE64_STANDARD.encode(values.get(&"d.1").unwrap()),
base64::encode(values.get(&"d.2").unwrap()), BASE64_STANDARD.encode(values.get(&"d.2").unwrap()),
base64::encode(values.get(&"e").unwrap()), BASE64_STANDARD.encode(values.get(&"e").unwrap()),
) )
.into_bytes(), .into_bytes(),
) )
@ -120,12 +121,12 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "a", "ct": ct.get("a").unwrap(), "v": [base64::encode(values.get("a").unwrap())]}, {"sk": "a", "ct": ct.get("a").unwrap(), "v": [BASE64_STANDARD.encode(values.get("a").unwrap())]},
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [base64::encode(values.get("b").unwrap())]}, {"sk": "b", "ct": ct.get("b").unwrap(), "v": [BASE64_STANDARD.encode(values.get("b").unwrap())]},
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap())]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]} {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -141,10 +142,10 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap())]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]} {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -160,9 +161,9 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -178,8 +179,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [base64::encode(values.get("b").unwrap())]}, {"sk": "b", "ct": ct.get("b").unwrap(), "v": [BASE64_STANDARD.encode(values.get("b").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -195,8 +196,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [base64::encode(values.get("b").unwrap())]}, {"sk": "b", "ct": ct.get("b").unwrap(), "v": [BASE64_STANDARD.encode(values.get("b").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -212,7 +213,7 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "a", "ct": ct.get("a").unwrap(), "v": [base64::encode(values.get("a").unwrap())]} {"sk": "a", "ct": ct.get("a").unwrap(), "v": [BASE64_STANDARD.encode(values.get("a").unwrap())]}
], ],
"more": true, "more": true,
"nextStart": "b", "nextStart": "b",
@ -228,8 +229,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap())]} {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -255,10 +256,10 @@ async fn test_batch() {
{{"pk": "root", "sk": "d.2", "ct": null, "v": "{}"}} {{"pk": "root", "sk": "d.2", "ct": null, "v": "{}"}}
]"#, ]"#,
ct.get(&"b").unwrap(), ct.get(&"b").unwrap(),
base64::encode(values.get(&"c'").unwrap()), BASE64_STANDARD.encode(values.get(&"c'").unwrap()),
ct.get(&"d.1").unwrap(), ct.get(&"d.1").unwrap(),
base64::encode(values.get(&"d.1'").unwrap()), BASE64_STANDARD.encode(values.get(&"d.1'").unwrap()),
base64::encode(values.get(&"d.2'").unwrap()), BASE64_STANDARD.encode(values.get(&"d.2'").unwrap()),
) )
.into_bytes(), .into_bytes(),
) )
@ -333,11 +334,11 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "a", "ct": ct.get("a").unwrap(), "v": [base64::encode(values.get("a").unwrap())]}, {"sk": "a", "ct": ct.get("a").unwrap(), "v": [BASE64_STANDARD.encode(values.get("a").unwrap())]},
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap()), base64::encode(values.get("c'").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap()), BASE64_STANDARD.encode(values.get("c'").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]} {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -353,8 +354,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -370,7 +371,7 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -386,7 +387,7 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
], ],
"more": true, "more": true,
"nextStart": "d.2", "nextStart": "d.2",
@ -402,7 +403,7 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -418,8 +419,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -435,8 +436,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -452,8 +453,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -563,8 +564,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap()), base64::encode(values.get("c'").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap()), BASE64_STANDARD.encode(values.get("c'").unwrap())]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]} {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -580,8 +581,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]}, {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]},
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap()), base64::encode(values.get("c'").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap()), BASE64_STANDARD.encode(values.get("c'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -599,10 +600,10 @@ async fn test_batch() {
"items": [ "items": [
{"sk": "a", "ct": ct.get("a").unwrap(), "v": [null]}, {"sk": "a", "ct": ct.get("a").unwrap(), "v": [null]},
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [null]}, {"sk": "b", "ct": ct.get("b").unwrap(), "v": [null]},
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap()), base64::encode(values.get("c'").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap()), BASE64_STANDARD.encode(values.get("c'").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [null]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [null]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [null]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [null]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]}, {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,

View file

@ -3,6 +3,7 @@ use std::time::Duration;
use crate::common; use crate::common;
use assert_json_diff::assert_json_eq; use assert_json_diff::assert_json_eq;
use base64::prelude::*;
use serde_json::json; use serde_json::json;
use super::json_body; use super::json_body;
@ -222,7 +223,10 @@ async fn test_items_and_indices() {
let res_json = json_body(res).await; let res_json = json_body(res).await;
assert_json_eq!( assert_json_eq!(
res_json, res_json,
[base64::encode(&content2), base64::encode(&content3)] [
BASE64_STANDARD.encode(&content2),
BASE64_STANDARD.encode(&content3)
]
); );
// ReadIndex -- now there should be some stuff // ReadIndex -- now there should be some stuff
@ -411,7 +415,7 @@ async fn test_item_return_format() {
"application/json" "application/json"
); );
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&single_value)])); assert_json_eq!(res_body, json!([BASE64_STANDARD.encode(&single_value)]));
// f2: binary // f2: binary
let res = ctx let res = ctx
@ -452,7 +456,7 @@ async fn test_item_return_format() {
"application/json" "application/json"
); );
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&single_value)])); assert_json_eq!(res_body, json!([BASE64_STANDARD.encode(&single_value)]));
// -- Test with a second, concurrent value -- // -- Test with a second, concurrent value --
let res = ctx let res = ctx
@ -488,8 +492,8 @@ async fn test_item_return_format() {
assert_json_eq!( assert_json_eq!(
res_body, res_body,
json!([ json!([
base64::encode(&single_value), BASE64_STANDARD.encode(&single_value),
base64::encode(&concurrent_value) BASE64_STANDARD.encode(&concurrent_value)
]) ])
); );
@ -512,8 +516,8 @@ async fn test_item_return_format() {
assert_json_eq!( assert_json_eq!(
res_body, res_body,
json!([ json!([
base64::encode(&single_value), BASE64_STANDARD.encode(&single_value),
base64::encode(&concurrent_value) BASE64_STANDARD.encode(&concurrent_value)
]) ])
); );
@ -550,8 +554,8 @@ async fn test_item_return_format() {
assert_json_eq!( assert_json_eq!(
res_body, res_body,
json!([ json!([
base64::encode(&single_value), BASE64_STANDARD.encode(&single_value),
base64::encode(&concurrent_value) BASE64_STANDARD.encode(&concurrent_value)
]) ])
); );
@ -587,7 +591,10 @@ async fn test_item_return_format() {
"application/json" "application/json"
); );
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&concurrent_value), null])); assert_json_eq!(
res_body,
json!([BASE64_STANDARD.encode(&concurrent_value), null])
);
// f1: not specified // f1: not specified
let res = ctx let res = ctx
@ -612,7 +619,10 @@ async fn test_item_return_format() {
.unwrap() .unwrap()
.to_string(); .to_string();
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&concurrent_value), null])); assert_json_eq!(
res_body,
json!([BASE64_STANDARD.encode(&concurrent_value), null])
);
// f2: binary // f2: binary
let res = ctx let res = ctx
@ -644,7 +654,10 @@ async fn test_item_return_format() {
"application/json" "application/json"
); );
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&concurrent_value), null])); assert_json_eq!(
res_body,
json!([BASE64_STANDARD.encode(&concurrent_value), null])
);
// -- Delete everything -- // -- Delete everything --
let res = ctx let res = ctx

View file

@ -9,19 +9,19 @@ repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
readme = "../../README.md" readme = "../../README.md"
[dependencies] [dependencies]
base64 = "0.13.0" base64 = "0.21"
http = "0.2.6" http = "0.2"
log = "0.4" log = "0.4"
rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"] } rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"] }
rusoto_credential = "0.48.0" rusoto_credential = "0.48.0"
rusoto_signature = "0.48.0" rusoto_signature = "0.48.0"
serde = "1.0.137" serde = "1.0"
serde_json = "1.0.81" serde_json = "1.0"
thiserror = "1.0.31" thiserror = "1.0"
tokio = "1.17.0" tokio = "1.24"
# cli deps # cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } clap = { version = "4.1", optional = true, features = ["derive", "env"] }
garage_util = { version = "0.8.1", path = "../util", optional = true } garage_util = { version = "0.8.1", path = "../util", optional = true }

View file

@ -22,13 +22,13 @@ garage_util = { version = "0.8.1", path = "../util" }
async-trait = "0.1.7" async-trait = "0.1.7"
arc-swap = "1.0" arc-swap = "1.0"
blake2 = "0.9" blake2 = "0.10"
err-derive = "0.3" err-derive = "0.3"
hex = "0.4" hex = "0.4"
base64 = "0.13" base64 = "0.21"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
zstd = { version = "0.9", default-features = false } zstd = { version = "0.12", default-features = false }
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11" serde_bytes = "0.11"

View file

@ -1,3 +1,5 @@
use base64::prelude::*;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::convert::TryInto; use std::convert::TryInto;
@ -41,11 +43,12 @@ impl CausalContext {
bytes.extend(u64::to_be_bytes(i)); bytes.extend(u64::to_be_bytes(i));
} }
base64::encode_config(bytes, base64::URL_SAFE_NO_PAD) BASE64_URL_SAFE_NO_PAD.encode(bytes)
} }
/// Parse from base64-encoded binary representation /// Parse from base64-encoded binary representation
pub fn parse(s: &str) -> Result<Self, String> { pub fn parse(s: &str) -> Result<Self, String> {
let bytes = base64::decode_config(s, base64::URL_SAFE_NO_PAD) let bytes = BASE64_URL_SAFE_NO_PAD
.decode(s)
.map_err(|e| format!("bad causality token base64: {}", e))?; .map_err(|e| format!("bad causality token base64: {}", e))?;
if bytes.len() % 16 != 8 || bytes.len() < 8 { if bytes.len() % 16 != 8 || bytes.len() < 8 {
return Err("bad causality token length".into()); return Err("bad causality token length".into());

View file

@ -173,9 +173,9 @@ impl Crdt for DvvsEntry {
impl PartitionKey for K2VItemPartition { impl PartitionKey for K2VItemPartition {
fn hash(&self) -> Hash { fn hash(&self) -> Hash {
use blake2::{Blake2b, Digest}; use blake2::{Blake2b512, Digest};
let mut hasher = Blake2b::new(); let mut hasher = Blake2b512::new();
hasher.update(self.bucket_id.as_slice()); hasher.update(self.bucket_id.as_slice());
hasher.update(self.partition_key.as_bytes()); hasher.update(self.partition_key.as_bytes());
let mut hash = [0u8; 32]; let mut hash = [0u8; 32];

View file

@ -20,7 +20,7 @@ arc-swap = "1.0"
bytes = "1.0" bytes = "1.0"
gethostname = "0.2" gethostname = "0.2"
hex = "0.4" hex = "0.4"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
systemstat = "0.2.3" systemstat = "0.2.3"

View file

@ -25,7 +25,7 @@ arc-swap = "1.0"
bytes = "1.0" bytes = "1.0"
hex = "0.4" hex = "0.4"
hexdump = "0.1" hexdump = "0.1"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }

View file

@ -18,7 +18,7 @@ garage_db = { version = "0.8.1", path = "../db" }
arc-swap = "1.0" arc-swap = "1.0"
async-trait = "0.1" async-trait = "0.1"
blake2 = "0.9" blake2 = "0.10"
bytes = "1.0" bytes = "1.0"
digest = "0.10" digest = "0.10"
err-derive = "0.3" err-derive = "0.3"
@ -27,7 +27,7 @@ hexdump = "0.1"
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] } xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
hex = "0.4" hex = "0.4"
lazy_static = "1.4" lazy_static = "1.4"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
sha2 = "0.10" sha2 = "0.10"
@ -35,7 +35,7 @@ chrono = "0.4"
rmp-serde = "0.15" rmp-serde = "0.15"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_json = "1.0" serde_json = "1.0"
toml = "0.5" toml = "0.6"
futures = "0.3" futures = "0.3"
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
@ -48,7 +48,7 @@ hyper = "0.14"
opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] } opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
[dev-dependencies] [dev-dependencies]
mktemp = "0.4" mktemp = "0.5"
[features] [features]
k2v = [] k2v = []

View file

@ -115,9 +115,9 @@ pub fn sha256sum(data: &[u8]) -> Hash {
/// Compute the blake2 of a slice /// Compute the blake2 of a slice
pub fn blake2sum(data: &[u8]) -> Hash { pub fn blake2sum(data: &[u8]) -> Hash {
use blake2::{Blake2b, Digest}; use blake2::{Blake2b512, Digest};
let mut hasher = Blake2b::new(); let mut hasher = Blake2b512::new();
hasher.update(data); hasher.update(data);
let mut hash = [0u8; 32]; let mut hash = [0u8; 32];
hash.copy_from_slice(&hasher.finalize()[..32]); hash.copy_from_slice(&hasher.finalize()[..32]);

View file

@ -25,6 +25,6 @@ pub fn increment_logical_clock_2(prev: u64, prev2: u64) -> u64 {
pub fn msec_to_rfc3339(msecs: u64) -> String { pub fn msec_to_rfc3339(msecs: u64) -> String {
let secs = msecs as i64 / 1000; let secs = msecs as i64 / 1000;
let nanos = (msecs as i64 % 1000) as u32 * 1_000_000; let nanos = (msecs as i64 % 1000) as u32 * 1_000_000;
let timestamp = Utc.timestamp(secs, nanos); let timestamp = Utc.timestamp_opt(secs, nanos).unwrap();
timestamp.to_rfc3339_opts(SecondsFormat::Millis, true) timestamp.to_rfc3339_opts(SecondsFormat::Millis, true)
} }

View file

@ -20,7 +20,7 @@ garage_util = { version = "0.8.1", path = "../util" }
garage_table = { version = "0.8.1", path = "../table" } garage_table = { version = "0.8.1", path = "../table" }
err-derive = "0.3" err-derive = "0.3"
tracing = "0.1.30" tracing = "0.1"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"
futures = "0.3" futures = "0.3"