Merge branch 'main' into report-disk-usage

This commit is contained in:
Alex 2023-01-26 16:20:41 +01:00
commit 94d559ae00
23 changed files with 2754 additions and 2554 deletions

1527
Cargo.lock generated

File diff suppressed because it is too large Load diff

3523
Cargo.nix

File diff suppressed because it is too large Load diff

View file

@ -21,28 +21,28 @@ garage_util = { version = "0.8.1", path = "../util" }
garage_rpc = { version = "0.8.1", path = "../rpc" } garage_rpc = { version = "0.8.1", path = "../rpc" }
async-trait = "0.1.7" async-trait = "0.1.7"
base64 = "0.13" base64 = "0.21"
bytes = "1.0" bytes = "1.0"
chrono = "0.4" chrono = "0.4"
crypto-common = "0.1" crypto-common = "0.1"
err-derive = "0.3" err-derive = "0.3"
hex = "0.4" hex = "0.4"
hmac = "0.12" hmac = "0.12"
idna = "0.2" idna = "0.3"
tracing = "0.1.30" tracing = "0.1"
md-5 = "0.10" md-5 = "0.10"
nom = "7.1" nom = "7.1"
sha2 = "0.10" sha2 = "0.10"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
pin-project = "1.0.11" pin-project = "1.0.12"
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
tokio-stream = "0.1" tokio-stream = "0.1"
form_urlencoded = "1.0.0" form_urlencoded = "1.0.0"
http = "0.2" http = "0.2"
httpdate = "0.3" httpdate = "1.0"
http-range = "0.1" http-range = "0.1"
hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] }
multer = "2.0" multer = "2.0"
@ -52,7 +52,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_bytes = "0.11" serde_bytes = "0.11"
serde_json = "1.0" serde_json = "1.0"
quick-xml = { version = "0.21", features = [ "serialize" ] } quick-xml = { version = "0.21", features = [ "serialize" ] }
url = "2.1" url = "2.3"
opentelemetry = "0.17" opentelemetry = "0.17"
opentelemetry-prometheus = { version = "0.10", optional = true } opentelemetry-prometheus = { version = "0.10", optional = true }

View file

@ -1,5 +1,6 @@
use std::sync::Arc; use std::sync::Arc;
use base64::prelude::*;
use hyper::{Body, Request, Response, StatusCode}; use hyper::{Body, Request, Response, StatusCode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -31,9 +32,11 @@ pub async fn handle_insert_batch(
.transpose() .transpose()
.ok_or_bad_request("Invalid causality token")?; .ok_or_bad_request("Invalid causality token")?;
let v = match it.v { let v = match it.v {
Some(vs) => { Some(vs) => DvvsValue::Value(
DvvsValue::Value(base64::decode(vs).ok_or_bad_request("Invalid base64 value")?) BASE64_STANDARD
} .decode(vs)
.ok_or_bad_request("Invalid base64 value")?,
),
None => DvvsValue::Deleted, None => DvvsValue::Deleted,
}; };
items2.push((it.pk, it.sk, ct, v)); items2.push((it.pk, it.sk, ct, v));
@ -322,7 +325,7 @@ impl ReadBatchResponseItem {
.values() .values()
.iter() .iter()
.map(|v| match v { .map(|v| match v {
DvvsValue::Value(x) => Some(base64::encode(x)), DvvsValue::Value(x) => Some(BASE64_STANDARD.encode(x)),
DvvsValue::Deleted => None, DvvsValue::Deleted => None,
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();

View file

@ -1,5 +1,6 @@
use std::sync::Arc; use std::sync::Arc;
use base64::prelude::*;
use http::header; use http::header;
use hyper::{Body, Request, Response, StatusCode}; use hyper::{Body, Request, Response, StatusCode};
@ -81,7 +82,7 @@ impl ReturnFormat {
.iter() .iter()
.map(|v| match v { .map(|v| match v {
DvvsValue::Deleted => serde_json::Value::Null, DvvsValue::Deleted => serde_json::Value::Null,
DvvsValue::Value(v) => serde_json::Value::String(base64::encode(v)), DvvsValue::Value(v) => serde_json::Value::String(BASE64_STANDARD.encode(v)),
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let json_body = let json_body =

View file

@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::iter::{Iterator, Peekable}; use std::iter::{Iterator, Peekable};
use std::sync::Arc; use std::sync::Arc;
use base64::prelude::*;
use hyper::{Body, Response}; use hyper::{Body, Response};
use garage_util::data::*; use garage_util::data::*;
@ -129,11 +130,11 @@ pub async fn handle_list(
next_continuation_token: match (query.is_v2, &pagination) { next_continuation_token: match (query.is_v2, &pagination) {
(true, Some(RangeBegin::AfterKey { key })) => Some(s3_xml::Value(format!( (true, Some(RangeBegin::AfterKey { key })) => Some(s3_xml::Value(format!(
"]{}", "]{}",
base64::encode(key.as_bytes()) BASE64_STANDARD.encode(key.as_bytes())
))), ))),
(true, Some(RangeBegin::IncludingKey { key, .. })) => Some(s3_xml::Value(format!( (true, Some(RangeBegin::IncludingKey { key, .. })) => Some(s3_xml::Value(format!(
"[{}", "[{}",
base64::encode(key.as_bytes()) BASE64_STANDARD.encode(key.as_bytes())
))), ))),
_ => None, _ => None,
}, },
@ -583,14 +584,16 @@ impl ListObjectsQuery {
(Some(token), _) => match &token[..1] { (Some(token), _) => match &token[..1] {
"[" => Ok(RangeBegin::IncludingKey { "[" => Ok(RangeBegin::IncludingKey {
key: String::from_utf8( key: String::from_utf8(
base64::decode(token[1..].as_bytes()) BASE64_STANDARD
.decode(token[1..].as_bytes())
.ok_or_bad_request("Invalid continuation token")?, .ok_or_bad_request("Invalid continuation token")?,
)?, )?,
fallback_key: None, fallback_key: None,
}), }),
"]" => Ok(RangeBegin::AfterKey { "]" => Ok(RangeBegin::AfterKey {
key: String::from_utf8( key: String::from_utf8(
base64::decode(token[1..].as_bytes()) BASE64_STANDARD
.decode(token[1..].as_bytes())
.ok_or_bad_request("Invalid continuation token")?, .ok_or_bad_request("Invalid continuation token")?,
)?, )?,
}), }),

View file

@ -4,6 +4,7 @@ use std::ops::RangeInclusive;
use std::sync::Arc; use std::sync::Arc;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use base64::prelude::*;
use bytes::Bytes; use bytes::Bytes;
use chrono::{DateTime, Duration, Utc}; use chrono::{DateTime, Duration, Utc};
use futures::{Stream, StreamExt}; use futures::{Stream, StreamExt};
@ -138,7 +139,9 @@ pub async fn handle_post_object(
.get_existing_bucket(bucket_id) .get_existing_bucket(bucket_id)
.await?; .await?;
let decoded_policy = base64::decode(&policy).ok_or_bad_request("Invalid policy")?; let decoded_policy = BASE64_STANDARD
.decode(&policy)
.ok_or_bad_request("Invalid policy")?;
let decoded_policy: Policy = let decoded_policy: Policy =
serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?; serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?;

View file

@ -1,6 +1,7 @@
use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::Arc; use std::sync::Arc;
use base64::prelude::*;
use futures::prelude::*; use futures::prelude::*;
use hyper::body::{Body, Bytes}; use hyper::body::{Body, Bytes};
use hyper::header::{HeaderMap, HeaderValue}; use hyper::header::{HeaderMap, HeaderValue};
@ -207,7 +208,7 @@ fn ensure_checksum_matches(
} }
} }
if let Some(expected_md5) = content_md5 { if let Some(expected_md5) = content_md5 {
if expected_md5.trim_matches('"') != base64::encode(data_md5sum) { if expected_md5.trim_matches('"') != BASE64_STANDARD.encode(data_md5sum) {
return Err(Error::bad_request("Unable to validate content-md5")); return Err(Error::bad_request("Unable to validate content-md5"));
} else { } else {
trace!("Successfully validated content-md5"); trace!("Successfully validated content-md5");

View file

@ -25,11 +25,11 @@ arc-swap = "1.5"
async-trait = "0.1.7" async-trait = "0.1.7"
bytes = "1.0" bytes = "1.0"
hex = "0.4" hex = "0.4"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
async-compression = { version = "0.3", features = ["tokio", "zstd"] } async-compression = { version = "0.3", features = ["tokio", "zstd"] }
zstd = { version = "0.9", default-features = false } zstd = { version = "0.12", default-features = false }
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11" serde_bytes = "0.11"

View file

@ -19,18 +19,18 @@ required-features = ["cli"]
[dependencies] [dependencies]
err-derive = "0.3" err-derive = "0.3"
hexdump = "0.1" hexdump = "0.1"
tracing = "0.1.30" tracing = "0.1"
heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true } heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true }
rusqlite = { version = "0.27", optional = true } rusqlite = { version = "0.28", optional = true }
sled = { version = "0.34", optional = true } sled = { version = "0.34", optional = true }
# cli deps # cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } clap = { version = "4.1", optional = true, features = ["derive", "env"] }
pretty_env_logger = { version = "0.4", optional = true } pretty_env_logger = { version = "0.4", optional = true }
[dev-dependencies] [dev-dependencies]
mktemp = "0.4" mktemp = "0.5"
[features] [features]
default = [ "sled" ] default = [ "sled" ]

View file

@ -33,10 +33,10 @@ garage_web = { version = "0.8.1", path = "../web" }
backtrace = "0.3" backtrace = "0.3"
bytes = "1.0" bytes = "1.0"
bytesize = "1.1" bytesize = "1.1"
timeago = "0.3" timeago = "0.4"
parse_duration = "2.1" parse_duration = "2.1"
hex = "0.4" hex = "0.4"
tracing = { version = "0.1.30" } tracing = { version = "0.1" }
tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] }
rand = "0.8" rand = "0.8"
async-trait = "0.1.7" async-trait = "0.1.7"
@ -45,7 +45,7 @@ sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11" serde_bytes = "0.11"
structopt = { version = "0.3", default-features = false } structopt = { version = "0.3", default-features = false }
toml = "0.5" toml = "0.6"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
@ -69,7 +69,7 @@ sha2 = "0.10"
static_init = "1.0" static_init = "1.0"
assert-json-diff = "2.0" assert-json-diff = "2.0"
serde_json = "1.0" serde_json = "1.0"
base64 = "0.13" base64 = "0.21"
[features] [features]

View file

@ -3,6 +3,7 @@ use std::collections::HashMap;
use crate::common; use crate::common;
use assert_json_diff::assert_json_eq; use assert_json_diff::assert_json_eq;
use base64::prelude::*;
use serde_json::json; use serde_json::json;
use super::json_body; use super::json_body;
@ -36,12 +37,12 @@ async fn test_batch() {
{{"pk": "root", "sk": "d.2", "ct": null, "v": "{}"}}, {{"pk": "root", "sk": "d.2", "ct": null, "v": "{}"}},
{{"pk": "root", "sk": "e", "ct": null, "v": "{}"}} {{"pk": "root", "sk": "e", "ct": null, "v": "{}"}}
]"#, ]"#,
base64::encode(values.get(&"a").unwrap()), BASE64_STANDARD.encode(values.get(&"a").unwrap()),
base64::encode(values.get(&"b").unwrap()), BASE64_STANDARD.encode(values.get(&"b").unwrap()),
base64::encode(values.get(&"c").unwrap()), BASE64_STANDARD.encode(values.get(&"c").unwrap()),
base64::encode(values.get(&"d.1").unwrap()), BASE64_STANDARD.encode(values.get(&"d.1").unwrap()),
base64::encode(values.get(&"d.2").unwrap()), BASE64_STANDARD.encode(values.get(&"d.2").unwrap()),
base64::encode(values.get(&"e").unwrap()), BASE64_STANDARD.encode(values.get(&"e").unwrap()),
) )
.into_bytes(), .into_bytes(),
) )
@ -120,12 +121,12 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "a", "ct": ct.get("a").unwrap(), "v": [base64::encode(values.get("a").unwrap())]}, {"sk": "a", "ct": ct.get("a").unwrap(), "v": [BASE64_STANDARD.encode(values.get("a").unwrap())]},
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [base64::encode(values.get("b").unwrap())]}, {"sk": "b", "ct": ct.get("b").unwrap(), "v": [BASE64_STANDARD.encode(values.get("b").unwrap())]},
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap())]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]} {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -141,10 +142,10 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap())]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]} {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -160,9 +161,9 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -178,8 +179,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [base64::encode(values.get("b").unwrap())]}, {"sk": "b", "ct": ct.get("b").unwrap(), "v": [BASE64_STANDARD.encode(values.get("b").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -195,8 +196,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap())]},
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [base64::encode(values.get("b").unwrap())]}, {"sk": "b", "ct": ct.get("b").unwrap(), "v": [BASE64_STANDARD.encode(values.get("b").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -212,7 +213,7 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "a", "ct": ct.get("a").unwrap(), "v": [base64::encode(values.get("a").unwrap())]} {"sk": "a", "ct": ct.get("a").unwrap(), "v": [BASE64_STANDARD.encode(values.get("a").unwrap())]}
], ],
"more": true, "more": true,
"nextStart": "b", "nextStart": "b",
@ -228,8 +229,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap())]} {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -255,10 +256,10 @@ async fn test_batch() {
{{"pk": "root", "sk": "d.2", "ct": null, "v": "{}"}} {{"pk": "root", "sk": "d.2", "ct": null, "v": "{}"}}
]"#, ]"#,
ct.get(&"b").unwrap(), ct.get(&"b").unwrap(),
base64::encode(values.get(&"c'").unwrap()), BASE64_STANDARD.encode(values.get(&"c'").unwrap()),
ct.get(&"d.1").unwrap(), ct.get(&"d.1").unwrap(),
base64::encode(values.get(&"d.1'").unwrap()), BASE64_STANDARD.encode(values.get(&"d.1'").unwrap()),
base64::encode(values.get(&"d.2'").unwrap()), BASE64_STANDARD.encode(values.get(&"d.2'").unwrap()),
) )
.into_bytes(), .into_bytes(),
) )
@ -333,11 +334,11 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "a", "ct": ct.get("a").unwrap(), "v": [base64::encode(values.get("a").unwrap())]}, {"sk": "a", "ct": ct.get("a").unwrap(), "v": [BASE64_STANDARD.encode(values.get("a").unwrap())]},
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap()), base64::encode(values.get("c'").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap()), BASE64_STANDARD.encode(values.get("c'").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]} {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -353,8 +354,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -370,7 +371,7 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -386,7 +387,7 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
], ],
"more": true, "more": true,
"nextStart": "d.2", "nextStart": "d.2",
@ -402,7 +403,7 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -418,8 +419,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -435,8 +436,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -452,8 +453,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [base64::encode(values.get("d.1'").unwrap())]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.1'").unwrap())]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [base64::encode(values.get("d.2").unwrap()), base64::encode(values.get("d.2'").unwrap())]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [BASE64_STANDARD.encode(values.get("d.2").unwrap()), BASE64_STANDARD.encode(values.get("d.2'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -563,8 +564,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap()), base64::encode(values.get("c'").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap()), BASE64_STANDARD.encode(values.get("c'").unwrap())]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]} {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]}
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -580,8 +581,8 @@ async fn test_batch() {
"tombstones": false, "tombstones": false,
"singleItem": false, "singleItem": false,
"items": [ "items": [
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]}, {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]},
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap()), base64::encode(values.get("c'").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap()), BASE64_STANDARD.encode(values.get("c'").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,
@ -599,10 +600,10 @@ async fn test_batch() {
"items": [ "items": [
{"sk": "a", "ct": ct.get("a").unwrap(), "v": [null]}, {"sk": "a", "ct": ct.get("a").unwrap(), "v": [null]},
{"sk": "b", "ct": ct.get("b").unwrap(), "v": [null]}, {"sk": "b", "ct": ct.get("b").unwrap(), "v": [null]},
{"sk": "c", "ct": ct.get("c").unwrap(), "v": [base64::encode(values.get("c").unwrap()), base64::encode(values.get("c'").unwrap())]}, {"sk": "c", "ct": ct.get("c").unwrap(), "v": [BASE64_STANDARD.encode(values.get("c").unwrap()), BASE64_STANDARD.encode(values.get("c'").unwrap())]},
{"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [null]}, {"sk": "d.1", "ct": ct.get("d.1").unwrap(), "v": [null]},
{"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [null]}, {"sk": "d.2", "ct": ct.get("d.2").unwrap(), "v": [null]},
{"sk": "e", "ct": ct.get("e").unwrap(), "v": [base64::encode(values.get("e").unwrap())]}, {"sk": "e", "ct": ct.get("e").unwrap(), "v": [BASE64_STANDARD.encode(values.get("e").unwrap())]},
], ],
"more": false, "more": false,
"nextStart": null, "nextStart": null,

View file

@ -3,6 +3,7 @@ use std::time::Duration;
use crate::common; use crate::common;
use assert_json_diff::assert_json_eq; use assert_json_diff::assert_json_eq;
use base64::prelude::*;
use serde_json::json; use serde_json::json;
use super::json_body; use super::json_body;
@ -222,7 +223,10 @@ async fn test_items_and_indices() {
let res_json = json_body(res).await; let res_json = json_body(res).await;
assert_json_eq!( assert_json_eq!(
res_json, res_json,
[base64::encode(&content2), base64::encode(&content3)] [
BASE64_STANDARD.encode(&content2),
BASE64_STANDARD.encode(&content3)
]
); );
// ReadIndex -- now there should be some stuff // ReadIndex -- now there should be some stuff
@ -411,7 +415,7 @@ async fn test_item_return_format() {
"application/json" "application/json"
); );
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&single_value)])); assert_json_eq!(res_body, json!([BASE64_STANDARD.encode(&single_value)]));
// f2: binary // f2: binary
let res = ctx let res = ctx
@ -452,7 +456,7 @@ async fn test_item_return_format() {
"application/json" "application/json"
); );
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&single_value)])); assert_json_eq!(res_body, json!([BASE64_STANDARD.encode(&single_value)]));
// -- Test with a second, concurrent value -- // -- Test with a second, concurrent value --
let res = ctx let res = ctx
@ -488,8 +492,8 @@ async fn test_item_return_format() {
assert_json_eq!( assert_json_eq!(
res_body, res_body,
json!([ json!([
base64::encode(&single_value), BASE64_STANDARD.encode(&single_value),
base64::encode(&concurrent_value) BASE64_STANDARD.encode(&concurrent_value)
]) ])
); );
@ -512,8 +516,8 @@ async fn test_item_return_format() {
assert_json_eq!( assert_json_eq!(
res_body, res_body,
json!([ json!([
base64::encode(&single_value), BASE64_STANDARD.encode(&single_value),
base64::encode(&concurrent_value) BASE64_STANDARD.encode(&concurrent_value)
]) ])
); );
@ -550,8 +554,8 @@ async fn test_item_return_format() {
assert_json_eq!( assert_json_eq!(
res_body, res_body,
json!([ json!([
base64::encode(&single_value), BASE64_STANDARD.encode(&single_value),
base64::encode(&concurrent_value) BASE64_STANDARD.encode(&concurrent_value)
]) ])
); );
@ -587,7 +591,10 @@ async fn test_item_return_format() {
"application/json" "application/json"
); );
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&concurrent_value), null])); assert_json_eq!(
res_body,
json!([BASE64_STANDARD.encode(&concurrent_value), null])
);
// f1: not specified // f1: not specified
let res = ctx let res = ctx
@ -612,7 +619,10 @@ async fn test_item_return_format() {
.unwrap() .unwrap()
.to_string(); .to_string();
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&concurrent_value), null])); assert_json_eq!(
res_body,
json!([BASE64_STANDARD.encode(&concurrent_value), null])
);
// f2: binary // f2: binary
let res = ctx let res = ctx
@ -644,7 +654,10 @@ async fn test_item_return_format() {
"application/json" "application/json"
); );
let res_body = json_body(res).await; let res_body = json_body(res).await;
assert_json_eq!(res_body, json!([base64::encode(&concurrent_value), null])); assert_json_eq!(
res_body,
json!([BASE64_STANDARD.encode(&concurrent_value), null])
);
// -- Delete everything -- // -- Delete everything --
let res = ctx let res = ctx

View file

@ -9,19 +9,19 @@ repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
readme = "../../README.md" readme = "../../README.md"
[dependencies] [dependencies]
base64 = "0.13.0" base64 = "0.21"
http = "0.2.6" http = "0.2"
log = "0.4" log = "0.4"
rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"] } rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"] }
rusoto_credential = "0.48.0" rusoto_credential = "0.48.0"
rusoto_signature = "0.48.0" rusoto_signature = "0.48.0"
serde = "1.0.137" serde = "1.0"
serde_json = "1.0.81" serde_json = "1.0"
thiserror = "1.0.31" thiserror = "1.0"
tokio = "1.17.0" tokio = "1.24"
# cli deps # cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } clap = { version = "4.1", optional = true, features = ["derive", "env"] }
garage_util = { version = "0.8.1", path = "../util", optional = true } garage_util = { version = "0.8.1", path = "../util", optional = true }

View file

@ -22,13 +22,13 @@ garage_util = { version = "0.8.1", path = "../util" }
async-trait = "0.1.7" async-trait = "0.1.7"
arc-swap = "1.0" arc-swap = "1.0"
blake2 = "0.9" blake2 = "0.10"
err-derive = "0.3" err-derive = "0.3"
hex = "0.4" hex = "0.4"
base64 = "0.13" base64 = "0.21"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
zstd = { version = "0.9", default-features = false } zstd = { version = "0.12", default-features = false }
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_bytes = "0.11" serde_bytes = "0.11"

View file

@ -1,3 +1,5 @@
use base64::prelude::*;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::convert::TryInto; use std::convert::TryInto;
@ -41,11 +43,12 @@ impl CausalContext {
bytes.extend(u64::to_be_bytes(i)); bytes.extend(u64::to_be_bytes(i));
} }
base64::encode_config(bytes, base64::URL_SAFE_NO_PAD) BASE64_URL_SAFE_NO_PAD.encode(bytes)
} }
/// Parse from base64-encoded binary representation /// Parse from base64-encoded binary representation
pub fn parse(s: &str) -> Result<Self, String> { pub fn parse(s: &str) -> Result<Self, String> {
let bytes = base64::decode_config(s, base64::URL_SAFE_NO_PAD) let bytes = BASE64_URL_SAFE_NO_PAD
.decode(s)
.map_err(|e| format!("bad causality token base64: {}", e))?; .map_err(|e| format!("bad causality token base64: {}", e))?;
if bytes.len() % 16 != 8 || bytes.len() < 8 { if bytes.len() % 16 != 8 || bytes.len() < 8 {
return Err("bad causality token length".into()); return Err("bad causality token length".into());

View file

@ -173,9 +173,9 @@ impl Crdt for DvvsEntry {
impl PartitionKey for K2VItemPartition { impl PartitionKey for K2VItemPartition {
fn hash(&self) -> Hash { fn hash(&self) -> Hash {
use blake2::{Blake2b, Digest}; use blake2::{Blake2b512, Digest};
let mut hasher = Blake2b::new(); let mut hasher = Blake2b512::new();
hasher.update(self.bucket_id.as_slice()); hasher.update(self.bucket_id.as_slice());
hasher.update(self.partition_key.as_bytes()); hasher.update(self.partition_key.as_bytes());
let mut hash = [0u8; 32]; let mut hash = [0u8; 32];

View file

@ -20,7 +20,7 @@ arc-swap = "1.0"
bytes = "1.0" bytes = "1.0"
gethostname = "0.2" gethostname = "0.2"
hex = "0.4" hex = "0.4"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
systemstat = "0.2.3" systemstat = "0.2.3"

View file

@ -25,7 +25,7 @@ arc-swap = "1.0"
bytes = "1.0" bytes = "1.0"
hex = "0.4" hex = "0.4"
hexdump = "0.1" hexdump = "0.1"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }

View file

@ -18,7 +18,7 @@ garage_db = { version = "0.8.1", path = "../db" }
arc-swap = "1.0" arc-swap = "1.0"
async-trait = "0.1" async-trait = "0.1"
blake2 = "0.9" blake2 = "0.10"
bytes = "1.0" bytes = "1.0"
digest = "0.10" digest = "0.10"
err-derive = "0.3" err-derive = "0.3"
@ -27,7 +27,7 @@ hexdump = "0.1"
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] } xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
hex = "0.4" hex = "0.4"
lazy_static = "1.4" lazy_static = "1.4"
tracing = "0.1.30" tracing = "0.1"
rand = "0.8" rand = "0.8"
sha2 = "0.10" sha2 = "0.10"
@ -35,7 +35,7 @@ chrono = "0.4"
rmp-serde = "0.15" rmp-serde = "0.15"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_json = "1.0" serde_json = "1.0"
toml = "0.5" toml = "0.6"
futures = "0.3" futures = "0.3"
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
@ -48,7 +48,7 @@ hyper = "0.14"
opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] } opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
[dev-dependencies] [dev-dependencies]
mktemp = "0.4" mktemp = "0.5"
[features] [features]
k2v = [] k2v = []

View file

@ -115,9 +115,9 @@ pub fn sha256sum(data: &[u8]) -> Hash {
/// Compute the blake2 of a slice /// Compute the blake2 of a slice
pub fn blake2sum(data: &[u8]) -> Hash { pub fn blake2sum(data: &[u8]) -> Hash {
use blake2::{Blake2b, Digest}; use blake2::{Blake2b512, Digest};
let mut hasher = Blake2b::new(); let mut hasher = Blake2b512::new();
hasher.update(data); hasher.update(data);
let mut hash = [0u8; 32]; let mut hash = [0u8; 32];
hash.copy_from_slice(&hasher.finalize()[..32]); hash.copy_from_slice(&hasher.finalize()[..32]);

View file

@ -25,6 +25,6 @@ pub fn increment_logical_clock_2(prev: u64, prev2: u64) -> u64 {
pub fn msec_to_rfc3339(msecs: u64) -> String { pub fn msec_to_rfc3339(msecs: u64) -> String {
let secs = msecs as i64 / 1000; let secs = msecs as i64 / 1000;
let nanos = (msecs as i64 % 1000) as u32 * 1_000_000; let nanos = (msecs as i64 % 1000) as u32 * 1_000_000;
let timestamp = Utc.timestamp(secs, nanos); let timestamp = Utc.timestamp_opt(secs, nanos).unwrap();
timestamp.to_rfc3339_opts(SecondsFormat::Millis, true) timestamp.to_rfc3339_opts(SecondsFormat::Millis, true)
} }

View file

@ -20,7 +20,7 @@ garage_util = { version = "0.8.1", path = "../util" }
garage_table = { version = "0.8.1", path = "../table" } garage_table = { version = "0.8.1", path = "../table" }
err-derive = "0.3" err-derive = "0.3"
tracing = "0.1.30" tracing = "0.1"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"
futures = "0.3" futures = "0.3"