Deuxfleurs/garage
48
56
Fork 73
Code Issues 100 Pull requests 13 Releases 30 Wiki Activity

Support STREAMING-AWS4-HMAC-SHA256-PAYLOAD (#64) #156

Merged
lx merged 11 commits from KokaKiwi/garage:aws4-payload-signing into main 2022-01-17 09:55:31 +00:00
Conversation 0 Commits 11 Files changed 13 +267 -11
5 changed files with 267 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit f1bfc939aa - Show all commits

39
Cargo.lock generated
View file

@ -433,7 +433,9 @@ dependencies = [
"idna",
"log",
"md-5",
"nom",
"percent-encoding",
"pin-project",
"quick-xml",
"roxmltree",
"serde",
@ -967,6 +969,12 @@ dependencies = [
"autocfg",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "mio"
version = "0.7.13"
@ -1011,6 +1019,17 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "nom"
version = "7.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109"
dependencies = [
"memchr",
"minimal-lexical",
"version_check",
]
[[package]]
name = "ntapi"
version = "0.3.6"
@ -1092,6 +1111,26 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pin-project"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pin-project-lite"
version = "0.2.7"

53
Cargo.nix
View file

@ -671,7 +671,9 @@ in
idna = rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.2.3" { inherit profileName; };
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.9.1" { inherit profileName; };
nom = rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.0" { inherit profileName; };
percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.8" { inherit profileName; };
quick_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.21.0" { inherit profileName; };
roxmltree = rustPackages."registry+https://github.com/rust-lang/crates.io-index".roxmltree."0.14.1" { inherit profileName; };
serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; };
@ -1321,6 +1323,16 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".minimal-lexical."0.2.1" = overridableMkRustCrate (profileName: rec {
name = "minimal-lexical";
version = "0.2.1";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"; };
features = builtins.concatLists [
[ "std" ]
];
});
"registry+https://github.com/rust-lang/crates.io-index".mio."0.7.13" = overridableMkRustCrate (profileName: rec {
name = "mio";
version = "0.7.13";
@ -1381,6 +1393,25 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".nom."7.1.0" = overridableMkRustCrate (profileName: rec {
name = "nom";
version = "7.1.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109"; };
features = builtins.concatLists [
[ "alloc" ]
[ "default" ]
[ "std" ]
];
dependencies = {
memchr = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; };
minimal_lexical = rustPackages."registry+https://github.com/rust-lang/crates.io-index".minimal-lexical."0.2.1" { inherit profileName; };
};
buildDependencies = {
version_check = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.3" { profileName = "__noProfile"; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.6" = overridableMkRustCrate (profileName: rec {
name = "ntapi";
version = "0.3.6";
@ -1490,6 +1521,28 @@ in
src = fetchCratesIo { inherit name version; sha256 = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"; };
});
"registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.8" = overridableMkRustCrate (profileName: rec {
name = "pin-project";
version = "1.0.8";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08"; };
dependencies = {
pin_project_internal = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-internal."1.0.8" { profileName = "__noProfile"; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".pin-project-internal."1.0.8" = overridableMkRustCrate (profileName: rec {
name = "pin-project-internal";
version = "1.0.8";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389"; };
dependencies = {
proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.30" { inherit profileName; };
quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.10" { inherit profileName; };
syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.80" { inherit profileName; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.7" = overridableMkRustCrate (profileName: rec {
name = "pin-project-lite";
version = "0.2.7";

2
src/api/Cargo.toml
View file

@ -28,10 +28,12 @@ hmac = "0.10"
idna = "0.2"
log = "0.4"
md-5 = "0.9"
nom = "7.1"
sha2 = "0.9"
futures = "0.3"
futures-util = "0.3"
pin-project = "1.0"
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
http = "0.2"

176
src/api/s3_put.rs
View file

@ -1,7 +1,9 @@
use std::collections::{BTreeMap, VecDeque};
use std::pin::Pin;
use std::sync::Arc;
use futures::prelude::*;
use futures::task;
use futures::{prelude::*, TryFutureExt};
use hyper::body::{Body, Bytes};
use hyper::{Request, Response};
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
@ -27,7 +29,7 @@ pub async fn handle_put(
req: Request<Body>,
bucket_id: Uuid,
key: &str,
content_sha256: Option<Hash>,
mut content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> {
// Generate identity of new version
let version_uuid = gen_uuid();
@ -41,10 +43,24 @@ pub async fn handle_put(
Some(x) => Some(x.to_str()?.to_string()),
None => None,
};
let payload_seed_signature = match req.headers().get("x-amz-content-sha256") {
Some(header) if header == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => {
let content_sha256 = content_sha256
.take()
.ok_or_bad_request("No signature provided")?;
Some(content_sha256)
}
_ => None,
};
// Parse body of uploaded file
let body = req.into_body();
let body = match payload_seed_signature {
Some(_) => SignedPayloadChunker::new(body).map_err(Error::from).boxed(),
None => body.map_err(Error::from).boxed(),
};
let mut chunker = StreamChunker::new(body, garage.config.block_size);
let first_block = chunker.next().await?.unwrap_or_default();
@ -179,7 +195,7 @@ fn ensure_checksum_matches(
Ok(())
}
async fn read_and_put_blocks<E: Into<GarageError>, S: Stream<Item = Result<Bytes, E>> + Unpin>(
async fn read_and_put_blocks<E: Into<Error>, S: Stream<Item = Result<Bytes, E>> + Unpin>(
garage: &Garage,
version: &Version,
part_number: u64,
@ -207,9 +223,9 @@ async fn read_and_put_blocks<E: Into<GarageError>, S: Stream<Item = Result<Bytes
loop {
let (_, _, next_block) = futures::try_join!(
put_curr_block,
put_curr_version_block,
chunker.next().map_err(Into::into),
put_curr_block.map_err(Into::into),
put_curr_version_block.map_err(Into::into),
chunker.next().map_err(Into::into)
)?;
if let Some(block) = next_block {
md5hasher.update(&block[..]);
@ -270,6 +286,152 @@ async fn put_block_meta(
Ok(())
}
mod payload {
#[derive(Debug)]
pub struct Header {
pub size: usize,
pub signature: Box<[u8]>,
}
impl Header {
pub fn parse(input: &[u8]) -> nom::IResult<&[u8], Self> {
use nom::bytes::complete::tag;
use nom::character::complete::hex_digit1;
use nom::combinator::map_res;
use nom::number::complete::hex_u32;
let (input, size) = hex_u32(input)?;
let (input, _) = tag(";")(input)?;
let (input, _) = tag("chunk-signature=")(input)?;
let (input, data) = map_res(hex_digit1, hex::decode)(input)?;
let (input, _) = tag("\r\n")(input)?;
lx marked this conversation as resolved Outdated
lx commented 2021-12-15 10:01:22 +00:00
Outdated
Review
Copy link

Is there a good reason to use Box<[u8]> instead of Vec<u8>? Or maybe it should be a fixed-size array ? (we have our own type for that: FixedBytes32, aka Hash)

Is there a good reason to use `Box<[u8]>` instead of `Vec<u8>`? Or maybe it should be a fixed-size array ? (we have our own type for that: `FixedBytes32`, aka `Hash`)
KokaKiwi marked this conversation as resolved Outdated
lx commented 2021-12-15 14:18:33 +00:00
Outdated
Review
Copy link

Is there a good reason to use a boxed slice instead of either 1/ a Vec<u8>, or 2/ a constant-sized slice ? (we have the type Hash which we use everywhere for 32-byte slices, this should be a good place for it too)

Is there a good reason to use a boxed slice instead of either 1/ a `Vec<u8>`, or 2/ a constant-sized slice ? (we have the type `Hash` which we use everywhere for 32-byte slices, this should be a good place for it too)
let header = Header {
size: size as usize,
signature: data.into_boxed_slice(),
};
Ok((input, header))
}
}
}
enum SignedPayloadChunkerError<StreamE> {
Stream(StreamE),
Message(String),
}
impl<StreamE> From<SignedPayloadChunkerError<StreamE>> for Error
where
StreamE: Into<Error>,
{
fn from(err: SignedPayloadChunkerError<StreamE>) -> Self {
match err {
SignedPayloadChunkerError::Stream(e) => e.into(),
SignedPayloadChunkerError::Message(e) => {
Error::BadRequest(format!("Chunk format error: {}", e))
}
}
}
}
impl<E, I> From<nom::error::Error<I>> for SignedPayloadChunkerError<E> {
lx marked this conversation as resolved Outdated
lx commented 2021-12-15 10:03:23 +00:00
Outdated
Review
Copy link

If signature was a Hash we would benefit from the custom impl Debug for Hash and we could just derive Debug for Header and not implement it manually

If signature was a `Hash` we would benefit from the custom `impl Debug for Hash` and we could just derive Debug for Header and not implement it manually
fn from(err: nom::error::Error<I>) -> Self {
KokaKiwi marked this conversation as resolved Outdated
lx commented 2021-12-15 14:19:19 +00:00
Outdated
Review
Copy link

If signature was of type Hash it would already have its own debug that shows (part of) the bytes in hex, so we could derive Debug instead of having a custom impl.

If `signature` was of type `Hash` it would already have its own debug that shows (part of) the bytes in hex, so we could derive Debug instead of having a custom impl.
Self::Message(err.code.description().into())
}
}
#[pin_project::pin_project]
struct SignedPayloadChunker<S, E>
where
S: Stream<Item = Result<Bytes, E>>,
{
#[pin]
stream: S,
buf: bytes::BytesMut,
}
impl<S, E> SignedPayloadChunker<S, E>
where
S: Stream<Item = Result<Bytes, E>>,
{
fn new(stream: S) -> Self {
Self {
stream,
buf: bytes::BytesMut::new(),
}
}
}
impl<S, E> Stream for SignedPayloadChunker<S, E>
where
S: Stream<Item = Result<Bytes, E>> + Unpin,
{
type Item = Result<Bytes, SignedPayloadChunkerError<E>>;
fn poll_next(
self: Pin<&mut Self>,
cx: &mut task::Context<'_>,
) -> task::Poll<Option<Self::Item>> {
use std::task::Poll;
use nom::bytes::complete::{tag, take};
let mut this = self.project();
lx marked this conversation as resolved Outdated
lx commented 2021-12-15 10:08:28 +00:00
Outdated
Review
Copy link

Why is this called a Chunker? As far as I understand, it doesn't give chunks that we control, it just gives a normal stream of (arrays of) bytes

Why is this called a Chunker? As far as I understand, it doesn't give chunks that we control, it just gives a normal stream of (arrays of) bytes
KokaKiwi marked this conversation as resolved Outdated
lx commented 2021-12-15 14:20:38 +00:00
Outdated
Review
Copy link

  1. Why is this called a Chunker ? The target here is not to create chunks in a specific way but just to process a stream which happens to be made of byte slices. Shouldn't it rather be called a SignedPayloadVerifier or StreamingSignedPayloadVerifier ?

  2. I think this struct and its associated error type (and also the payload submodule) should be moved in signature/streaming.rs

1. Why is this called a Chunker ? The target here is not to create chunks in a specific way but just to process a stream which happens to be made of byte slices. Shouldn't it rather be called a `SignedPayloadVerifier` or `StreamingSignedPayloadVerifier` ? 2. I think this struct and its associated error type (and also the `payload` submodule) should be moved in `signature/streaming.rs`
macro_rules! parse_try {
($expr:expr) => {
match $expr {
Ok(value) => value,
Err(nom::Err::Incomplete(_)) => continue,
Err(nom::Err::Error(e @ nom::error::Error { .. }))
| Err(nom::Err::Failure(e)) => return Poll::Ready(Some(Err(e.into()))),
}
};
}
loop {
match futures::ready!(this.stream.as_mut().poll_next(cx)) {
Some(Ok(bytes)) => {
this.buf.extend(bytes);
}
Some(Err(e)) => {
return Poll::Ready(Some(Err(SignedPayloadChunkerError::Stream(e))))
}
None => {
if this.buf.is_empty() {
return Poll::Ready(None);
}
}
}
let input: &[u8] = this.buf;
let (input, header) = parse_try!(payload::Header::parse(input));
// 0-sized chunk is the last
if header.size == 0 {
this.buf.clear();
return Poll::Ready(None);
}
let (input, data) = parse_try!(take(header.size)(input));
let (input, _) = parse_try!(tag("\r\n")(input));
let data = Bytes::from(data.to_vec());
*this.buf = input.into();
return Poll::Ready(Some(Ok(data)));
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.stream.size_hint()
}
}
struct StreamChunker<S: Stream<Item = Result<Bytes, E>>, E> {
stream: S,
read_all: bool,
@ -292,7 +454,7 @@ impl<S: Stream<Item = Result<Bytes, E>> + Unpin, E> StreamChunker<S, E> {
if let Some(block) = self.stream.next().await {
let bytes = block?;
trace!("Body next: {} bytes", bytes.len());
self.buf.extend(&bytes[..]);
self.buf.extend(bytes);
} else {
self.read_all = true;
}

8
src/api/signature.rs
View file

@ -97,13 +97,13 @@ pub async fn check_signature(
let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" {
None
} else if authorization.content_sha256 == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" {
let bytes = hex::decode(authorization.signature).ok_or_bad_request("Invalid signature")?;
Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid signature")?)
} else {
let bytes = hex::decode(authorization.content_sha256)
.ok_or_bad_request("Invalid content sha256 hash")?;
Some(
Hash::try_from(&bytes[..])
.ok_or_else(|| Error::BadRequest("Invalid content sha256 hash".to_string()))?,
)
Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid content sha256 hash")?)
};
Ok((key, content_sha256))