Deuxfleurs/garage
44
56
Fork 73
Code Issues 100 Pull requests 14 Releases 30 Wiki Activity

Support STREAMING-AWS4-HMAC-SHA256-PAYLOAD (#64) #156

Merged
lx merged 11 commits from KokaKiwi/garage:aws4-payload-signing into main 2022-01-17 09:55:31 +00:00
Conversation 0 Commits 11 Files changed 13 +208 -65
8 changed files with 208 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 006e5cc231 - Show all commits

6
src/api/api_server.rs
View file

@ -15,7 +15,7 @@ use garage_model::garage::Garage;
use garage_model::key_table::Key; use garage_model::key_table::Key;
use crate::error::*; use crate::error::*;
use crate::signature::check_signature; use crate::signature::payload::check_payload_signature;
use crate::helpers::*; use crate::helpers::*;
use crate::s3_bucket::*; use crate::s3_bucket::*;
@ -90,7 +90,7 @@ async fn handler(
} }
async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> { async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
let (api_key, content_sha256) = check_signature(&garage, &req).await?; let (api_key, content_sha256) = check_payload_signature(&garage, &req).await?;
let authority = req let authority = req
.headers() .headers()
@ -176,7 +176,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
.await .await
} }
Endpoint::PutObject { key, .. } => { Endpoint::PutObject { key, .. } => {
handle_put(garage, req, bucket_id, &key, content_sha256).await handle_put(garage, req, bucket_id, &key, &api_key, content_sha256).await
} }
Endpoint::AbortMultipartUpload { key, upload_id, .. } => { Endpoint::AbortMultipartUpload { key, upload_id, .. } => {
handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await

5
src/api/s3_bucket.rs
View file

@ -120,7 +120,10 @@ pub async fn handle_create_bucket(
bucket_name: String, bucket_name: String,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
let body = hyper::body::to_bytes(req.into_body()).await?; let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?;
if let Some(content_sha256) = content_sha256 {
verify_signed_content(content_sha256, &body[..])?;
}
let cmd = let cmd =
parse_create_bucket_xml(&body[..]).ok_or_bad_request("Invalid create bucket XML query")?; parse_create_bucket_xml(&body[..]).ok_or_bad_request("Invalid create bucket XML query")?;

3
src/api/s3_delete.rs
View file

@ -80,6 +80,9 @@ pub async fn handle_delete_objects(
req: Request<Body>, req: Request<Body>,
content_sha256: Option<Hash>, content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
let content_sha256 =
content_sha256.ok_or_bad_request("Request content hash not signed, aborting.")?;
let body = hyper::body::to_bytes(req.into_body()).await?; let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?; verify_signed_content(content_sha256, &body[..])?;

117
src/api/s3_put.rs
View file

@ -2,6 +2,7 @@ use std::collections::{BTreeMap, VecDeque};
use std::pin::Pin; use std::pin::Pin;
use std::sync::Arc; use std::sync::Arc;
use chrono::{DateTime, NaiveDateTime, Utc};
use futures::task; use futures::task;
use futures::{prelude::*, TryFutureExt}; use futures::{prelude::*, TryFutureExt};
use hyper::body::{Body, Bytes}; use hyper::body::{Body, Bytes};
@ -17,18 +18,21 @@ use garage_util::time::*;
use garage_model::block::INLINE_THRESHOLD; use garage_model::block::INLINE_THRESHOLD;
use garage_model::block_ref_table::*; use garage_model::block_ref_table::*;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_model::key_table::Key;
use garage_model::object_table::*; use garage_model::object_table::*;
use garage_model::version_table::*; use garage_model::version_table::*;
use crate::error::*; use crate::error::*;
use crate::s3_xml; use crate::s3_xml;
use crate::signature::verify_signed_content; use crate::signature::LONG_DATETIME;
use crate::signature::{streaming::check_streaming_payload_signature, verify_signed_content};
pub async fn handle_put( pub async fn handle_put(
garage: Arc<Garage>, garage: Arc<Garage>,
req: Request<Body>, req: Request<Body>,
bucket_id: Uuid, bucket_id: Uuid,
key: &str, key: &str,
api_key: &Key,
mut content_sha256: Option<Hash>, mut content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
// Generate identity of new version // Generate identity of new version
@ -54,11 +58,29 @@ pub async fn handle_put(
}; };
// Parse body of uploaded file // Parse body of uploaded file
let body = req.into_body(); let (head, body) = req.into_parts();
let body = match payload_seed_signature { let body = if let Some(signature) = payload_seed_signature {
Some(_) => SignedPayloadChunker::new(body).map_err(Error::from).boxed(), let secret_key = &api_key
None => body.map_err(Error::from).boxed(), .state
.as_option()
.ok_or_internal_error("Deleted key state")?
.secret_key;
let date = head
.headers
.get("x-amz-date")
.ok_or_bad_request("Missing X-Amz-Date field")?
.to_str()?;
let date: NaiveDateTime =
NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?;
let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
SignedPayloadChunker::new(body, garage.clone(), date, secret_key, signature)
.map_err(Error::from)
.boxed()
} else {
body.map_err(Error::from).boxed()
}; };
let mut chunker = StreamChunker::new(body, garage.config.block_size); let mut chunker = StreamChunker::new(body, garage.config.block_size);
@ -287,7 +309,8 @@ async fn put_block_meta(
} }
mod payload { mod payload {
#[derive(Debug)] use std::fmt;
pub struct Header { pub struct Header {
pub size: usize, pub size: usize,
pub signature: Box<[u8]>, pub signature: Box<[u8]>,
@ -295,10 +318,10 @@ mod payload {
impl Header { impl Header {
pub fn parse(input: &[u8]) -> nom::IResult<&[u8], Self> { pub fn parse(input: &[u8]) -> nom::IResult<&[u8], Self> {
use nom::bytes::complete::tag; use nom::bytes::streaming::tag;
use nom::character::complete::hex_digit1; use nom::character::streaming::hex_digit1;
use nom::combinator::map_res; use nom::combinator::map_res;
use nom::number::complete::hex_u32; use nom::number::streaming::hex_u32;
let (input, size) = hex_u32(input)?; let (input, size) = hex_u32(input)?;
let (input, _) = tag(";")(input)?; let (input, _) = tag(";")(input)?;
@ -316,13 +339,29 @@ mod payload {
Ok((input, header)) Ok((input, header))
} }
} }
impl fmt::Debug for Header {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Header")
.field("size", &self.size)
.field("signature", &hex::encode(&self.signature))
.finish()
}
}
} }
enum SignedPayloadChunkerError<StreamE> { enum SignedPayloadChunkerError<StreamE> {
Stream(StreamE), Stream(StreamE),
InvalidSignature,
Message(String), Message(String),
} }
impl<StreamE> SignedPayloadChunkerError<StreamE> {
fn message(msg: &str) -> Self {
SignedPayloadChunkerError::Message(msg.into())
}
}
impl<StreamE> From<SignedPayloadChunkerError<StreamE>> for Error impl<StreamE> From<SignedPayloadChunkerError<StreamE>> for Error
where where
StreamE: Into<Error>, StreamE: Into<Error>,
@ -330,6 +369,9 @@ where
fn from(err: SignedPayloadChunkerError<StreamE>) -> Self { fn from(err: SignedPayloadChunkerError<StreamE>) -> Self {
match err { match err {
SignedPayloadChunkerError::Stream(e) => e.into(), SignedPayloadChunkerError::Stream(e) => e.into(),
SignedPayloadChunkerError::InvalidSignature => {
Error::BadRequest("Invalid payload signature".into())
}
SignedPayloadChunkerError::Message(e) => { SignedPayloadChunkerError::Message(e) => {
Error::BadRequest(format!("Chunk format error: {}", e)) Error::BadRequest(format!("Chunk format error: {}", e))
} }
@ -339,7 +381,7 @@ where
impl<E, I> From<nom::error::Error<I>> for SignedPayloadChunkerError<E> { impl<E, I> From<nom::error::Error<I>> for SignedPayloadChunkerError<E> {
fn from(err: nom::error::Error<I>) -> Self { fn from(err: nom::error::Error<I>) -> Self {
Self::Message(err.code.description().into()) Self::message(err.code.description())
} }
} }
@ -351,16 +393,30 @@ where
#[pin] #[pin]
stream: S, stream: S,
buf: bytes::BytesMut, buf: bytes::BytesMut,
garage: Arc<Garage>,
datetime: DateTime<Utc>,
secret_key: String,
previous_signature: Hash,
} }
impl<S, E> SignedPayloadChunker<S, E> impl<S, E> SignedPayloadChunker<S, E>
where where
S: Stream<Item = Result<Bytes, E>>, S: Stream<Item = Result<Bytes, E>>,
{ {
fn new(stream: S) -> Self { fn new(
stream: S,
garage: Arc<Garage>,
datetime: DateTime<Utc>,
secret_key: &str,
seed_signature: Hash,
) -> Self {
Self { Self {
stream, stream,
buf: bytes::BytesMut::new(), buf: bytes::BytesMut::new(),
garage,
datetime,
secret_key: secret_key.into(),
previous_signature: seed_signature,
} }
} }
} }
@ -377,18 +433,18 @@ where
) -> task::Poll<Option<Self::Item>> { ) -> task::Poll<Option<Self::Item>> {
use std::task::Poll; use std::task::Poll;
use nom::bytes::complete::{tag, take}; use nom::bytes::streaming::{tag, take};
let mut this = self.project(); let mut this = self.project();
macro_rules! parse_try { macro_rules! try_parse {
($expr:expr) => { ($expr:expr) => {
match $expr { match $expr {
Ok(value) => value, Ok(value) => Ok(value),
Err(nom::Err::Incomplete(_)) => continue, Err(nom::Err::Incomplete(_)) => continue,
Err(nom::Err::Error(e @ nom::error::Error { .. })) Err(nom::Err::Error(e @ nom::error::Error { .. }))
| Err(nom::Err::Failure(e)) => return Poll::Ready(Some(Err(e.into()))), | Err(nom::Err::Failure(e)) => Err(e),
} }?
}; };
} }
@ -409,7 +465,9 @@ where
let input: &[u8] = this.buf; let input: &[u8] = this.buf;
let (input, header) = parse_try!(payload::Header::parse(input)); let (input, header) = try_parse!(payload::Header::parse(input));
let signature = Hash::try_from(&*header.signature)
.ok_or_else(|| SignedPayloadChunkerError::message("Invalid signature"))?;
// 0-sized chunk is the last // 0-sized chunk is the last
if header.size == 0 { if header.size == 0 {
@ -417,12 +475,30 @@ where
return Poll::Ready(None); return Poll::Ready(None);
} }
let (input, data) = parse_try!(take(header.size)(input)); let (input, data) = try_parse!(take(header.size)(input));
let (input, _) = parse_try!(tag("\r\n")(input)); let (input, _) = try_parse!(tag("\r\n")(input));
let data = Bytes::from(data.to_vec()); let data = Bytes::from(data.to_vec());
let data_sha256sum = sha256sum(&data);
let expected_signature = check_streaming_payload_signature(
this.garage,
this.secret_key,
*this.datetime,
*this.previous_signature,
data_sha256sum,
)
.map_err(|e| {
SignedPayloadChunkerError::Message(format!("Could not build signature: {}", e))
})?;
if signature != expected_signature {
return Poll::Ready(Some(Err(SignedPayloadChunkerError::InvalidSignature)));
}
*this.buf = input.into(); *this.buf = input.into();
*this.previous_signature = signature;
return Poll::Ready(Some(Ok(data))); return Poll::Ready(Some(Ok(data)));
} }
} }
@ -611,6 +687,9 @@ pub async fn handle_complete_multipart_upload(
upload_id: &str, upload_id: &str,
content_sha256: Option<Hash>, content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
let content_sha256 =
content_sha256.ok_or_bad_request("Request content hash not signed, aborting.")?;
let body = hyper::body::to_bytes(req.into_body()).await?; let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?; verify_signed_content(content_sha256, &body[..])?;

3
src/api/s3_website.rs
View file

@ -43,6 +43,9 @@ pub async fn handle_put_website(
req: Request<Body>, req: Request<Body>,
content_sha256: Option<Hash>, content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
let content_sha256 =
content_sha256.ok_or_bad_request("Request content hash not signed, aborting.")?;
let body = hyper::body::to_bytes(req.into_body()).await?; let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?; verify_signed_content(content_sha256, &body[..])?;

43
src/api/signature/mod.rs Normal file
View file

@ -0,0 +1,43 @@
use chrono::{DateTime, Utc};
use hmac::{Hmac, Mac, NewMac};
use sha2::Sha256;
use garage_util::data::{sha256sum, Hash};
use crate::error::*;
pub mod payload;
pub mod streaming;
pub const SHORT_DATE: &str = "%Y%m%d";
pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
type HmacSha256 = Hmac<Sha256>;
pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> {
if expected_sha256 != sha256sum(body) {
return Err(Error::BadRequest(
"Request content hash does not match signed hash".to_string(),
));
}
Ok(())
}
fn signing_hmac(
datetime: &DateTime<Utc>,
secret_key: &str,
region: &str,
service: &str,
) -> Result<HmacSha256, crypto_mac::InvalidKeyLength> {
let secret = String::from("AWS4") + secret_key;
let mut date_hmac = HmacSha256::new_varkey(secret.as_bytes())?;
date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes());
let mut region_hmac = HmacSha256::new_varkey(&date_hmac.finalize().into_bytes())?;
region_hmac.update(region.as_bytes());
let mut service_hmac = HmacSha256::new_varkey(&region_hmac.finalize().into_bytes())?;
service_hmac.update(service.as_bytes());
let mut signing_hmac = HmacSha256::new_varkey(&service_hmac.finalize().into_bytes())?;
signing_hmac.update(b"aws4_request");
let hmac = HmacSha256::new_varkey(&signing_hmac.finalize().into_bytes())?;
Ok(hmac)
}

44
src/api/signature.rs → src/api/signature/payload.rs
View file

@ -1,25 +1,23 @@
use std::collections::HashMap; use std::collections::HashMap;
use chrono::{DateTime, Duration, NaiveDateTime, Utc}; use chrono::{DateTime, Duration, NaiveDateTime, Utc};
use hmac::{Hmac, Mac, NewMac}; use hmac::Mac;
use hyper::{Body, Method, Request}; use hyper::{Body, Method, Request};
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use garage_table::*; use garage_table::*;
use garage_util::data::{sha256sum, Hash}; use garage_util::data::Hash;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_model::key_table::*; use garage_model::key_table::*;
use super::signing_hmac;
use super::{LONG_DATETIME, SHORT_DATE};
use crate::encoding::uri_encode; use crate::encoding::uri_encode;
use crate::error::*; use crate::error::*;
const SHORT_DATE: &str = "%Y%m%d"; pub async fn check_payload_signature(
const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ";
type HmacSha256 = Hmac<Sha256>;
pub async fn check_signature(
garage: &Garage, garage: &Garage,
request: &Request<Body>, request: &Request<Body>,
) -> Result<(Key, Option<Hash>), Error> { ) -> Result<(Key, Option<Hash>), Error> {
@ -222,25 +220,6 @@ fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &
.join("\n") .join("\n")
} }
fn signing_hmac(
datetime: &DateTime<Utc>,
secret_key: &str,
region: &str,
service: &str,
) -> Result<HmacSha256, crypto_mac::InvalidKeyLength> {
let secret = String::from("AWS4") + secret_key;
let mut date_hmac = HmacSha256::new_varkey(secret.as_bytes())?;
date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes());
let mut region_hmac = HmacSha256::new_varkey(&date_hmac.finalize().into_bytes())?;
region_hmac.update(region.as_bytes());
let mut service_hmac = HmacSha256::new_varkey(&region_hmac.finalize().into_bytes())?;
service_hmac.update(service.as_bytes());
let mut signing_hmac = HmacSha256::new_varkey(&service_hmac.finalize().into_bytes())?;
signing_hmac.update(b"aws4_request");
let hmac = HmacSha256::new_varkey(&signing_hmac.finalize().into_bytes())?;
Ok(hmac)
}
fn canonical_request( fn canonical_request(
method: &Method, method: &Method,
url_path: &str, url_path: &str,
@ -288,14 +267,3 @@ fn canonical_query_string(uri: &hyper::Uri) -> String {
"".to_string() "".to_string()
} }
} }
pub fn verify_signed_content(content_sha256: Option<Hash>, body: &[u8]) -> Result<(), Error> {
let expected_sha256 =
content_sha256.ok_or_bad_request("Request content hash not signed, aborting.")?;
if expected_sha256 != sha256sum(body) {
return Err(Error::BadRequest(
"Request content hash does not match signed hash".to_string(),
));
}
Ok(())
}

44
src/api/signature/streaming.rs Normal file
View file

@ -0,0 +1,44 @@
use chrono::{DateTime, Utc};
use garage_model::garage::Garage;
use garage_util::data::Hash;
use hmac::Mac;
use super::signing_hmac;
use super::{LONG_DATETIME, SHORT_DATE};
use crate::error::*;
/// Result of `sha256("")`
const EMPTY_STRING_HEX_DIGEST: &str =
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
pub fn check_streaming_payload_signature(
garage: &Garage,
secret_key: &str,
date: DateTime<Utc>,
previous_signature: Hash,
content_sha256: Hash,
) -> Result<Hash, Error> {
let scope = format!(
"{}/{}/s3/aws4_request",
date.format(SHORT_DATE),
garage.config.s3_api.s3_region
);
let string_to_sign = [
"AWS4-HMAC-SHA256-PAYLOAD",
&date.format(LONG_DATETIME).to_string(),
&scope,
&hex::encode(previous_signature),
EMPTY_STRING_HEX_DIGEST,
&hex::encode(content_sha256),
]
.join("\n");
let mut hmac = signing_hmac(&date, secret_key, &garage.config.s3_api.s3_region, "s3")
.ok_or_internal_error("Unable to build signing HMAC")?;
hmac.update(string_to_sign.as_bytes());
Hash::try_from(&hmac.finalize().into_bytes()).ok_or_bad_request("Invalid signature")
}