From 81f1db6060ce260c8a10ab29aaedda280cabb1a6 Mon Sep 17 00:00:00 2001 From: KokaKiwi Date: Fri, 18 Feb 2022 17:05:19 +0100 Subject: [PATCH 1/5] garage_api: Handle streaming payload early in request handling --- src/api/api_server.rs | 57 ++++++++++++++++++++++++++++++++-- src/api/s3_put.rs | 55 +++----------------------------- src/api/signature/streaming.rs | 6 ++-- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/src/api/api_server.rs b/src/api/api_server.rs index a6bf5a44..e7b86d9e 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -1,7 +1,9 @@ use std::net::SocketAddr; use std::sync::Arc; +use chrono::{DateTime, NaiveDateTime, Utc}; use futures::future::Future; +use futures::prelude::*; use hyper::header; use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; @@ -24,7 +26,10 @@ use garage_model::key_table::Key; use garage_table::util::*; use crate::error::*; +use crate::signature::compute_scope; use crate::signature::payload::check_payload_signature; +use crate::signature::streaming::SignedPayloadStream; +use crate::signature::LONG_DATETIME; use crate::helpers::*; use crate::s3_bucket::*; @@ -158,7 +163,7 @@ async fn handler_stage2( let authority = req .headers() .get(header::HOST) - .ok_or_else(|| Error::BadRequest("HOST header required".to_owned()))? + .ok_or_bad_request("Host header required")? .to_str()?; let host = authority_to_host(authority)?; @@ -221,11 +226,57 @@ async fn handler_stage3( return handle_options_s3api(garage, &req, bucket_name).await; } - let (api_key, content_sha256) = check_payload_signature(&garage, &req).await?; + let (api_key, mut content_sha256) = check_payload_signature(&garage, &req).await?; let api_key = api_key.ok_or_else(|| { Error::Forbidden("Garage does not support anonymous access yet".to_string()) })?; + let req = match req.headers().get("x-amz-content-sha256") { + Some(header) if header == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => { + let signature = content_sha256 + .take() + .ok_or_bad_request("No signature provided")?; + + let secret_key = &api_key + .state + .as_option() + .ok_or_internal_error("Deleted key state")? + .secret_key; + + let date = req + .headers() + .get("x-amz-date") + .ok_or_bad_request("Missing X-Amz-Date field")? + .to_str()?; + let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME) + .ok_or_bad_request("Invalid date")?; + let date: DateTime = DateTime::from_utc(date, Utc); + + let scope = compute_scope(&date, &garage.config.s3_api.s3_region); + let signing_hmac = crate::signature::signing_hmac( + &date, + secret_key, + &garage.config.s3_api.s3_region, + "s3", + ) + .ok_or_internal_error("Unable to build signing HMAC")?; + + req.map(move |body| { + Body::wrap_stream( + SignedPayloadStream::new( + body.map_err(Error::from), + signing_hmac, + date, + &scope, + signature, + ) + .map_err(Error::from), + ) + }) + } + _ => req, + }; + let bucket_name = match bucket_name { None => return handle_request_without_bucket(garage, req, api_key, endpoint).await, Some(bucket) => bucket.to_string(), @@ -307,7 +358,7 @@ async fn handler_stage3( .await } Endpoint::PutObject { key } => { - handle_put(garage, req, bucket_id, &key, &api_key, content_sha256).await + handle_put(garage, req, bucket_id, &key, content_sha256).await } Endpoint::AbortMultipartUpload { key, upload_id } => { handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index 5735fd10..08e490ae 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -1,8 +1,7 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::sync::Arc; -use chrono::{DateTime, NaiveDateTime, Utc}; -use futures::{prelude::*, TryFutureExt}; +use futures::prelude::*; use hyper::body::{Body, Bytes}; use hyper::header::{HeaderMap, HeaderValue}; use hyper::{Request, Response}; @@ -17,23 +16,19 @@ use garage_util::time::*; use garage_model::block::INLINE_THRESHOLD; use garage_model::block_ref_table::*; use garage_model::garage::Garage; -use garage_model::key_table::Key; use garage_model::object_table::*; use garage_model::version_table::*; use crate::error::*; use crate::s3_xml; -use crate::signature::streaming::SignedPayloadStream; -use crate::signature::LONG_DATETIME; -use crate::signature::{compute_scope, verify_signed_content}; +use crate::signature::verify_signed_content; pub async fn handle_put( garage: Arc, req: Request, bucket_id: Uuid, key: &str, - api_key: &Key, - mut content_sha256: Option, + content_sha256: Option, ) -> Result, Error> { // Retrieve interesting headers from request let headers = get_headers(req.headers())?; @@ -43,52 +38,10 @@ pub async fn handle_put( Some(x) => Some(x.to_str()?.to_string()), None => None, }; - let payload_seed_signature = match req.headers().get("x-amz-content-sha256") { - Some(header) if header == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => { - let content_sha256 = content_sha256 - .take() - .ok_or_bad_request("No signature provided")?; - Some(content_sha256) - } - _ => None, - }; - // Parse body of uploaded file - let (head, body) = req.into_parts(); + let (_head, body) = req.into_parts(); let body = body.map_err(Error::from); - let body = if let Some(signature) = payload_seed_signature { - let secret_key = &api_key - .state - .as_option() - .ok_or_internal_error("Deleted key state")? - .secret_key; - - let date = head - .headers - .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field")? - .to_str()?; - let date: NaiveDateTime = - NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; - let date: DateTime = DateTime::from_utc(date, Utc); - - let scope = compute_scope(&date, &garage.config.s3_api.s3_region); - let signing_hmac = crate::signature::signing_hmac( - &date, - secret_key, - &garage.config.s3_api.s3_region, - "s3", - ) - .ok_or_internal_error("Unable to build signing HMAC")?; - - SignedPayloadStream::new(body, signing_hmac, date, &scope, signature)? - .map_err(Error::from) - .boxed() - } else { - body.boxed() - }; - save_stream( garage, headers, diff --git a/src/api/signature/streaming.rs b/src/api/signature/streaming.rs index b2dc1591..b04e5c72 100644 --- a/src/api/signature/streaming.rs +++ b/src/api/signature/streaming.rs @@ -164,15 +164,15 @@ where datetime: DateTime, scope: &str, seed_signature: Hash, - ) -> Result { - Ok(Self { + ) -> Self { + Self { stream, buf: bytes::BytesMut::new(), datetime, scope: scope.into(), signing_hmac, previous_signature: seed_signature, - }) + } } fn parse_next(input: &[u8]) -> nom::IResult<&[u8], SignedPayload, SignedPayloadStreamError> { -- 2.43.0 From 4458996ea50b9e4cb78150affe6facda7c58eb13 Mon Sep 17 00:00:00 2001 From: KokaKiwi Date: Fri, 18 Feb 2022 18:14:23 +0100 Subject: [PATCH 2/5] garage_api: Update streaming payload stream unit tests --- src/api/signature/streaming.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/signature/streaming.rs b/src/api/signature/streaming.rs index b04e5c72..969a45d6 100644 --- a/src/api/signature/streaming.rs +++ b/src/api/signature/streaming.rs @@ -305,7 +305,7 @@ mod tests { let seed_signature = Hash::default(); let mut stream = - SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature).unwrap(); + SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature); assert!(stream.try_next().await.is_err()); match stream.try_next().await { -- 2.43.0 From 6ea6a8d6d1bc37877668e86af709e512f8a1d532 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Fri, 11 Mar 2022 17:35:08 +0100 Subject: [PATCH 3/5] add test framework for arbitraty S3 requests and implement some basic test with it --- Cargo.lock | 56 ++-- Cargo.nix | 150 ++++++----- src/api/lib.rs | 3 +- src/api/signature/payload.rs | 14 +- src/garage/Cargo.toml | 5 +- src/garage/tests/common/custom_requester.rs | 280 ++++++++++++++++++++ src/garage/tests/common/garage.rs | 2 +- src/garage/tests/common/mod.rs | 11 +- src/garage/tests/lib.rs | 1 + src/garage/tests/multipart.rs | 2 +- src/garage/tests/objects.rs | 2 +- src/garage/tests/simple.rs | 2 +- src/garage/tests/streaming_signature.rs | 101 +++++++ src/garage/tests/website.rs | 2 +- 14 files changed, 516 insertions(+), 115 deletions(-) create mode 100644 src/garage/tests/common/custom_requester.rs create mode 100644 src/garage/tests/streaming_signature.rs diff --git a/Cargo.lock b/Cargo.lock index 1e050e16..6a638ef8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,9 +80,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-endpoint" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6" +checksum = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167" dependencies = [ "aws-smithy-http", "aws-types", @@ -93,9 +93,9 @@ dependencies = [ [[package]] name = "aws-http" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1" +checksum = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9" +checksum = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7" dependencies = [ "aws-endpoint", "aws-http", @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "aws-sig-auth" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc" +checksum = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6" dependencies = [ "aws-sigv4", "aws-smithy-eventstream", @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c" +checksum = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6" dependencies = [ "aws-smithy-eventstream", "aws-smithy-http", @@ -168,9 +168,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e" +checksum = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8" dependencies = [ "futures-util", "pin-project-lite", @@ -180,9 +180,9 @@ dependencies = [ [[package]] name = "aws-smithy-client" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb" +checksum = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -204,9 +204,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9" +checksum = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371" dependencies = [ "aws-smithy-types", "bytes 1.1.0", @@ -215,9 +215,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696" +checksum = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b" dependencies = [ "aws-smithy-eventstream", "aws-smithy-types", @@ -236,9 +236,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-tower" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d" +checksum = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8" dependencies = [ "aws-smithy-http", "bytes 1.1.0", @@ -251,9 +251,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f" +checksum = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2" dependencies = [ "itoa", "num-integer", @@ -263,9 +263,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4" +checksum = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f" dependencies = [ "thiserror", "xmlparser", @@ -273,11 +273,12 @@ dependencies = [ [[package]] name = "aws-types" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142" +checksum = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38" dependencies = [ "aws-smithy-async", + "aws-smithy-client", "aws-smithy-types", "rustc_version", "tracing", @@ -822,6 +823,7 @@ dependencies = [ "async-trait", "aws-sdk-s3", "bytes 1.1.0", + "chrono", "futures", "futures-util", "garage_admin", @@ -833,6 +835,7 @@ dependencies = [ "garage_web", "git-version", "hex", + "hmac", "http", "hyper", "kuska-sodiumoxide", @@ -842,6 +845,7 @@ dependencies = [ "rmp-serde 0.15.5", "serde", "serde_bytes", + "sha2", "sled", "static_init", "structopt", diff --git a/Cargo.nix b/Cargo.nix index c06887cd..7f95649f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -150,29 +150,29 @@ in src = fetchCratesIo { inherit name version; sha256 = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"; }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" = overridableMkRustCrate (profileName: rec { name = "aws-endpoint"; - version = "0.6.0"; + version = "0.8.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6"; }; + src = fetchCratesIo { inherit name version; sha256 = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167"; }; dependencies = { - aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; - aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; }; + aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; }; + aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; regex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.5" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" = overridableMkRustCrate (profileName: rec { name = "aws-http"; - version = "0.6.0"; + version = "0.8.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1"; }; + src = fetchCratesIo { inherit name version; sha256 = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506"; }; dependencies = { - aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; - aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; - aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; }; + aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; }; + aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; }; + aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; @@ -180,29 +180,29 @@ in }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" = overridableMkRustCrate (profileName: rec { name = "aws-sdk-s3"; - version = "0.6.0"; + version = "0.8.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9"; }; + src = fetchCratesIo { inherit name version; sha256 = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7"; }; features = builtins.concatLists [ [ "default" ] [ "rt-tokio" ] [ "rustls" ] ]; dependencies = { - aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" { inherit profileName; }; - aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" { inherit profileName; }; - aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" { inherit profileName; }; - aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; }; - aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; }; - aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" { inherit profileName; }; - aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; }; - aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; - aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; }; - aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; - aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" { inherit profileName; }; - aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; }; + aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" { inherit profileName; }; + aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" { inherit profileName; }; + aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" { inherit profileName; }; + aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; }; + aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; }; + aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; }; + aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; }; + aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; }; + aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; }; + aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; }; + aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" { inherit profileName; }; + aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md5."0.7.0" { inherit profileName; }; @@ -211,31 +211,31 @@ in }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" = overridableMkRustCrate (profileName: rec { name = "aws-sig-auth"; - version = "0.6.0"; + version = "0.8.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc"; }; + src = fetchCratesIo { inherit name version; sha256 = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6"; }; features = builtins.concatLists [ [ "aws-smithy-eventstream" ] [ "sign-eventstream" ] ]; dependencies = { - aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; }; - aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; }; - aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; - aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; }; + aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; }; + aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; }; + aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; }; + aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" = overridableMkRustCrate (profileName: rec { name = "aws-sigv4"; - version = "0.6.0"; + version = "0.8.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c"; }; + src = fetchCratesIo { inherit name version; sha256 = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"; }; features = builtins.concatLists [ [ "aws-smithy-eventstream" ] [ "bytes" ] @@ -247,8 +247,8 @@ in [ "sign-http" ] ]; dependencies = { - aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; }; - aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; + aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; }; + aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; form_urlencoded = rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; @@ -262,11 +262,11 @@ in }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" = overridableMkRustCrate (profileName: rec { name = "aws-smithy-async"; - version = "0.36.0"; + version = "0.38.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e"; }; + src = fetchCratesIo { inherit name version; sha256 = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8"; }; features = builtins.concatLists [ [ "rt-tokio" ] ]; @@ -278,11 +278,11 @@ in }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" = overridableMkRustCrate (profileName: rec { name = "aws-smithy-client"; - version = "0.36.0"; + version = "0.38.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb"; }; + src = fetchCratesIo { inherit name version; sha256 = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a"; }; features = builtins.concatLists [ [ "client-hyper" ] [ "hyper" ] @@ -292,10 +292,10 @@ in [ "rustls" ] ]; dependencies = { - aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; }; - aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; - aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; }; - aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; + aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; }; + aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; }; + aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; }; + aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; fastrand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.7.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; @@ -311,23 +311,23 @@ in }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" = overridableMkRustCrate (profileName: rec { name = "aws-smithy-eventstream"; - version = "0.36.0"; + version = "0.38.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9"; }; + src = fetchCratesIo { inherit name version; sha256 = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"; }; dependencies = { - aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; + aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; crc32fast = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }; }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" = overridableMkRustCrate (profileName: rec { name = "aws-smithy-http"; - version = "0.36.0"; + version = "0.38.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696"; }; + src = fetchCratesIo { inherit name version; sha256 = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"; }; features = builtins.concatLists [ [ "aws-smithy-eventstream" ] [ "event-stream" ] @@ -336,8 +336,8 @@ in [ "tokio-util" ] ]; dependencies = { - aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; }; - aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; + aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; }; + aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; bytes_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.1" { inherit profileName; }; futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; @@ -352,13 +352,13 @@ in }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" = overridableMkRustCrate (profileName: rec { name = "aws-smithy-http-tower"; - version = "0.36.0"; + version = "0.38.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d"; }; + src = fetchCratesIo { inherit name version; sha256 = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"; }; dependencies = { - aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; }; + aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; }; @@ -368,11 +368,11 @@ in }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" = overridableMkRustCrate (profileName: rec { name = "aws-smithy-types"; - version = "0.36.0"; + version = "0.38.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f"; }; + src = fetchCratesIo { inherit name version; sha256 = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2"; }; dependencies = { itoa = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; }; num_integer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }; @@ -381,25 +381,26 @@ in }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" = overridableMkRustCrate (profileName: rec { name = "aws-smithy-xml"; - version = "0.36.0"; + version = "0.38.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4"; }; + src = fetchCratesIo { inherit name version; sha256 = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f"; }; dependencies = { thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; xmlparser = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" { inherit profileName; }; }; }); - "registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" = overridableMkRustCrate (profileName: rec { name = "aws-types"; - version = "0.6.0"; + version = "0.8.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142"; }; + src = fetchCratesIo { inherit name version; sha256 = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38"; }; dependencies = { - aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; }; - aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; }; + aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; }; + aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; }; + aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; zeroize = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.3" { inherit profileName; }; }; @@ -1196,9 +1197,12 @@ in tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; devDependencies = { - aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" { inherit profileName; }; + aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" { inherit profileName; }; + chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; }; + hmac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.10.1" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; + sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; }; static_init = rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.2" { inherit profileName; }; }; }); diff --git a/src/api/lib.rs b/src/api/lib.rs index f865325e..de60ec53 100644 --- a/src/api/lib.rs +++ b/src/api/lib.rs @@ -10,7 +10,8 @@ mod encoding; mod api_server; pub use api_server::run_api_server; -mod signature; +/// This mode is public only to help testing. Don't expect stability here +pub mod signature; pub mod helpers; mod s3_bucket; diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 88b58922..88ec1f00 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -51,8 +51,7 @@ pub async fn check_payload_signature( let canonical_request = canonical_request( request.method(), - request.uri().path(), - &canonical_query_string(request.uri()), + request.uri(), &headers, &authorization.signed_headers, &authorization.content_sha256, @@ -215,7 +214,7 @@ fn parse_credential(cred: &str) -> Result<(String, String), Error> { )) } -fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { +pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { let mut hasher = Sha256::default(); hasher.update(canonical_req.as_bytes()); [ @@ -227,18 +226,17 @@ fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: & .join("\n") } -fn canonical_request( +pub fn canonical_request( method: &Method, - url_path: &str, - canonical_query_string: &str, + uri: &hyper::Uri, headers: &HashMap, signed_headers: &str, content_sha256: &str, ) -> String { [ method.as_str(), - url_path, - canonical_query_string, + &uri.path().to_string(), + &canonical_query_string(uri), &canonical_header_string(headers, signed_headers), "", signed_headers, diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 88eff20b..86f52495 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -55,8 +55,11 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi netapp = "0.4.1" [dev-dependencies] -aws-sdk-s3 = "0.6" +aws-sdk-s3 = "0.8" +chrono = "0.4" http = "0.2" +hmac = "0.10" hyper = { version = "0.14", features = ["client", "http1", "runtime"] } +sha2 = "0.9" static_init = "1.0" diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs new file mode 100644 index 00000000..11dc10d6 --- /dev/null +++ b/src/garage/tests/common/custom_requester.rs @@ -0,0 +1,280 @@ +#![allow(dead_code)] + +use std::collections::HashMap; +use std::convert::TryFrom; + +use chrono::{offset::Utc, DateTime}; +use hmac::{Hmac, Mac}; +use hyper::client::HttpConnector; +use hyper::{Body, Client, Method, Request, Response, Uri}; + +use super::garage::{Instance, Key}; +use garage_api::signature; + +/// You should ever only use this to send requests AWS sdk won't send, +/// like to reproduce behavior of unusual implementations found to be +/// problematic. +pub struct CustomRequester { + key: Key, + uri: Uri, + client: Client, +} + +impl CustomRequester { + pub fn new(instance: &Instance) -> Self { + /* + let credentials = Credentials::new( + &instance.key.id, + &instance.key.secret, + None, + None, + "garage-integ-test", + ); + let endpoint = Endpoint::immutable(instance.uri()); + */ + CustomRequester { + key: instance.key.clone(), + uri: instance.uri(), + client: Client::new(), + } + } + + pub fn builder(&self, bucket: String) -> RequestBuilder<'_> { + RequestBuilder { + requester: self, + bucket, + method: Method::GET, + path: String::new(), + query_params: HashMap::new(), + signed_headers: HashMap::new(), + unsigned_headers: HashMap::new(), + body: Vec::new(), + body_signature: BodySignature::Classic, + vhost_style: false, + } + } + /* + pub async fn request(&self, method: &str, path: String, headers: &HashMap, body: &[u8], vhost_style: bool) -> hyper::Result> { + let request = Request::builder() + .method( + self.client.request(todo!()).await + } + */ +} + +pub struct RequestBuilder<'a> { + requester: &'a CustomRequester, + bucket: String, + method: Method, + path: String, + query_params: HashMap>, + signed_headers: HashMap, + unsigned_headers: HashMap, + body: Vec, + body_signature: BodySignature, + vhost_style: bool, +} + +impl<'a> RequestBuilder<'a> { + pub fn method(&mut self, method: Method) -> &mut Self { + self.method = method; + self + } + + pub fn path(&mut self, path: String) -> &mut Self { + self.path = path; + self + } + + pub fn query_params(&mut self, query_params: HashMap>) -> &mut Self { + self.query_params = query_params; + self + } + + pub fn signed_headers(&mut self, signed_headers: HashMap) -> &mut Self { + self.signed_headers = signed_headers; + self + } + + pub fn unsigned_headers(&mut self, unsigned_headers: HashMap) -> &mut Self { + self.unsigned_headers = unsigned_headers; + self + } + + pub fn body(&mut self, body: Vec) -> &mut Self { + self.body = body; + self + } + + pub fn body_signature(&mut self, body_signature: BodySignature) -> &mut Self { + self.body_signature = body_signature; + self + } + + pub fn vhost_style(&mut self, vhost_style: bool) -> &mut Self { + self.vhost_style = vhost_style; + self + } + + pub async fn send(&mut self) -> hyper::Result> { + // TODO this is a bit incorrect in that path and query params should be url-encoded and + // aren't, but this is good enought for now. + + let query = query_param_to_string(&self.query_params); + let (host, path) = if self.vhost_style { + ( + format!("{}.s3.garage", self.bucket), + format!("{}{}", self.path, query), + ) + } else { + ( + "s3.garage".to_owned(), + format!("{}/{}{}", self.bucket, self.path, query), + ) + }; + let uri = format!("{}{}", self.requester.uri, path); + + let now = Utc::now(); + let scope = signature::compute_scope(&now, super::REGION.as_ref()); + let mut signer = signature::signing_hmac( + &now, + &self.requester.key.secret, + super::REGION.as_ref(), + "s3", + ) + .unwrap(); + let streaming_signer = signer.clone(); + + let mut all_headers = self.signed_headers.clone(); + + let date = now.format(signature::LONG_DATETIME).to_string(); + all_headers.insert("x-amz-date".to_owned(), date); + all_headers.insert("host".to_owned(), host); + + let body_sha = match self.body_signature { + BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(), + BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)), + BodySignature::Streaming(size) => { + all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned()); + all_headers.insert( + "x-amz-decoded-content-length".to_owned(), + self.body.len().to_string(), + ); + // this is a pretty lazy and inefficient way to do it, but it's enought for + // test code. + all_headers.insert( + "content-length".to_owned(), + to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "") + .len() + .to_string(), + ); + + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned() + } + }; + all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone()); + + let mut signed_headers = all_headers + .iter() + .map(|(k, _)| k.as_ref()) + .collect::>(); + signed_headers.sort(); + let signed_headers = signed_headers.join(";"); + + all_headers.extend(self.unsigned_headers.clone()); + + let canonical_request = signature::payload::canonical_request( + &self.method, + &Uri::try_from(&uri).unwrap(), + &all_headers, + &signed_headers, + &body_sha, + ); + + let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request); + + signer.update(string_to_sign.as_bytes()); + let signature = hex::encode(signer.finalize().into_bytes()); + let authorization = format!( + "AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}", + self.requester.key.id, scope, signed_headers, signature + ); + all_headers.insert("authorization".to_owned(), authorization); + + let mut request = Request::builder(); + for (k, v) in all_headers { + request = request.header(k, v); + } + + let body = if let BodySignature::Streaming(size) = self.body_signature { + to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope) + } else { + self.body.clone() + }; + let request = request + .uri(uri) + .method(self.method.clone()) + .body(Body::from(body)) + .unwrap(); + self.requester.client.request(request).await + } +} + +pub enum BodySignature { + Unsigned, + Classic, + Streaming(usize), +} + +fn query_param_to_string(params: &HashMap>) -> String { + if params.is_empty() { + return String::new(); + } + + "?".to_owned() + + ¶ms + .iter() + .map(|(k, v)| { + if let Some(v) = v { + format!("{}={}", k, v) + } else { + k.clone() + } + }) + .collect::>() + .join("&") +} + +fn to_streaming_body( + body: &[u8], + chunk_size: usize, + mut seed: String, + hasher: Hmac, + now: DateTime, + scope: &str, +) -> Vec { + const SHA_NULL: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + let now = now.format(signature::LONG_DATETIME).to_string(); + let mut res = Vec::with_capacity(body.len()); + for chunk in body.chunks(chunk_size).chain(std::iter::once(&[][..])) { + let to_sign = format!( + "AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}", + now, + scope, + seed, + SHA_NULL, + hex::encode(garage_util::data::sha256sum(chunk)) + ); + + let mut hasher = hasher.clone(); + hasher.update(to_sign.as_bytes()); + seed = hex::encode(hasher.finalize().into_bytes()); + + let header = format!("{:x};chunk-signature={}\r\n", chunk.len(), seed); + res.extend_from_slice(header.as_bytes()); + res.extend_from_slice(chunk); + res.extend_from_slice(b"\r\n"); + } + + res +} diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs index 36adb55e..88c51501 100644 --- a/src/garage/tests/common/garage.rs +++ b/src/garage/tests/common/garage.rs @@ -11,7 +11,7 @@ pub const DEFAULT_PORT: u16 = 49995; static GARAGE_TEST_SECRET: &str = "c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44"; -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct Key { pub name: String, pub id: String, diff --git a/src/garage/tests/common/mod.rs b/src/garage/tests/common/mod.rs index 32fa3848..8f88c731 100644 --- a/src/garage/tests/common/mod.rs +++ b/src/garage/tests/common/mod.rs @@ -5,22 +5,31 @@ use ext::*; pub mod macros; pub mod client; +pub mod custom_requester; pub mod ext; pub mod garage; +use custom_requester::CustomRequester; + const REGION: Region = Region::from_static("garage-integ-test"); pub struct Context { pub garage: &'static garage::Instance, pub client: Client, + pub custom_request: CustomRequester, } impl Context { fn new() -> Self { let garage = garage::instance(); let client = client::build_client(garage); + let custom_request = CustomRequester::new(garage); - Context { garage, client } + Context { + garage, + client, + custom_request, + } } /// Create an unique bucket with a random suffix. diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs index 9d7e4322..8799c395 100644 --- a/src/garage/tests/lib.rs +++ b/src/garage/tests/lib.rs @@ -7,4 +7,5 @@ mod list; mod multipart; mod objects; mod simple; +mod streaming_signature; mod website; diff --git a/src/garage/tests/multipart.rs b/src/garage/tests/multipart.rs index 7fec4de6..895a2993 100644 --- a/src/garage/tests/multipart.rs +++ b/src/garage/tests/multipart.rs @@ -1,6 +1,6 @@ use crate::common; use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart}; -use aws_sdk_s3::ByteStream; +use aws_sdk_s3::types::ByteStream; const SZ_5MB: usize = 5 * 1024 * 1024; const SZ_10MB: usize = 10 * 1024 * 1024; diff --git a/src/garage/tests/objects.rs b/src/garage/tests/objects.rs index 9086073e..e1175b81 100644 --- a/src/garage/tests/objects.rs +++ b/src/garage/tests/objects.rs @@ -1,6 +1,6 @@ use crate::common; use aws_sdk_s3::model::{Delete, ObjectIdentifier}; -use aws_sdk_s3::ByteStream; +use aws_sdk_s3::types::ByteStream; const STD_KEY: &str = "hello world"; const CTRL_KEY: &str = "\x00\x01\x02\x00"; diff --git a/src/garage/tests/simple.rs b/src/garage/tests/simple.rs index a15cbf80..f54ae9ac 100644 --- a/src/garage/tests/simple.rs +++ b/src/garage/tests/simple.rs @@ -2,7 +2,7 @@ use crate::common; #[tokio::test] async fn test_simple() { - use aws_sdk_s3::ByteStream; + use aws_sdk_s3::types::ByteStream; let ctx = common::context(); let bucket = ctx.create_bucket("test-simple"); diff --git a/src/garage/tests/streaming_signature.rs b/src/garage/tests/streaming_signature.rs new file mode 100644 index 00000000..66f985ac --- /dev/null +++ b/src/garage/tests/streaming_signature.rs @@ -0,0 +1,101 @@ +use std::collections::HashMap; + +use crate::common; +use common::custom_requester::BodySignature; +use hyper::Method; + +const STD_KEY: &str = "hello-world"; +//const CTRL_KEY: &str = "\x00\x01\x02\x00"; +const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +#[tokio::test] +async fn test_putobject_streaming() { + let ctx = common::context(); + let bucket = ctx.create_bucket("putobject-streaming"); + + { + // Send an empty object (can serve as a directory marker) + // with a content type + let etag = "\"d41d8cd98f00b204e9800998ecf8427e\""; + let content_type = "text/csv"; + let mut headers = HashMap::new(); + headers.insert("content-type".to_owned(), content_type.to_owned()); + let _ = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + .path(STD_KEY.to_owned()) + .unsigned_headers(headers) + .vhost_style(true) + .body(vec![]) + .body_signature(BodySignature::Streaming(10)) + .send() + .await + .unwrap(); + + // assert_eq!(r.e_tag.unwrap().as_str(), etag); + // We return a version ID here + // We should check if Amazon is returning one when versioning is not enabled + // assert!(r.version_id.is_some()); + + //let _version = r.version_id.unwrap(); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, b""); + assert_eq!(o.e_tag.unwrap(), etag); + // We do not return version ID + // We should check if Amazon is returning one when versioning is not enabled + // assert_eq!(o.version_id.unwrap(), _version); + assert_eq!(o.content_type.unwrap(), content_type); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length, 0); + assert_eq!(o.parts_count, 0); + assert_eq!(o.tag_count, 0); + } + + { + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + + let _ = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + //.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this + //fail + .path("abc".to_owned()) + .vhost_style(true) + .body(BODY.to_vec()) + .body_signature(BodySignature::Streaming(16)) + .send() + .await + .unwrap(); + + // assert_eq!(r.e_tag.unwrap().as_str(), etag); + // assert!(r.version_id.is_some()); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + //.key(CTRL_KEY) + .key("abc") + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, BODY); + assert_eq!(o.e_tag.unwrap(), etag); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length, 62); + assert_eq!(o.parts_count, 0); + assert_eq!(o.tag_count, 0); + } +} diff --git a/src/garage/tests/website.rs b/src/garage/tests/website.rs index 34093a79..963d11ea 100644 --- a/src/garage/tests/website.rs +++ b/src/garage/tests/website.rs @@ -2,7 +2,7 @@ use crate::common; use crate::common::ext::*; use aws_sdk_s3::{ model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, - ByteStream, + types::ByteStream, }; use http::Request; use hyper::{ -- 2.43.0 From 84eeaa780336e8c5e97b5fde26573806ca2c9014 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Mon, 21 Mar 2022 21:07:56 +0100 Subject: [PATCH 4/5] add test for create bucket and put website with streaming signature --- src/garage/tests/streaming_signature.rs | 84 +++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/src/garage/tests/streaming_signature.rs b/src/garage/tests/streaming_signature.rs index 66f985ac..c68f7dfc 100644 --- a/src/garage/tests/streaming_signature.rs +++ b/src/garage/tests/streaming_signature.rs @@ -99,3 +99,87 @@ async fn test_putobject_streaming() { assert_eq!(o.tag_count, 0); } } + +#[tokio::test] +async fn test_create_bucket_streaming() { + let ctx = common::context(); + let bucket = "createbucket-streaming"; + + { + // create bucket + let _ = ctx + .custom_request + .builder(bucket.to_owned()) + .method(Method::PUT) + .body_signature(BodySignature::Streaming(10)) + .send() + .await + .unwrap(); + + // test if the bucket exists and works properly + let etag = "\"d41d8cd98f00b204e9800998ecf8427e\""; + let content_type = "text/csv"; + let _ = ctx + .client + .put_object() + .bucket(bucket) + .key(STD_KEY) + .content_type(content_type) + .send() + .await + .unwrap(); + + let o = ctx + .client + .get_object() + .bucket(bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + + assert_eq!(o.e_tag.unwrap(), etag); + } +} + +#[tokio::test] +async fn test_put_website_streaming() { + let ctx = common::context(); + let bucket = ctx.create_bucket("putwebsite-streaming"); + + { + let website_config = r#" + + + err/error.html + + + home.html + +"#; + + let mut query = HashMap::new(); + query.insert("website".to_owned(), None); + let _ = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + .query_params(query) + .body(website_config.as_bytes().to_vec()) + .body_signature(BodySignature::Streaming(10)) + .send() + .await + .unwrap(); + + let o = ctx + .client + .get_bucket_website() + .bucket(&bucket) + .send() + .await + .unwrap(); + + assert_eq!(o.index_document.unwrap().suffix.unwrap(), "home.html"); + assert_eq!(o.error_document.unwrap().key.unwrap(), "err/error.html"); + } +} -- 2.43.0 From a5374a4fdf981c3317563febd45354beee169b4e Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Tue, 22 Mar 2022 18:20:39 +0100 Subject: [PATCH 5/5] cleanup --- src/garage/tests/common/custom_requester.rs | 23 ++++----------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs index 11dc10d6..580691a1 100644 --- a/src/garage/tests/common/custom_requester.rs +++ b/src/garage/tests/common/custom_requester.rs @@ -22,16 +22,6 @@ pub struct CustomRequester { impl CustomRequester { pub fn new(instance: &Instance) -> Self { - /* - let credentials = Credentials::new( - &instance.key.id, - &instance.key.secret, - None, - None, - "garage-integ-test", - ); - let endpoint = Endpoint::immutable(instance.uri()); - */ CustomRequester { key: instance.key.clone(), uri: instance.uri(), @@ -53,13 +43,6 @@ impl CustomRequester { vhost_style: false, } } - /* - pub async fn request(&self, method: &str, path: String, headers: &HashMap, body: &[u8], vhost_style: bool) -> hyper::Result> { - let request = Request::builder() - .method( - self.client.request(todo!()).await - } - */ } pub struct RequestBuilder<'a> { @@ -160,8 +143,10 @@ impl<'a> RequestBuilder<'a> { "x-amz-decoded-content-length".to_owned(), self.body.len().to_string(), ); - // this is a pretty lazy and inefficient way to do it, but it's enought for - // test code. + // Get lenght of body by doing the conversion to a streaming body with an + // invalid signature (we don't know the seed) just to get its length. This + // is a pretty lazy and inefficient way to do it, but it's enought for test + // code. all_headers.insert( "content-length".to_owned(), to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "") -- 2.43.0