add test framework for arbitraty S3 requests

and implement some basic test with it

Cargo.lock

@@ -53,9 +53,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "aws-endpoint"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6"
+checksum = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167"
 dependencies = [
  "aws-smithy-http",
  "aws-types",
@@ -66,9 +66,9 @@ dependencies = [
 
 [[package]]
 name = "aws-http"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1"
+checksum = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506"
 dependencies = [
  "aws-smithy-http",
  "aws-smithy-types",
@@ -81,9 +81,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-s3"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9"
+checksum = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7"
 dependencies = [
  "aws-endpoint",
  "aws-http",
@@ -106,9 +106,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sig-auth"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc"
+checksum = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6"
 dependencies = [
  "aws-sigv4",
  "aws-smithy-eventstream",
@@ -121,9 +121,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sigv4"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c"
+checksum = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"
 dependencies = [
  "aws-smithy-eventstream",
  "aws-smithy-http",
@@ -141,9 +141,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-async"
-version = "0.36.0"
+version = "0.38.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e"
+checksum = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8"
 dependencies = [
  "futures-util",
  "pin-project-lite",
@@ -153,9 +153,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-client"
-version = "0.36.0"
+version = "0.38.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb"
+checksum = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-http",
@@ -177,9 +177,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-eventstream"
-version = "0.36.0"
+version = "0.38.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9"
+checksum = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"
 dependencies = [
  "aws-smithy-types",
  "bytes 1.1.0",
@@ -188,9 +188,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-http"
-version = "0.36.0"
+version = "0.38.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696"
+checksum = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"
 dependencies = [
  "aws-smithy-eventstream",
  "aws-smithy-types",
@@ -209,9 +209,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-http-tower"
-version = "0.36.0"
+version = "0.38.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d"
+checksum = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"
 dependencies = [
  "aws-smithy-http",
  "bytes 1.1.0",
@@ -224,9 +224,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-types"
-version = "0.36.0"
+version = "0.38.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f"
+checksum = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2"
 dependencies = [
  "itoa 1.0.1",
  "num-integer",
@@ -236,9 +236,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-xml"
-version = "0.36.0"
+version = "0.38.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4"
+checksum = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f"
 dependencies = [
  "thiserror",
  "xmlparser",
@@ -246,11 +246,12 @@ dependencies = [
 
 [[package]]
 name = "aws-types"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142"
+checksum = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38"
 dependencies = [
  "aws-smithy-async",
+ "aws-smithy-client",
  "aws-smithy-types",
  "rustc_version",
  "tracing",
@@ -664,6 +665,7 @@ dependencies = [
  "async-trait",
  "aws-sdk-s3",
  "bytes 1.1.0",
+ "chrono",
  "futures",
  "futures-util",
  "garage_api",
@@ -674,6 +676,7 @@ dependencies = [
  "garage_web",
  "git-version",
  "hex",
+ "hmac",
  "http",
  "hyper",
  "kuska-sodiumoxide",
@@ -684,6 +687,7 @@ dependencies = [
  "rmp-serde 0.15.5",
  "serde",
  "serde_bytes",
+ "sha2",
  "sled",
  "static_init",
  "structopt",

Cargo.nix

@@ -114,29 +114,29 @@ in
     src = fetchCratesIo { inherit name version; sha256 = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"; };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-endpoint";
-    version = "0.6.0";
+    version = "0.8.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6"; };
+    src = fetchCratesIo { inherit name version; sha256 = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167"; };
     dependencies = {
-      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
+      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
-      aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
+      aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
       http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
       regex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.4" { inherit profileName; };
       tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.29" { inherit profileName; };
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-http";
-    version = "0.6.0";
+    version = "0.8.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1"; };
+    src = fetchCratesIo { inherit name version; sha256 = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506"; };
     dependencies = {
-      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
+      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
-      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
+      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
-      aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
+      aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
       http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
       lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; };
       percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
@@ -144,29 +144,29 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-sdk-s3";
-    version = "0.6.0";
+    version = "0.8.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9"; };
+    src = fetchCratesIo { inherit name version; sha256 = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7"; };
     features = builtins.concatLists [
       [ "default" ]
       [ "rt-tokio" ]
       [ "rustls" ]
     ];
     dependencies = {
-      aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" { inherit profileName; };
+      aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" { inherit profileName; };
-      aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" { inherit profileName; };
+      aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" { inherit profileName; };
-      aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" { inherit profileName; };
+      aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" { inherit profileName; };
-      aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; };
+      aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; };
-      aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; };
+      aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
-      aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" { inherit profileName; };
+      aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; };
-      aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
+      aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
-      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
+      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
-      aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; };
+      aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; };
-      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
+      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
-      aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" { inherit profileName; };
+      aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" { inherit profileName; };
-      aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
+      aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
       bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
       http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
       md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md5."0.7.0" { inherit profileName; };
@@ -175,31 +175,31 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-sig-auth";
-    version = "0.6.0";
+    version = "0.8.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc"; };
+    src = fetchCratesIo { inherit name version; sha256 = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6"; };
     features = builtins.concatLists [
       [ "aws-smithy-eventstream" ]
       [ "sign-eventstream" ]
     ];
     dependencies = {
-      aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; };
+      aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; };
-      aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
+      aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
-      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
+      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
-      aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
+      aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
       http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
       thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
       tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.29" { inherit profileName; };
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-sigv4";
-    version = "0.6.0";
+    version = "0.8.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c"; };
+    src = fetchCratesIo { inherit name version; sha256 = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"; };
     features = builtins.concatLists [
       [ "aws-smithy-eventstream" ]
       [ "bytes" ]
@@ -211,8 +211,8 @@ in
       [ "sign-http" ]
     ];
     dependencies = {
-      aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
+      aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
-      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
+      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
       bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
       form_urlencoded = rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; };
       hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
@@ -226,11 +226,11 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-smithy-async";
-    version = "0.36.0";
+    version = "0.38.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e"; };
+    src = fetchCratesIo { inherit name version; sha256 = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8"; };
     features = builtins.concatLists [
       [ "rt-tokio" ]
     ];
@@ -242,11 +242,11 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-smithy-client";
-    version = "0.36.0";
+    version = "0.38.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb"; };
+    src = fetchCratesIo { inherit name version; sha256 = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a"; };
     features = builtins.concatLists [
       [ "client-hyper" ]
       [ "hyper" ]
@@ -256,10 +256,10 @@ in
       [ "rustls" ]
     ];
     dependencies = {
-      aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; };
+      aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
-      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
+      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
-      aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; };
+      aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; };
-      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
+      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
       bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
       fastrand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.7.0" { inherit profileName; };
       http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
@@ -275,23 +275,23 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-smithy-eventstream";
-    version = "0.36.0";
+    version = "0.38.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9"; };
+    src = fetchCratesIo { inherit name version; sha256 = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"; };
     dependencies = {
-      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
+      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
       bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
       crc32fast = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.2.1" { inherit profileName; };
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-smithy-http";
-    version = "0.36.0";
+    version = "0.38.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696"; };
+    src = fetchCratesIo { inherit name version; sha256 = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"; };
     features = builtins.concatLists [
       [ "aws-smithy-eventstream" ]
       [ "event-stream" ]
@@ -300,8 +300,8 @@ in
       [ "tokio-util" ]
     ];
     dependencies = {
-      aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
+      aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
-      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
+      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
       bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
       bytes_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.1" { inherit profileName; };
       futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.17" { inherit profileName; };
@@ -316,13 +316,13 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-smithy-http-tower";
-    version = "0.36.0";
+    version = "0.38.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d"; };
+    src = fetchCratesIo { inherit name version; sha256 = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"; };
     dependencies = {
-      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
+      aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
       bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
       http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
       http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; };
@@ -332,11 +332,11 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-smithy-types";
-    version = "0.36.0";
+    version = "0.38.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f"; };
+    src = fetchCratesIo { inherit name version; sha256 = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2"; };
     dependencies = {
       itoa = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; };
       num_integer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; };
@@ -345,25 +345,26 @@ in
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-smithy-xml";
-    version = "0.36.0";
+    version = "0.38.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4"; };
+    src = fetchCratesIo { inherit name version; sha256 = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f"; };
     dependencies = {
       thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
       xmlparser = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" { inherit profileName; };
     };
   });
   
-  "registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" = overridableMkRustCrate (profileName: rec {
+  "registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "aws-types";
-    version = "0.6.0";
+    version = "0.8.0";
     registry = "registry+https://github.com/rust-lang/crates.io-index";
-    src = fetchCratesIo { inherit name version; sha256 = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142"; };
+    src = fetchCratesIo { inherit name version; sha256 = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38"; };
     dependencies = {
-      aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; };
+      aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
-      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
+      aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; };
+      aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
       tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.29" { inherit profileName; };
       zeroize = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.1" { inherit profileName; };
     };
@@ -997,9 +998,12 @@ in
       toml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; };
     };
     devDependencies = {
-      aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" { inherit profileName; };
+      aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" { inherit profileName; };
+      chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
+      hmac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.10.1" { inherit profileName; };
       http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
       hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; };
+      sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.8" { inherit profileName; };
       static_init = rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.2" { inherit profileName; };
     };
   });
@@ -2335,7 +2339,7 @@ in
     ];
     dependencies = {
       ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.115" { inherit profileName; };
-      ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.8.0" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.8.0" { inherit profileName; };
       ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; };
       untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; };
       ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; };

src/api/api_server.rs

@@ -1,7 +1,9 @@
 use std::net::SocketAddr;
 use std::sync::Arc;
 
+use chrono::{DateTime, NaiveDateTime, Utc};
 use futures::future::Future;
+use futures::prelude::*;
 use hyper::header;
 use hyper::server::conn::AddrStream;
 use hyper::service::{make_service_fn, service_fn};
@@ -16,7 +18,10 @@ use garage_model::key_table::Key;
 use garage_table::util::*;
 
 use crate::error::*;
+use crate::signature::compute_scope;
 use crate::signature::payload::check_payload_signature;
+use crate::signature::streaming::SignedPayloadStream;
+use crate::signature::LONG_DATETIME;
 
 use crate::helpers::*;
 use crate::s3_bucket::*;
@@ -96,7 +101,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 	let authority = req
 		.headers()
 		.get(header::HOST)
-		.ok_or_else(|| Error::BadRequest("HOST header required".to_owned()))?
+		.ok_or_bad_request("Host header required")?
 		.to_str()?;
 
 	let host = authority_to_host(authority)?;
@@ -119,11 +124,57 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 		return handle_options_s3api(garage, &req, bucket_name).await;
 	}
 
-	let (api_key, content_sha256) = check_payload_signature(&garage, &req).await?;
+	let (api_key, mut content_sha256) = check_payload_signature(&garage, &req).await?;
 	let api_key = api_key.ok_or_else(|| {
 		Error::Forbidden("Garage does not support anonymous access yet".to_string())
 	})?;
 
+	let req = match req.headers().get("x-amz-content-sha256") {
+		Some(header) if header == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => {
+			let signature = content_sha256
+				.take()
+				.ok_or_bad_request("No signature provided")?;
+
+			let secret_key = &api_key
+				.state
+				.as_option()
+				.ok_or_internal_error("Deleted key state")?
+				.secret_key;
+
+			let date = req
+				.headers()
+				.get("x-amz-date")
+				.ok_or_bad_request("Missing X-Amz-Date field")?
+				.to_str()?;
+			let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME)
+				.ok_or_bad_request("Invalid date")?;
+			let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
+
+			let scope = compute_scope(&date, &garage.config.s3_api.s3_region);
+			let signing_hmac = crate::signature::signing_hmac(
+				&date,
+				secret_key,
+				&garage.config.s3_api.s3_region,
+				"s3",
+			)
+			.ok_or_internal_error("Unable to build signing HMAC")?;
+
+			req.map(move |body| {
+				Body::wrap_stream(
+					SignedPayloadStream::new(
+						body.map_err(Error::from),
+						signing_hmac,
+						date,
+						&scope,
+						signature,
+					)
+					.map_err(Error::from),
+				)
+			})
+		}
+		_ => req,
+	};
+
 	let bucket_name = match bucket_name {
 		None => return handle_request_without_bucket(garage, req, api_key, endpoint).await,
 		Some(bucket) => bucket.to_string(),
@@ -205,7 +256,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 			.await
 		}
 		Endpoint::PutObject { key } => {
-			handle_put(garage, req, bucket_id, &key, &api_key, content_sha256).await
+			handle_put(garage, req, bucket_id, &key, content_sha256).await
 		}
 		Endpoint::AbortMultipartUpload { key, upload_id } => {
 			handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await

src/api/lib.rs

@@ -10,7 +10,8 @@ mod encoding;
 mod api_server;
 pub use api_server::run_api_server;
 
-mod signature;
+/// This mode is public only to help testing. Don't expect stability here
+pub mod signature;
 
 pub mod helpers;
 mod s3_bucket;

src/api/s3_put.rs

@@ -1,8 +1,7 @@
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::sync::Arc;
 
-use chrono::{DateTime, NaiveDateTime, Utc};
+use futures::prelude::*;
-use futures::{prelude::*, TryFutureExt};
 use hyper::body::{Body, Bytes};
 use hyper::header::{HeaderMap, HeaderValue};
 use hyper::{Request, Response};
@@ -17,23 +16,19 @@ use garage_util::time::*;
 use garage_model::block::INLINE_THRESHOLD;
 use garage_model::block_ref_table::*;
 use garage_model::garage::Garage;
-use garage_model::key_table::Key;
 use garage_model::object_table::*;
 use garage_model::version_table::*;
 
 use crate::error::*;
 use crate::s3_xml;
-use crate::signature::streaming::SignedPayloadStream;
+use crate::signature::verify_signed_content;
-use crate::signature::LONG_DATETIME;
-use crate::signature::{compute_scope, verify_signed_content};
 
 pub async fn handle_put(
 	garage: Arc<Garage>,
 	req: Request<Body>,
 	bucket_id: Uuid,
 	key: &str,
-	api_key: &Key,
+	content_sha256: Option<Hash>,
-	mut content_sha256: Option<Hash>,
 ) -> Result<Response<Body>, Error> {
 	// Retrieve interesting headers from request
 	let headers = get_headers(req.headers())?;
@@ -43,52 +38,10 @@ pub async fn handle_put(
 		Some(x) => Some(x.to_str()?.to_string()),
 		None => None,
 	};
-	let payload_seed_signature = match req.headers().get("x-amz-content-sha256") {
-		Some(header) if header == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => {
-			let content_sha256 = content_sha256
-				.take()
-				.ok_or_bad_request("No signature provided")?;
-			Some(content_sha256)
-		}
-		_ => None,
-	};
 
-	// Parse body of uploaded file
+	let (_head, body) = req.into_parts();
-	let (head, body) = req.into_parts();
 	let body = body.map_err(Error::from);
 
-	let body = if let Some(signature) = payload_seed_signature {
-		let secret_key = &api_key
-			.state
-			.as_option()
-			.ok_or_internal_error("Deleted key state")?
-			.secret_key;
-
-		let date = head
-			.headers
-			.get("x-amz-date")
-			.ok_or_bad_request("Missing X-Amz-Date field")?
-			.to_str()?;
-		let date: NaiveDateTime =
-			NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?;
-		let date: DateTime<Utc> = DateTime::from_utc(date, Utc);
-
-		let scope = compute_scope(&date, &garage.config.s3_api.s3_region);
-		let signing_hmac = crate::signature::signing_hmac(
-			&date,
-			secret_key,
-			&garage.config.s3_api.s3_region,
-			"s3",
-		)
-		.ok_or_internal_error("Unable to build signing HMAC")?;
-
-		SignedPayloadStream::new(body, signing_hmac, date, &scope, signature)?
-			.map_err(Error::from)
-			.boxed()
-	} else {
-		body.boxed()
-	};
-
 	save_stream(
 		garage,
 		headers,

src/api/signature/payload.rs

@@ -51,8 +51,7 @@ pub async fn check_payload_signature(
 
 	let canonical_request = canonical_request(
 		request.method(),
-		&request.uri().path().to_string(),
+		&request.uri(),
-		&canonical_query_string(request.uri()),
 		&headers,
 		&authorization.signed_headers,
 		&authorization.content_sha256,
@@ -215,7 +214,7 @@ fn parse_credential(cred: &str) -> Result<(String, String), Error> {
 	))
 }
 
-fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String {
+pub fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String {
 	let mut hasher = Sha256::default();
 	hasher.update(canonical_req.as_bytes());
 	[
@@ -227,18 +226,17 @@ fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &
 	.join("\n")
 }
 
-fn canonical_request(
+pub fn canonical_request(
 	method: &Method,
-	url_path: &str,
+	uri: &hyper::Uri,
-	canonical_query_string: &str,
 	headers: &HashMap<String, String>,
 	signed_headers: &str,
 	content_sha256: &str,
 ) -> String {
 	[
 		method.as_str(),
-		url_path,
+		&uri.path().to_string(),
-		canonical_query_string,
+		&canonical_query_string(uri),
 		&canonical_header_string(headers, signed_headers),
 		"",
 		signed_headers,

src/api/signature/streaming.rs

@@ -164,15 +164,15 @@ where
 		datetime: DateTime<Utc>,
 		scope: &str,
 		seed_signature: Hash,
-	) -> Result<Self, Error> {
+	) -> Self {
-		Ok(Self {
+		Self {
 			stream,
 			buf: bytes::BytesMut::new(),
 			datetime,
 			scope: scope.into(),
 			signing_hmac,
 			previous_signature: seed_signature,
-		})
+		}
 	}
 
 	fn parse_next(input: &[u8]) -> nom::IResult<&[u8], SignedPayload, SignedPayloadStreamError> {
@@ -305,7 +305,7 @@ mod tests {
 		let seed_signature = Hash::default();
 
 		let mut stream =
-			SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature).unwrap();
+			SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature);
 
 		assert!(stream.try_next().await.is_err());
 		match stream.try_next().await {

src/garage/Cargo.toml

@@ -53,8 +53,11 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
 netapp = "0.3.0"
 
 [dev-dependencies]
-aws-sdk-s3 = "0.6"
+aws-sdk-s3 = "0.8"
+chrono = "0.4"
 http = "0.2"
+hmac = "0.10"
 hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
+sha2 = "0.9"
 
 static_init = "1.0"

src/garage/tests/common/custom_requester.rs

@@ -0,0 +1,280 @@
+#![allow(dead_code)]
+
+use std::collections::HashMap;
+use std::convert::TryFrom;
+
+use chrono::{offset::Utc, DateTime};
+use hmac::{Hmac, Mac};
+use hyper::client::HttpConnector;
+use hyper::{Body, Client, Method, Request, Response, Uri};
+
+use super::garage::{Instance, Key};
+use garage_api::signature;
+
+/// You should ever only use this to send requests AWS sdk won't send,
+/// like to reproduce behavior of unusual implementations found to be
+/// problematic.
+pub struct CustomRequester {
+	key: Key,
+	uri: Uri,
+	client: Client<HttpConnector>,
+}
+
+impl CustomRequester {
+	pub fn new(instance: &Instance) -> Self {
+		/*
+		let credentials = Credentials::new(
+			&instance.key.id,
+			&instance.key.secret,
+			None,
+			None,
+			"garage-integ-test",
+		);
+		let endpoint = Endpoint::immutable(instance.uri());
+		*/
+		CustomRequester {
+			key: instance.key.clone(),
+			uri: instance.uri(),
+			client: Client::new(),
+		}
+	}
+
+	pub fn builder(&self, bucket: String) -> RequestBuilder<'_> {
+		RequestBuilder {
+			requester: self,
+			bucket,
+			method: Method::GET,
+			path: String::new(),
+			query_params: HashMap::new(),
+			signed_headers: HashMap::new(),
+			unsigned_headers: HashMap::new(),
+			body: Vec::new(),
+			body_signature: BodySignature::Classic,
+			vhost_style: false,
+		}
+	}
+	/*
+	pub async fn request(&self, method: &str, path: String, headers: &HashMap<String, (bool, String)>, body: &[u8], vhost_style: bool) -> hyper::Result<Response<Body>> {
+		let request = Request::builder()
+			.method(
+		self.client.request(todo!()).await
+	}
+	*/
+}
+
+pub struct RequestBuilder<'a> {
+	requester: &'a CustomRequester,
+	bucket: String,
+	method: Method,
+	path: String,
+	query_params: HashMap<String, Option<String>>,
+	signed_headers: HashMap<String, String>,
+	unsigned_headers: HashMap<String, String>,
+	body: Vec<u8>,
+	body_signature: BodySignature,
+	vhost_style: bool,
+}
+
+impl<'a> RequestBuilder<'a> {
+	pub fn method(&mut self, method: Method) -> &mut Self {
+		self.method = method;
+		self
+	}
+
+	pub fn path(&mut self, path: String) -> &mut Self {
+		self.path = path;
+		self
+	}
+
+	pub fn query_params(&mut self, query_params: HashMap<String, Option<String>>) -> &mut Self {
+		self.query_params = query_params;
+		self
+	}
+
+	pub fn signed_headers(&mut self, signed_headers: HashMap<String, String>) -> &mut Self {
+		self.signed_headers = signed_headers;
+		self
+	}
+
+	pub fn unsigned_headers(&mut self, unsigned_headers: HashMap<String, String>) -> &mut Self {
+		self.unsigned_headers = unsigned_headers;
+		self
+	}
+
+	pub fn body(&mut self, body: Vec<u8>) -> &mut Self {
+		self.body = body;
+		self
+	}
+
+	pub fn body_signature(&mut self, body_signature: BodySignature) -> &mut Self {
+		self.body_signature = body_signature;
+		self
+	}
+
+	pub fn vhost_style(&mut self, vhost_style: bool) -> &mut Self {
+		self.vhost_style = vhost_style;
+		self
+	}
+
+	pub async fn send(&mut self) -> hyper::Result<Response<Body>> {
+		// TODO this is a bit incorrect in that path and query params should be url-encoded and
+		// aren't, but this is good enought for now.
+
+		let query = query_param_to_string(&self.query_params);
+		let (host, path) = if self.vhost_style {
+			(
+				format!("{}.s3.garage", self.bucket),
+				format!("{}{}", self.path, query),
+			)
+		} else {
+			(
+				"s3.garage".to_owned(),
+				format!("{}/{}{}", self.bucket, self.path, query),
+			)
+		};
+		let uri = format!("{}{}", self.requester.uri, path);
+
+		let now = Utc::now();
+		let scope = signature::compute_scope(&now, super::REGION.as_ref());
+		let mut signer = signature::signing_hmac(
+			&now,
+			&self.requester.key.secret,
+			super::REGION.as_ref(),
+			"s3",
+		)
+		.unwrap();
+		let streaming_signer = signer.clone();
+
+		let mut all_headers = self.signed_headers.clone();
+
+		let date = now.format(signature::LONG_DATETIME).to_string();
+		all_headers.insert("x-amz-date".to_owned(), date);
+		all_headers.insert("host".to_owned(), host);
+
+		let body_sha = match self.body_signature {
+			BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(),
+			BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)),
+			BodySignature::Streaming(size) => {
+				all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned());
+				all_headers.insert(
+					"x-amz-decoded-content-length".to_owned(),
+					self.body.len().to_string(),
+				);
+				// this is a pretty lazy and inefficient way to do it, but it's enought for
+				// test code.
+				all_headers.insert(
+					"content-length".to_owned(),
+					to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "")
+						.len()
+						.to_string(),
+				);
+
+				"STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned()
+			}
+		};
+		all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone());
+
+		let mut signed_headers = all_headers
+			.iter()
+			.map(|(k, _)| k.as_ref())
+			.collect::<Vec<&str>>();
+		signed_headers.sort();
+		let signed_headers = signed_headers.join(";");
+
+		all_headers.extend(self.unsigned_headers.clone());
+
+		let canonical_request = signature::payload::canonical_request(
+			&self.method,
+			&Uri::try_from(&uri).unwrap(),
+			&all_headers,
+			&signed_headers,
+			&body_sha,
+		);
+
+		let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request);
+
+		signer.update(string_to_sign.as_bytes());
+		let signature = hex::encode(signer.finalize().into_bytes());
+		let authorization = format!(
+			"AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}",
+			self.requester.key.id, scope, signed_headers, signature
+		);
+		all_headers.insert("authorization".to_owned(), authorization);
+
+		let mut request = Request::builder();
+		for (k, v) in all_headers {
+			request = request.header(k, v);
+		}
+
+		let body = if let BodySignature::Streaming(size) = self.body_signature {
+			to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope)
+		} else {
+			self.body.clone()
+		};
+		let request = request
+			.uri(uri)
+			.method(self.method.clone())
+			.body(Body::from(body))
+			.unwrap();
+		self.requester.client.request(request).await
+	}
+}
+
+pub enum BodySignature {
+	Unsigned,
+	Classic,
+	Streaming(usize),
+}
+
+fn query_param_to_string(params: &HashMap<String, Option<String>>) -> String {
+	if params.is_empty() {
+		return String::new();
+	}
+
+	"?".to_owned()
+		+ &params
+			.iter()
+			.map(|(k, v)| {
+				if let Some(v) = v {
+					format!("{}={}", k, v)
+				} else {
+					k.clone()
+				}
+			})
+			.collect::<Vec<String>>()
+			.join("&")
+}
+
+fn to_streaming_body(
+	body: &[u8],
+	chunk_size: usize,
+	mut seed: String,
+	hasher: Hmac<sha2::Sha256>,
+	now: DateTime<Utc>,
+	scope: &str,
+) -> Vec<u8> {
+	const SHA_NULL: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+	let now = now.format(signature::LONG_DATETIME).to_string();
+	let mut res = Vec::with_capacity(body.len());
+	for chunk in body.chunks(chunk_size).chain(std::iter::once(&[][..])) {
+		let to_sign = format!(
+			"AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}",
+			now,
+			scope,
+			seed,
+			SHA_NULL,
+			hex::encode(garage_util::data::sha256sum(chunk))
+		);
+
+		let mut hasher = hasher.clone();
+		hasher.update(to_sign.as_bytes());
+		seed = hex::encode(hasher.finalize().into_bytes());
+
+		let header = format!("{:x};chunk-signature={}\r\n", chunk.len(), seed);
+		res.extend_from_slice(header.as_bytes());
+		res.extend_from_slice(chunk);
+		res.extend_from_slice(b"\r\n");
+	}
+
+	res
+}

src/garage/tests/common/garage.rs

@@ -11,7 +11,7 @@ pub const DEFAULT_PORT: u16 = 49995;
 static GARAGE_TEST_SECRET: &str =
 	"c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44";
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 pub struct Key {
 	pub name: String,
 	pub id: String,

src/garage/tests/common/mod.rs

@@ -5,22 +5,31 @@ use ext::*;
 pub mod macros;
 
 pub mod client;
+pub mod custom_requester;
 pub mod ext;
 pub mod garage;
 
+use custom_requester::CustomRequester;
+
 const REGION: Region = Region::from_static("garage-integ-test");
 
 pub struct Context {
 	pub garage: &'static garage::Instance,
 	pub client: Client,
+	pub custom_request: CustomRequester,
 }
 
 impl Context {
 	fn new() -> Self {
 		let garage = garage::instance();
 		let client = client::build_client(garage);
+		let custom_request = CustomRequester::new(garage);
 
-		Context { garage, client }
+		Context {
+			garage,
+			client,
+			custom_request,
+		}
 	}
 
 	/// Create an unique bucket with a random suffix.

src/garage/tests/lib.rs

@@ -7,4 +7,5 @@ mod list;
 mod multipart;
 mod objects;
 mod simple;
+mod streaming_signature;
 mod website;

src/garage/tests/multipart.rs

@@ -1,6 +1,6 @@
 use crate::common;
 use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
-use aws_sdk_s3::ByteStream;
+use aws_sdk_s3::types::ByteStream;
 
 const SZ_5MB: usize = 5 * 1024 * 1024;
 const SZ_10MB: usize = 10 * 1024 * 1024;

src/garage/tests/objects.rs

@@ -1,6 +1,6 @@
 use crate::common;
 use aws_sdk_s3::model::{Delete, ObjectIdentifier};
-use aws_sdk_s3::ByteStream;
+use aws_sdk_s3::types::ByteStream;
 
 const STD_KEY: &str = "hello world";
 const CTRL_KEY: &str = "\x00\x01\x02\x00";

src/garage/tests/simple.rs

@@ -2,7 +2,7 @@ use crate::common;
 
 #[tokio::test]
 async fn test_simple() {
-	use aws_sdk_s3::ByteStream;
+	use aws_sdk_s3::types::ByteStream;
 
 	let ctx = common::context();
 	let bucket = ctx.create_bucket("test-simple");

src/garage/tests/streaming_signature.rs

@@ -0,0 +1,101 @@
+use std::collections::HashMap;
+
+use crate::common;
+use common::custom_requester::BodySignature;
+use hyper::Method;
+
+const STD_KEY: &str = "hello-world";
+//const CTRL_KEY: &str = "\x00\x01\x02\x00";
+const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+#[tokio::test]
+async fn test_putobject_streaming() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("putobject-streaming");
+
+	{
+		// Send an empty object (can serve as a directory marker)
+		// with a content type
+		let etag = "\"d41d8cd98f00b204e9800998ecf8427e\"";
+		let content_type = "text/csv";
+		let mut headers = HashMap::new();
+		headers.insert("content-type".to_owned(), content_type.to_owned());
+		let _ = ctx
+			.custom_request
+			.builder(bucket.clone())
+			.method(Method::PUT)
+			.path(STD_KEY.to_owned())
+			.unsigned_headers(headers)
+			.vhost_style(true)
+			.body(vec![])
+			.body_signature(BodySignature::Streaming(10))
+			.send()
+			.await
+			.unwrap();
+
+		// assert_eq!(r.e_tag.unwrap().as_str(), etag);
+		// We return a version ID here
+		// We should check if Amazon is returning one when versioning is not enabled
+		// assert!(r.version_id.is_some());
+
+		//let _version = r.version_id.unwrap();
+
+		let o = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.send()
+			.await
+			.unwrap();
+
+		assert_bytes_eq!(o.body, b"");
+		assert_eq!(o.e_tag.unwrap(), etag);
+		// We do not return version ID
+		// We should check if Amazon is returning one when versioning is not enabled
+		// assert_eq!(o.version_id.unwrap(), _version);
+		assert_eq!(o.content_type.unwrap(), content_type);
+		assert!(o.last_modified.is_some());
+		assert_eq!(o.content_length, 0);
+		assert_eq!(o.parts_count, 0);
+		assert_eq!(o.tag_count, 0);
+	}
+
+	{
+		let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
+
+		let _ = ctx
+			.custom_request
+			.builder(bucket.clone())
+			.method(Method::PUT)
+			//.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this
+			//fail
+			.path("abc".to_owned())
+			.vhost_style(true)
+			.body(BODY.to_vec())
+			.body_signature(BodySignature::Streaming(16))
+			.send()
+			.await
+			.unwrap();
+
+		// assert_eq!(r.e_tag.unwrap().as_str(), etag);
+		// assert!(r.version_id.is_some());
+
+		let o = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			//.key(CTRL_KEY)
+			.key("abc")
+			.send()
+			.await
+			.unwrap();
+
+		assert_bytes_eq!(o.body, BODY);
+		assert_eq!(o.e_tag.unwrap(), etag);
+		assert!(o.last_modified.is_some());
+		assert_eq!(o.content_length, 62);
+		assert_eq!(o.parts_count, 0);
+		assert_eq!(o.tag_count, 0);
+	}
+}

src/garage/tests/website.rs

@@ -2,7 +2,7 @@ use crate::common;
 use crate::common::ext::*;
 use aws_sdk_s3::{
 	model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
-	ByteStream,
+	types::ByteStream,
 };
 use http::Request;
 use hyper::{