add test framework for arbitraty S3 requests

and implement some basic test with it
This commit is contained in:
trinity-1686a 2022-03-11 17:35:08 +01:00 committed by Gitea
parent e934934f14
commit 1eb7fdb08f
14 changed files with 516 additions and 115 deletions

56
Cargo.lock generated
View file

@ -80,9 +80,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "aws-endpoint"
version = "0.6.0"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6"
checksum = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167"
dependencies = [
"aws-smithy-http",
"aws-types",
@ -93,9 +93,9 @@ dependencies = [
[[package]]
name = "aws-http"
version = "0.6.0"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1"
checksum = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506"
dependencies = [
"aws-smithy-http",
"aws-smithy-types",
@ -108,9 +108,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "0.6.0"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9"
checksum = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7"
dependencies = [
"aws-endpoint",
"aws-http",
@ -133,9 +133,9 @@ dependencies = [
[[package]]
name = "aws-sig-auth"
version = "0.6.0"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc"
checksum = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6"
dependencies = [
"aws-sigv4",
"aws-smithy-eventstream",
@ -148,9 +148,9 @@ dependencies = [
[[package]]
name = "aws-sigv4"
version = "0.6.0"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c"
checksum = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-http",
@ -168,9 +168,9 @@ dependencies = [
[[package]]
name = "aws-smithy-async"
version = "0.36.0"
version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e"
checksum = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8"
dependencies = [
"futures-util",
"pin-project-lite",
@ -180,9 +180,9 @@ dependencies = [
[[package]]
name = "aws-smithy-client"
version = "0.36.0"
version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb"
checksum = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
@ -204,9 +204,9 @@ dependencies = [
[[package]]
name = "aws-smithy-eventstream"
version = "0.36.0"
version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9"
checksum = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"
dependencies = [
"aws-smithy-types",
"bytes 1.1.0",
@ -215,9 +215,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http"
version = "0.36.0"
version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696"
checksum = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-types",
@ -236,9 +236,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http-tower"
version = "0.36.0"
version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d"
checksum = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"
dependencies = [
"aws-smithy-http",
"bytes 1.1.0",
@ -251,9 +251,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "0.36.0"
version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f"
checksum = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2"
dependencies = [
"itoa",
"num-integer",
@ -263,9 +263,9 @@ dependencies = [
[[package]]
name = "aws-smithy-xml"
version = "0.36.0"
version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4"
checksum = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f"
dependencies = [
"thiserror",
"xmlparser",
@ -273,11 +273,12 @@ dependencies = [
[[package]]
name = "aws-types"
version = "0.6.0"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142"
checksum = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38"
dependencies = [
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-types",
"rustc_version",
"tracing",
@ -822,6 +823,7 @@ dependencies = [
"async-trait",
"aws-sdk-s3",
"bytes 1.1.0",
"chrono",
"futures",
"futures-util",
"garage_admin",
@ -833,6 +835,7 @@ dependencies = [
"garage_web",
"git-version",
"hex",
"hmac",
"http",
"hyper",
"kuska-sodiumoxide",
@ -842,6 +845,7 @@ dependencies = [
"rmp-serde 0.15.5",
"serde",
"serde_bytes",
"sha2",
"sled",
"static_init",
"structopt",

150
Cargo.nix
View file

@ -150,29 +150,29 @@ in
src = fetchCratesIo { inherit name version; sha256 = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"; };
});
"registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-endpoint";
version = "0.6.0";
version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6"; };
src = fetchCratesIo { inherit name version; sha256 = "d0990fe9d60185efea41850b10a205f4a9abe71499ec70298b11d2d830130167"; };
dependencies = {
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
regex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.5" { inherit profileName; };
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-http";
version = "0.6.0";
version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1"; };
src = fetchCratesIo { inherit name version; sha256 = "6794b0b27fb74ef2696c41e1be08e916993ef043bbeda7ec554c4f50c3b81506"; };
dependencies = {
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; };
percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
@ -180,29 +180,29 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-sdk-s3";
version = "0.6.0";
version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9"; };
src = fetchCratesIo { inherit name version; sha256 = "986a15277ad7adf67c32059359d60584426b4fa0c30ef34d153bbe47a83cbad7"; };
features = builtins.concatLists [
[ "default" ]
[ "rt-tokio" ]
[ "rustls" ]
];
dependencies = {
aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.6.0" { inherit profileName; };
aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.6.0" { inherit profileName; };
aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" { inherit profileName; };
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; };
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; };
aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" { inherit profileName; };
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
aws_endpoint = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-endpoint."0.8.0" { inherit profileName; };
aws_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-http."0.8.0" { inherit profileName; };
aws_sig_auth = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" { inherit profileName; };
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; };
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; };
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md5."0.7.0" { inherit profileName; };
@ -211,31 +211,31 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.6.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-sig-auth."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-sig-auth";
version = "0.6.0";
version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc"; };
src = fetchCratesIo { inherit name version; sha256 = "3fa501148ae6b5e0de5eeb8c4cf87fa3403d9a00077e543ad64011da781f73a6"; };
features = builtins.concatLists [
[ "aws-smithy-eventstream" ]
[ "sign-eventstream" ]
];
dependencies = {
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" { inherit profileName; };
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" { inherit profileName; };
aws_sigv4 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" { inherit profileName; };
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.6.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-sigv4."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-sigv4";
version = "0.6.0";
version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c"; };
src = fetchCratesIo { inherit name version; sha256 = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"; };
features = builtins.concatLists [
[ "aws-smithy-eventstream" ]
[ "bytes" ]
@ -247,8 +247,8 @@ in
[ "sign-http" ]
];
dependencies = {
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
form_urlencoded = rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; };
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
@ -262,11 +262,11 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-async";
version = "0.36.0";
version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e"; };
src = fetchCratesIo { inherit name version; sha256 = "8ec4efb4a27ced592009787f4f03925f348a5b4a55e6a617e6819788d6cd5ed8"; };
features = builtins.concatLists [
[ "rt-tokio" ]
];
@ -278,11 +278,11 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.36.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-client";
version = "0.36.0";
version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb"; };
src = fetchCratesIo { inherit name version; sha256 = "dad1857eb59d562e82f05c02fbcb9f46c1089301c86770a9798c9e64e5a4677a"; };
features = builtins.concatLists [
[ "client-hyper" ]
[ "hyper" ]
@ -292,10 +292,10 @@ in
[ "rustls" ]
];
dependencies = {
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
fastrand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.7.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
@ -311,23 +311,23 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-eventstream";
version = "0.36.0";
version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9"; };
src = fetchCratesIo { inherit name version; sha256 = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"; };
dependencies = {
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
crc32fast = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-http";
version = "0.36.0";
version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696"; };
src = fetchCratesIo { inherit name version; sha256 = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"; };
features = builtins.concatLists [
[ "aws-smithy-eventstream" ]
[ "event-stream" ]
@ -336,8 +336,8 @@ in
[ "tokio-util" ]
];
dependencies = {
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.36.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
bytes_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.1" { inherit profileName; };
futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
@ -352,13 +352,13 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.36.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-http-tower";
version = "0.36.0";
version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d"; };
src = fetchCratesIo { inherit name version; sha256 = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"; };
dependencies = {
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.36.0" { inherit profileName; };
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; };
@ -368,11 +368,11 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-types";
version = "0.36.0";
version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f"; };
src = fetchCratesIo { inherit name version; sha256 = "dfed653678d1059bed597054c65ce44892aa79cd94444e386d7611843db9f0a2"; };
dependencies = {
itoa = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; };
num_integer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; };
@ -381,25 +381,26 @@ in
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.36.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" = overridableMkRustCrate (profileName: rec {
name = "aws-smithy-xml";
version = "0.36.0";
version = "0.38.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4"; };
src = fetchCratesIo { inherit name version; sha256 = "7aa6c9de6c3f875faabcaaad1fb1f4ef241683bfc22795f731719e3568c3ca9f"; };
dependencies = {
thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
xmlparser = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" { inherit profileName; };
};
});
"registry+https://github.com/rust-lang/crates.io-index".aws-types."0.6.0" = overridableMkRustCrate (profileName: rec {
"registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" = overridableMkRustCrate (profileName: rec {
name = "aws-types";
version = "0.6.0";
version = "0.8.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142"; };
src = fetchCratesIo { inherit name version; sha256 = "b111a0d144e1c570675358d2fae7eb5ddf9010d9db63142fe3bb80353ff65f38"; };
dependencies = {
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.36.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.36.0" { inherit profileName; };
aws_smithy_async = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-async."0.38.0" { inherit profileName; };
aws_smithy_client = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-client."0.38.0" { inherit profileName; };
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
zeroize = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zeroize."1.5.3" { inherit profileName; };
};
@ -1196,9 +1197,12 @@ in
tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
};
devDependencies = {
aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" { inherit profileName; };
aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.8.0" { inherit profileName; };
chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
hmac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.10.1" { inherit profileName; };
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; };
sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; };
static_init = rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.2" { inherit profileName; };
};
});

View file

@ -10,7 +10,8 @@ mod encoding;
mod api_server;
pub use api_server::run_api_server;
mod signature;
/// This mode is public only to help testing. Don't expect stability here
pub mod signature;
pub mod helpers;
mod s3_bucket;

View file

@ -51,8 +51,7 @@ pub async fn check_payload_signature(
let canonical_request = canonical_request(
request.method(),
request.uri().path(),
&canonical_query_string(request.uri()),
request.uri(),
&headers,
&authorization.signed_headers,
&authorization.content_sha256,
@ -215,7 +214,7 @@ fn parse_credential(cred: &str) -> Result<(String, String), Error> {
))
}
fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String {
pub fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String {
let mut hasher = Sha256::default();
hasher.update(canonical_req.as_bytes());
[
@ -227,18 +226,17 @@ fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &
.join("\n")
}
fn canonical_request(
pub fn canonical_request(
method: &Method,
url_path: &str,
canonical_query_string: &str,
uri: &hyper::Uri,
headers: &HashMap<String, String>,
signed_headers: &str,
content_sha256: &str,
) -> String {
[
method.as_str(),
url_path,
canonical_query_string,
&uri.path().to_string(),
&canonical_query_string(uri),
&canonical_header_string(headers, signed_headers),
"",
signed_headers,

View file

@ -55,8 +55,11 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
netapp = "0.4.1"
[dev-dependencies]
aws-sdk-s3 = "0.6"
aws-sdk-s3 = "0.8"
chrono = "0.4"
http = "0.2"
hmac = "0.10"
hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
sha2 = "0.9"
static_init = "1.0"

View file

@ -0,0 +1,280 @@
#![allow(dead_code)]
use std::collections::HashMap;
use std::convert::TryFrom;
use chrono::{offset::Utc, DateTime};
use hmac::{Hmac, Mac};
use hyper::client::HttpConnector;
use hyper::{Body, Client, Method, Request, Response, Uri};
use super::garage::{Instance, Key};
use garage_api::signature;
/// You should ever only use this to send requests AWS sdk won't send,
/// like to reproduce behavior of unusual implementations found to be
/// problematic.
pub struct CustomRequester {
key: Key,
uri: Uri,
client: Client<HttpConnector>,
}
impl CustomRequester {
pub fn new(instance: &Instance) -> Self {
/*
let credentials = Credentials::new(
&instance.key.id,
&instance.key.secret,
None,
None,
"garage-integ-test",
);
let endpoint = Endpoint::immutable(instance.uri());
*/
CustomRequester {
key: instance.key.clone(),
uri: instance.uri(),
client: Client::new(),
}
}
pub fn builder(&self, bucket: String) -> RequestBuilder<'_> {
RequestBuilder {
requester: self,
bucket,
method: Method::GET,
path: String::new(),
query_params: HashMap::new(),
signed_headers: HashMap::new(),
unsigned_headers: HashMap::new(),
body: Vec::new(),
body_signature: BodySignature::Classic,
vhost_style: false,
}
}
/*
pub async fn request(&self, method: &str, path: String, headers: &HashMap<String, (bool, String)>, body: &[u8], vhost_style: bool) -> hyper::Result<Response<Body>> {
let request = Request::builder()
.method(
self.client.request(todo!()).await
}
*/
}
pub struct RequestBuilder<'a> {
requester: &'a CustomRequester,
bucket: String,
method: Method,
path: String,
query_params: HashMap<String, Option<String>>,
signed_headers: HashMap<String, String>,
unsigned_headers: HashMap<String, String>,
body: Vec<u8>,
body_signature: BodySignature,
vhost_style: bool,
}
impl<'a> RequestBuilder<'a> {
pub fn method(&mut self, method: Method) -> &mut Self {
self.method = method;
self
}
pub fn path(&mut self, path: String) -> &mut Self {
self.path = path;
self
}
pub fn query_params(&mut self, query_params: HashMap<String, Option<String>>) -> &mut Self {
self.query_params = query_params;
self
}
pub fn signed_headers(&mut self, signed_headers: HashMap<String, String>) -> &mut Self {
self.signed_headers = signed_headers;
self
}
pub fn unsigned_headers(&mut self, unsigned_headers: HashMap<String, String>) -> &mut Self {
self.unsigned_headers = unsigned_headers;
self
}
pub fn body(&mut self, body: Vec<u8>) -> &mut Self {
self.body = body;
self
}
pub fn body_signature(&mut self, body_signature: BodySignature) -> &mut Self {
self.body_signature = body_signature;
self
}
pub fn vhost_style(&mut self, vhost_style: bool) -> &mut Self {
self.vhost_style = vhost_style;
self
}
pub async fn send(&mut self) -> hyper::Result<Response<Body>> {
// TODO this is a bit incorrect in that path and query params should be url-encoded and
// aren't, but this is good enought for now.
let query = query_param_to_string(&self.query_params);
let (host, path) = if self.vhost_style {
(
format!("{}.s3.garage", self.bucket),
format!("{}{}", self.path, query),
)
} else {
(
"s3.garage".to_owned(),
format!("{}/{}{}", self.bucket, self.path, query),
)
};
let uri = format!("{}{}", self.requester.uri, path);
let now = Utc::now();
let scope = signature::compute_scope(&now, super::REGION.as_ref());
let mut signer = signature::signing_hmac(
&now,
&self.requester.key.secret,
super::REGION.as_ref(),
"s3",
)
.unwrap();
let streaming_signer = signer.clone();
let mut all_headers = self.signed_headers.clone();
let date = now.format(signature::LONG_DATETIME).to_string();
all_headers.insert("x-amz-date".to_owned(), date);
all_headers.insert("host".to_owned(), host);
let body_sha = match self.body_signature {
BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(),
BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)),
BodySignature::Streaming(size) => {
all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned());
all_headers.insert(
"x-amz-decoded-content-length".to_owned(),
self.body.len().to_string(),
);
// this is a pretty lazy and inefficient way to do it, but it's enought for
// test code.
all_headers.insert(
"content-length".to_owned(),
to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "")
.len()
.to_string(),
);
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned()
}
};
all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone());
let mut signed_headers = all_headers
.iter()
.map(|(k, _)| k.as_ref())
.collect::<Vec<&str>>();
signed_headers.sort();
let signed_headers = signed_headers.join(";");
all_headers.extend(self.unsigned_headers.clone());
let canonical_request = signature::payload::canonical_request(
&self.method,
&Uri::try_from(&uri).unwrap(),
&all_headers,
&signed_headers,
&body_sha,
);
let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request);
signer.update(string_to_sign.as_bytes());
let signature = hex::encode(signer.finalize().into_bytes());
let authorization = format!(
"AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}",
self.requester.key.id, scope, signed_headers, signature
);
all_headers.insert("authorization".to_owned(), authorization);
let mut request = Request::builder();
for (k, v) in all_headers {
request = request.header(k, v);
}
let body = if let BodySignature::Streaming(size) = self.body_signature {
to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope)
} else {
self.body.clone()
};
let request = request
.uri(uri)
.method(self.method.clone())
.body(Body::from(body))
.unwrap();
self.requester.client.request(request).await
}
}
pub enum BodySignature {
Unsigned,
Classic,
Streaming(usize),
}
fn query_param_to_string(params: &HashMap<String, Option<String>>) -> String {
if params.is_empty() {
return String::new();
}
"?".to_owned()
+ &params
.iter()
.map(|(k, v)| {
if let Some(v) = v {
format!("{}={}", k, v)
} else {
k.clone()
}
})
.collect::<Vec<String>>()
.join("&")
}
fn to_streaming_body(
body: &[u8],
chunk_size: usize,
mut seed: String,
hasher: Hmac<sha2::Sha256>,
now: DateTime<Utc>,
scope: &str,
) -> Vec<u8> {
const SHA_NULL: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
let now = now.format(signature::LONG_DATETIME).to_string();
let mut res = Vec::with_capacity(body.len());
for chunk in body.chunks(chunk_size).chain(std::iter::once(&[][..])) {
let to_sign = format!(
"AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}",
now,
scope,
seed,
SHA_NULL,
hex::encode(garage_util::data::sha256sum(chunk))
);
let mut hasher = hasher.clone();
hasher.update(to_sign.as_bytes());
seed = hex::encode(hasher.finalize().into_bytes());
let header = format!("{:x};chunk-signature={}\r\n", chunk.len(), seed);
res.extend_from_slice(header.as_bytes());
res.extend_from_slice(chunk);
res.extend_from_slice(b"\r\n");
}
res
}

View file

@ -11,7 +11,7 @@ pub const DEFAULT_PORT: u16 = 49995;
static GARAGE_TEST_SECRET: &str =
"c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44";
#[derive(Debug, Default)]
#[derive(Debug, Default, Clone)]
pub struct Key {
pub name: String,
pub id: String,

View file

@ -5,22 +5,31 @@ use ext::*;
pub mod macros;
pub mod client;
pub mod custom_requester;
pub mod ext;
pub mod garage;
use custom_requester::CustomRequester;
const REGION: Region = Region::from_static("garage-integ-test");
pub struct Context {
pub garage: &'static garage::Instance,
pub client: Client,
pub custom_request: CustomRequester,
}
impl Context {
fn new() -> Self {
let garage = garage::instance();
let client = client::build_client(garage);
let custom_request = CustomRequester::new(garage);
Context { garage, client }
Context {
garage,
client,
custom_request,
}
}
/// Create an unique bucket with a random suffix.

View file

@ -7,4 +7,5 @@ mod list;
mod multipart;
mod objects;
mod simple;
mod streaming_signature;
mod website;

View file

@ -1,6 +1,6 @@
use crate::common;
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::ByteStream;
use aws_sdk_s3::types::ByteStream;
const SZ_5MB: usize = 5 * 1024 * 1024;
const SZ_10MB: usize = 10 * 1024 * 1024;

View file

@ -1,6 +1,6 @@
use crate::common;
use aws_sdk_s3::model::{Delete, ObjectIdentifier};
use aws_sdk_s3::ByteStream;
use aws_sdk_s3::types::ByteStream;
const STD_KEY: &str = "hello world";
const CTRL_KEY: &str = "\x00\x01\x02\x00";

View file

@ -2,7 +2,7 @@ use crate::common;
#[tokio::test]
async fn test_simple() {
use aws_sdk_s3::ByteStream;
use aws_sdk_s3::types::ByteStream;
let ctx = common::context();
let bucket = ctx.create_bucket("test-simple");

View file

@ -0,0 +1,101 @@
use std::collections::HashMap;
use crate::common;
use common::custom_requester::BodySignature;
use hyper::Method;
const STD_KEY: &str = "hello-world";
//const CTRL_KEY: &str = "\x00\x01\x02\x00";
const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
#[tokio::test]
async fn test_putobject_streaming() {
let ctx = common::context();
let bucket = ctx.create_bucket("putobject-streaming");
{
// Send an empty object (can serve as a directory marker)
// with a content type
let etag = "\"d41d8cd98f00b204e9800998ecf8427e\"";
let content_type = "text/csv";
let mut headers = HashMap::new();
headers.insert("content-type".to_owned(), content_type.to_owned());
let _ = ctx
.custom_request
.builder(bucket.clone())
.method(Method::PUT)
.path(STD_KEY.to_owned())
.unsigned_headers(headers)
.vhost_style(true)
.body(vec![])
.body_signature(BodySignature::Streaming(10))
.send()
.await
.unwrap();
// assert_eq!(r.e_tag.unwrap().as_str(), etag);
// We return a version ID here
// We should check if Amazon is returning one when versioning is not enabled
// assert!(r.version_id.is_some());
//let _version = r.version_id.unwrap();
let o = ctx
.client
.get_object()
.bucket(&bucket)
.key(STD_KEY)
.send()
.await
.unwrap();
assert_bytes_eq!(o.body, b"");
assert_eq!(o.e_tag.unwrap(), etag);
// We do not return version ID
// We should check if Amazon is returning one when versioning is not enabled
// assert_eq!(o.version_id.unwrap(), _version);
assert_eq!(o.content_type.unwrap(), content_type);
assert!(o.last_modified.is_some());
assert_eq!(o.content_length, 0);
assert_eq!(o.parts_count, 0);
assert_eq!(o.tag_count, 0);
}
{
let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
let _ = ctx
.custom_request
.builder(bucket.clone())
.method(Method::PUT)
//.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this
//fail
.path("abc".to_owned())
.vhost_style(true)
.body(BODY.to_vec())
.body_signature(BodySignature::Streaming(16))
.send()
.await
.unwrap();
// assert_eq!(r.e_tag.unwrap().as_str(), etag);
// assert!(r.version_id.is_some());
let o = ctx
.client
.get_object()
.bucket(&bucket)
//.key(CTRL_KEY)
.key("abc")
.send()
.await
.unwrap();
assert_bytes_eq!(o.body, BODY);
assert_eq!(o.e_tag.unwrap(), etag);
assert!(o.last_modified.is_some());
assert_eq!(o.content_length, 62);
assert_eq!(o.parts_count, 0);
assert_eq!(o.tag_count, 0);
}
}

View file

@ -2,7 +2,7 @@ use crate::common;
use crate::common::ext::*;
use aws_sdk_s3::{
model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
ByteStream,
types::ByteStream,
};
use http::Request;
use hyper::{