diff --git a/Cargo.lock b/Cargo.lock index 84ab24ae..f278a361 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -675,6 +675,7 @@ dependencies = [ "git-version", "hex", "http", + "hyper", "kuska-sodiumoxide", "log", "netapp", diff --git a/Cargo.nix b/Cargo.nix index 8855e196..48f23a85 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -560,7 +560,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; }; dependencies = { - ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.115" { inherit profileName; }; + ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.115" { inherit profileName; }; }; }); @@ -999,6 +999,7 @@ in devDependencies = { aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; }; + hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; }; static_init = rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.2" { inherit profileName; }; }; }); @@ -2704,7 +2705,7 @@ in ]; dependencies = { bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.115" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.115" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" { inherit profileName; }; static_init_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; }; diff --git a/script/test-smoke.sh b/script/test-smoke.sh index c992803e..bb939d35 100755 --- a/script/test-smoke.sh +++ b/script/test-smoke.sh @@ -141,295 +141,8 @@ rm eprouvette/winscp EOF fi -# Advanced testing via S3API -if [ -z "$SKIP_AWS" ]; then - echo "🔌 Test S3API" - - echo "Test Objects" - aws s3api put-object --bucket eprouvette --key a - aws s3api put-object --bucket eprouvette --key a/a - aws s3api put-object --bucket eprouvette --key a/b - aws s3api put-object --bucket eprouvette --key a/c - aws s3api put-object --bucket eprouvette --key a/d/a - aws s3api put-object --bucket eprouvette --key a/é - aws s3api put-object --bucket eprouvette --key b - aws s3api put-object --bucket eprouvette --key c - - - aws s3api list-objects-v2 --bucket eprouvette >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 8 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects-v2 --bucket eprouvette --page-size 0 >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 8 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects-v2 --bucket eprouvette --page-size 999999999 >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 8 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects-v2 --bucket eprouvette --page-size 1 >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 8 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects-v2 --bucket eprouvette --delimiter '/' >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 3 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-objects-v2 --bucket eprouvette --delimiter '/' --page-size 1 >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 3 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-objects-v2 --bucket eprouvette --prefix 'a/' >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 5 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects-v2 --bucket eprouvette --prefix 'a/' --delimiter '/' >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 4 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-objects-v2 --bucket eprouvette --prefix 'a/' --page-size 1 >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 5 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects-v2 --bucket eprouvette --prefix 'a/' --delimiter '/' --page-size 1 >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 4 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-objects-v2 --bucket eprouvette --start-after 'Z' >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 8 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects-v2 --bucket eprouvette --start-after 'c' >$CMDOUT - ! [ -s $CMDOUT ] - - - aws s3api list-objects --bucket eprouvette >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 8 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects --bucket eprouvette --page-size 1 >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 8 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects --bucket eprouvette --delimiter '/' >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 3 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - # @FIXME it does not work as expected but might be a limitation of aws s3api - # The problem is the conjunction of a delimiter + pagination + v1 of listobjects - #aws s3api list-objects --bucket eprouvette --delimiter '/' --page-size 1 >$CMDOUT - #[ $(jq '.Contents | length' $CMDOUT) == 3 ] - #[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-objects --bucket eprouvette --prefix 'a/' >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 5 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects --bucket eprouvette --prefix 'a/' --delimiter '/' >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 4 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-objects --bucket eprouvette --prefix 'a/' --page-size 1 >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 5 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - # @FIXME idem - #aws s3api list-objects --bucket eprouvette --prefix 'a/' --delimiter '/' --page-size 1 >$CMDOUT - #[ $(jq '.Contents | length' $CMDOUT) == 4 ] - #[ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-objects --bucket eprouvette --starting-token 'Z' >$CMDOUT - [ $(jq '.Contents | length' $CMDOUT) == 8 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-objects --bucket eprouvette --starting-token 'c' >$CMDOUT - ! [ -s $CMDOUT ] - - aws s3api list-objects-v2 --bucket eprouvette | \ - jq -c '. | {Objects: [.Contents[] | {Key: .Key}], Quiet: true}' | \ - aws s3api delete-objects --bucket eprouvette --delete file:///dev/stdin - - - echo "Test Multipart Upload" - aws s3api create-multipart-upload --bucket eprouvette --key a - aws s3api create-multipart-upload --bucket eprouvette --key a - aws s3api create-multipart-upload --bucket eprouvette --key c - aws s3api create-multipart-upload --bucket eprouvette --key c/a - aws s3api create-multipart-upload --bucket eprouvette --key c/b - - aws s3api list-multipart-uploads --bucket eprouvette >$CMDOUT - [ $(jq '.Uploads | length' $CMDOUT) == 5 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-multipart-uploads --bucket eprouvette --page-size 1 >$CMDOUT - [ $(jq '.Uploads | length' $CMDOUT) == 5 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-multipart-uploads --bucket eprouvette --delimiter '/' >$CMDOUT - [ $(jq '.Uploads | length' $CMDOUT) == 3 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-multipart-uploads --bucket eprouvette --delimiter '/' --page-size 1 >$CMDOUT - [ $(jq '.Uploads | length' $CMDOUT) == 3 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-multipart-uploads --bucket eprouvette --prefix 'c' >$CMDOUT - [ $(jq '.Uploads | length' $CMDOUT) == 3 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-multipart-uploads --bucket eprouvette --prefix 'c' --page-size 1 >$CMDOUT - [ $(jq '.Uploads | length' $CMDOUT) == 3 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-multipart-uploads --bucket eprouvette --prefix 'c' --delimiter '/' >$CMDOUT - [ $(jq '.Uploads | length' $CMDOUT) == 1 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-multipart-uploads --bucket eprouvette --prefix 'c' --delimiter '/' --page-size 1 >$CMDOUT - [ $(jq '.Uploads | length' $CMDOUT) == 1 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 1 ] - aws s3api list-multipart-uploads --bucket eprouvette --starting-token 'ZZZZZ' >$CMDOUT - [ $(jq '.Uploads | length' $CMDOUT) == 5 ] - [ $(jq '.CommonPrefixes | length' $CMDOUT) == 0 ] - aws s3api list-multipart-uploads --bucket eprouvette --starting-token 'd' >$CMDOUT - ! [ -s $CMDOUT ] - - aws s3api list-multipart-uploads --bucket eprouvette | \ - jq -r '.Uploads[] | "\(.Key) \(.UploadId)"' | \ - while read r; do - key=$(echo $r|cut -d' ' -f 1); - uid=$(echo $r|cut -d' ' -f 2); - aws s3api abort-multipart-upload --bucket eprouvette --key $key --upload-id $uid; - echo "Deleted ${key}:${uid}" - done - - echo "Test for ListParts" - UPLOAD_ID=$(aws s3api create-multipart-upload --bucket eprouvette --key list-parts | jq -r .UploadId) - aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT - [ $(jq '.Parts | length' $CMDOUT) == 0 ] - [ $(jq -r '.StorageClass' $CMDOUT) == 'STANDARD' ] # check that the result is not empty - ETAG1=$(aws s3api upload-part --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --part-number 1 --body /tmp/garage.2.rnd | jq .ETag) - aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT - [ $(jq '.Parts | length' $CMDOUT) == 1 ] - [ $(jq '.Parts[0].PartNumber' $CMDOUT) == 1 ] - [ $(jq '.Parts[0].Size' $CMDOUT) == 5242880 ] - [ $(jq '.Parts[0].ETag' $CMDOUT) == $ETAG1 ] - - ETAG2=$(aws s3api upload-part --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --part-number 3 --body /tmp/garage.3.rnd | jq .ETag) - ETAG3=$(aws s3api upload-part --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --part-number 2 --body /tmp/garage.2.rnd | jq .ETag) - aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT - [ $(jq '.Parts | length' $CMDOUT) == 3 ] - [ $(jq '.Parts[1].ETag' $CMDOUT) == $ETAG3 ] - - aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --page-size 1 >$CMDOUT - [ $(jq '.Parts | length' $CMDOUT) == 3 ] - [ $(jq '.Parts[1].ETag' $CMDOUT) == $ETAG3 ] - - cat >/tmp/garage.multipart_struct <$CMDOUT - aws s3 rm "s3://eprouvette/list-parts" - - - # @FIXME We do not write tests with --starting-token due to a bug with awscli - # See here: https://github.com/aws/aws-cli/issues/6666 - - echo "Test for UploadPartCopy" - aws s3 cp "/tmp/garage.3.rnd" "s3://eprouvette/copy_part_source" - UPLOAD_ID=$(aws s3api create-multipart-upload --bucket eprouvette --key test_multipart | jq -r .UploadId) - PART1=$(aws s3api upload-part \ - --bucket eprouvette --key test_multipart \ - --upload-id $UPLOAD_ID --part-number 1 \ - --body /tmp/garage.2.rnd | jq .ETag) - PART2=$(aws s3api upload-part-copy \ - --bucket eprouvette --key test_multipart \ - --upload-id $UPLOAD_ID --part-number 2 \ - --copy-source "/eprouvette/copy_part_source" \ - --copy-source-range "bytes=500-5000500" \ - | jq .CopyPartResult.ETag) - PART3=$(aws s3api upload-part \ - --bucket eprouvette --key test_multipart \ - --upload-id $UPLOAD_ID --part-number 3 \ - --body /tmp/garage.3.rnd | jq .ETag) - cat >/tmp/garage.multipart_struct < /tmp/garage.test_multipart_reference - diff /tmp/garage.test_multipart /tmp/garage.test_multipart_reference >/tmp/garage.test_multipart_diff 2>&1 - - aws s3 rm "s3://eprouvette/copy_part_source" - aws s3 rm "s3://eprouvette/test_multipart" - - rm /tmp/garage.multipart_struct - rm /tmp/garage.test_multipart - rm /tmp/garage.test_multipart_reference - rm /tmp/garage.test_multipart_diff - - - echo "Test CORS endpoints" - garage -c /tmp/config.1.toml bucket website --allow eprouvette - aws s3api put-object --bucket eprouvette --key index.html - CORS='{"CORSRules":[{"AllowedHeaders":["*"],"AllowedMethods":["GET","PUT"],"AllowedOrigins":["*"]}]}' - aws s3api put-bucket-cors --bucket eprouvette --cors-configuration $CORS - [ `aws s3api get-bucket-cors --bucket eprouvette | jq -c` == $CORS ] - - curl -s -i -H 'Origin: http://example.com' --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ | grep access-control-allow-origin - curl -s -i -X OPTIONS -H 'Access-Control-Request-Method: PUT' -H 'Origin: http://example.com' --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ | grep access-control-allow-methods - curl -s -i -X OPTIONS -H 'Access-Control-Request-Method: DELETE' -H 'Origin: http://example.com' --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ | grep '403 Forbidden' - - #@TODO we may want to test the S3 endpoint but we need to handle authentication, which is way more complex. - - aws s3api delete-bucket-cors --bucket eprouvette - ! [ -s `aws s3api get-bucket-cors --bucket eprouvette` ] - curl -s -i -X OPTIONS -H 'Access-Control-Request-Method: PUT' -H 'Origin: http://example.com' --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ | grep '403 Forbidden' - aws s3api delete-object --bucket eprouvette --key index.html - garage -c /tmp/config.1.toml bucket website --deny eprouvette -fi - rm /tmp/garage.{1..3}.{rnd,b64} -if [ -z "$SKIP_AWS" ]; then - echo "🪣 Test bucket logic " - AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1` - [ $(aws s3 ls | wc -l) == 1 ] - garage -c /tmp/config.1.toml bucket create seau - garage -c /tmp/config.1.toml bucket allow --read seau --key $AWS_ACCESS_KEY_ID - [ $(aws s3 ls | wc -l) == 2 ] - garage -c /tmp/config.1.toml bucket deny --read seau --key $AWS_ACCESS_KEY_ID - [ $(aws s3 ls | wc -l) == 1 ] - garage -c /tmp/config.1.toml bucket allow --read seau --key $AWS_ACCESS_KEY_ID - [ $(aws s3 ls | wc -l) == 2 ] - garage -c /tmp/config.1.toml bucket delete --yes seau - [ $(aws s3 ls | wc -l) == 1 ] -fi - -if [ -z "$SKIP_AWS" ]; then - echo "🧪 Website Testing" - echo "

hello world

" > /tmp/garage-index.html - aws s3 cp /tmp/garage-index.html s3://eprouvette/index.html - [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ ` == 404 ] - garage -c /tmp/config.1.toml bucket website --allow eprouvette - [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ ` == 200 ] - garage -c /tmp/config.1.toml bucket website --deny eprouvette - [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.web.garage.localhost" http://127.0.0.1:3921/ ` == 404 ] - aws s3 rm s3://eprouvette/index.html - rm /tmp/garage-index.html -fi - echo "🏁 Teardown" AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1` AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index d6034bbd..463f83e7 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -55,5 +55,6 @@ netapp = "0.3.0" [dev-dependencies] aws-sdk-s3 = "0.6" http = "0.2" +hyper = { version = "0.14", features = ["client", "http1", "runtime"] } static_init = "1.0" diff --git a/src/garage/tests/admin.rs b/src/garage/tests/admin.rs new file mode 100644 index 00000000..37aefe38 --- /dev/null +++ b/src/garage/tests/admin.rs @@ -0,0 +1,74 @@ +use crate::common; +use crate::common::ext::*; + +const BCKT_NAME: &str = "seau"; + +#[tokio::test] +async fn test_admin_bucket_perms() { + let ctx = common::context(); + + let hb = || ctx.client.head_bucket().bucket(BCKT_NAME).send(); + + assert!(hb().await.is_err()); + + ctx.garage + .command() + .args(["bucket", "create", BCKT_NAME]) + .quiet() + .expect_success_status("Could not create bucket"); + + assert!(hb().await.is_err()); + + ctx.garage + .command() + .args([ + "bucket", + "allow", + "--read", + "--key", + &ctx.garage.key.id, + BCKT_NAME, + ]) + .quiet() + .expect_success_status("Could not create bucket"); + + assert!(hb().await.is_ok()); + + ctx.garage + .command() + .args([ + "bucket", + "deny", + "--read", + "--key", + &ctx.garage.key.name, + BCKT_NAME, + ]) + .quiet() + .expect_success_status("Could not create bucket"); + + assert!(hb().await.is_err()); + + ctx.garage + .command() + .args([ + "bucket", + "allow", + "--read", + "--key", + &ctx.garage.key.name, + BCKT_NAME, + ]) + .quiet() + .expect_success_status("Could not create bucket"); + + assert!(hb().await.is_ok()); + + ctx.garage + .command() + .args(["bucket", "delete", "--yes", BCKT_NAME]) + .quiet() + .expect_success_status("Could not delete bucket"); + + assert!(hb().await.is_err()); +} diff --git a/src/garage/tests/bucket.rs b/src/garage/tests/bucket.rs new file mode 100644 index 00000000..1a534042 --- /dev/null +++ b/src/garage/tests/bucket.rs @@ -0,0 +1,87 @@ +use crate::common; +use aws_sdk_s3::model::BucketLocationConstraint; +use aws_sdk_s3::output::DeleteBucketOutput; + +#[tokio::test] +async fn test_bucket_all() { + let ctx = common::context(); + let bucket_name = "hello"; + + { + // Create bucket + //@TODO check with an invalid bucket name + with an already existing bucket + let r = ctx + .client + .create_bucket() + .bucket(bucket_name) + .send() + .await + .unwrap(); + + assert_eq!(r.location.unwrap(), "/hello"); + } + { + // List buckets + let r = ctx.client.list_buckets().send().await.unwrap(); + assert!(r + .buckets + .as_ref() + .unwrap() + .into_iter() + .filter(|x| x.name.as_ref().is_some()) + .find(|x| x.name.as_ref().unwrap() == "hello") + .is_some()); + } + { + // Get its location + let r = ctx + .client + .get_bucket_location() + .bucket(bucket_name) + .send() + .await + .unwrap(); + + match r.location_constraint.unwrap() { + BucketLocationConstraint::Unknown(v) if v.as_str() == "garage-integ-test" => (), + _ => unreachable!("wrong region"), + } + } + { + // (Stub) check GetVersioning + let r = ctx + .client + .get_bucket_versioning() + .bucket(bucket_name) + .send() + .await + .unwrap(); + + assert!(r.status.is_none()); + } + { + // Delete bucket + // @TODO add a check with a non-empty bucket and check failure + let r = ctx + .client + .delete_bucket() + .bucket(bucket_name) + .send() + .await + .unwrap(); + + assert_eq!(r, DeleteBucketOutput::builder().build()); + } + { + // Check bucket is deleted with List buckets + let r = ctx.client.list_buckets().send().await.unwrap(); + assert!(r + .buckets + .as_ref() + .unwrap() + .into_iter() + .filter(|x| x.name.as_ref().is_some()) + .find(|x| x.name.as_ref().unwrap() == "hello") + .is_none()); + } +} diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs index b887856a..92aa2edf 100644 --- a/src/garage/tests/common/garage.rs +++ b/src/garage/tests/common/garage.rs @@ -6,7 +6,7 @@ use std::sync::Once; use super::ext::*; // https://xkcd.com/221/ -const DEFAULT_PORT: u16 = 49995; +pub const DEFAULT_PORT: u16 = 49995; static GARAGE_TEST_SECRET: &str = "c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44"; diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs index 627d4468..9d7e4322 100644 --- a/src/garage/tests/lib.rs +++ b/src/garage/tests/lib.rs @@ -1,4 +1,10 @@ #[macro_use] mod common; +mod admin; +mod bucket; +mod list; +mod multipart; +mod objects; mod simple; +mod website; diff --git a/src/garage/tests/list.rs b/src/garage/tests/list.rs new file mode 100644 index 00000000..6e092a3f --- /dev/null +++ b/src/garage/tests/list.rs @@ -0,0 +1,615 @@ +use crate::common; + +const KEYS: [&str; 8] = ["a", "a/a", "a/b", "a/c", "a/d/a", "a/é", "b", "c"]; +const KEYS_MULTIPART: [&str; 5] = ["a", "a", "c", "c/a", "c/b"]; + +#[tokio::test] +async fn test_listobjectsv2() { + let ctx = common::context(); + let bucket = ctx.create_bucket("listobjectsv2"); + + for k in KEYS { + ctx.client + .put_object() + .bucket(&bucket) + .key(k) + .send() + .await + .unwrap(); + } + + { + // Scoping the variable to avoid reusing it + // in a following assert due to copy paste + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 8); + assert!(r.common_prefixes.is_none()); + } + + //@FIXME aws-sdk-s3 automatically checks max-key values. + // If we set it to zero, it drops it, and it is probably + // the same behavior on values bigger than 1000. + // Boto and awscli do not perform these tests, we should write + // our own minimal library to bypass AWS SDK's tests and be + // sure that we behave correctly. + + { + // With 2 elements + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .max_keys(2) + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 2); + assert!(r.common_prefixes.is_none()); + assert!(r.next_continuation_token.is_some()); + } + + { + // With pagination + let mut cnt = 0; + let mut next = None; + let last_idx = KEYS.len() - 1; + + for i in 0..KEYS.len() { + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .set_continuation_token(next) + .max_keys(1) + .send() + .await + .unwrap(); + + cnt += 1; + next = r.next_continuation_token; + + assert_eq!(r.contents.unwrap().len(), 1); + assert!(r.common_prefixes.is_none()); + if i != last_idx { + assert!(next.is_some()); + } + } + assert_eq!(cnt, KEYS.len()); + } + + { + // With a delimiter + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .delimiter("/") + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 3); + assert_eq!(r.common_prefixes.unwrap().len(), 1); + } + + { + // With a delimiter and pagination + let mut cnt_pfx = 0; + let mut cnt_key = 0; + let mut next = None; + + for _i in 0..KEYS.len() { + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .set_continuation_token(next) + .delimiter("/") + .max_keys(1) + .send() + .await + .unwrap(); + + next = r.next_continuation_token; + match (r.contents, r.common_prefixes) { + (Some(k), None) if k.len() == 1 => cnt_key += 1, + (None, Some(pfx)) if pfx.len() == 1 => cnt_pfx += 1, + _ => unreachable!("logic error"), + }; + if next.is_none() { + break; + } + } + assert_eq!(cnt_key, 3); + assert_eq!(cnt_pfx, 1); + } + + { + // With a prefix + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .prefix("a/") + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 5); + assert!(r.common_prefixes.is_none()); + } + + { + // With a prefix and a delimiter + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .prefix("a/") + .delimiter("/") + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 4); + assert_eq!(r.common_prefixes.unwrap().len(), 1); + } + + { + // With a prefix, a delimiter and max_key + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .prefix("a/") + .delimiter("/") + .max_keys(1) + .send() + .await + .unwrap(); + + assert_eq!(r.contents.as_ref().unwrap().len(), 1); + assert_eq!( + r.contents + .unwrap() + .first() + .unwrap() + .key + .as_ref() + .unwrap() + .as_str(), + "a/a" + ); + assert!(r.common_prefixes.is_none()); + } + { + // With start_after before all keys + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .start_after("Z") + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 8); + assert!(r.common_prefixes.is_none()); + } + { + // With start_after after all keys + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .start_after("c") + .send() + .await + .unwrap(); + + assert!(r.contents.is_none()); + assert!(r.common_prefixes.is_none()); + } +} + +#[tokio::test] +async fn test_listobjectsv1() { + let ctx = common::context(); + let bucket = ctx.create_bucket("listobjects"); + + for k in KEYS { + ctx.client + .put_object() + .bucket(&bucket) + .key(k) + .send() + .await + .unwrap(); + } + + { + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 8); + assert!(r.common_prefixes.is_none()); + } + + { + // With 2 elements + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .max_keys(2) + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 2); + assert!(r.common_prefixes.is_none()); + assert!(r.next_marker.is_some()); + } + + { + // With pagination + let mut cnt = 0; + let mut next = None; + let last_idx = KEYS.len() - 1; + + for i in 0..KEYS.len() { + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .set_marker(next) + .max_keys(1) + .send() + .await + .unwrap(); + + cnt += 1; + next = r.next_marker; + + assert_eq!(r.contents.unwrap().len(), 1); + assert!(r.common_prefixes.is_none()); + if i != last_idx { + assert!(next.is_some()); + } + } + assert_eq!(cnt, KEYS.len()); + } + + { + // With a delimiter + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .delimiter("/") + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 3); + assert_eq!(r.common_prefixes.unwrap().len(), 1); + } + + { + // With a delimiter and pagination + let mut cnt_pfx = 0; + let mut cnt_key = 0; + let mut next = None; + + for _i in 0..KEYS.len() { + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .delimiter("/") + .set_marker(next) + .max_keys(1) + .send() + .await + .unwrap(); + + next = r.next_marker; + match (r.contents, r.common_prefixes) { + (Some(k), None) if k.len() == 1 => cnt_key += 1, + (None, Some(pfx)) if pfx.len() == 1 => cnt_pfx += 1, + _ => unreachable!("logic error"), + }; + if next.is_none() { + break; + } + } + assert_eq!(cnt_key, 3); + // We have no optimization to skip the whole prefix + // on listobjectsv1 so we return the same one 5 times, + // for each element. It is up to the client to merge its result. + // This is compliant with AWS spec. + assert_eq!(cnt_pfx, 5); + } + + { + // With a prefix + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .prefix("a/") + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 5); + assert!(r.common_prefixes.is_none()); + } + + { + // With a prefix and a delimiter + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .prefix("a/") + .delimiter("/") + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 4); + assert_eq!(r.common_prefixes.unwrap().len(), 1); + } + + { + // With a prefix, a delimiter and max_key + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .prefix("a/") + .delimiter("/") + .max_keys(1) + .send() + .await + .unwrap(); + + assert_eq!(r.contents.as_ref().unwrap().len(), 1); + assert_eq!( + r.contents + .unwrap() + .first() + .unwrap() + .key + .as_ref() + .unwrap() + .as_str(), + "a/a" + ); + assert!(r.common_prefixes.is_none()); + } + { + // With marker before all keys + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .marker("Z") + .send() + .await + .unwrap(); + + assert_eq!(r.contents.unwrap().len(), 8); + assert!(r.common_prefixes.is_none()); + } + { + // With start_after after all keys + let r = ctx + .client + .list_objects() + .bucket(&bucket) + .marker("c") + .send() + .await + .unwrap(); + + assert!(r.contents.is_none()); + assert!(r.common_prefixes.is_none()); + } +} + +#[tokio::test] +async fn test_listmultipart() { + let ctx = common::context(); + let bucket = ctx.create_bucket("listmultipartuploads"); + + for k in KEYS_MULTIPART { + ctx.client + .create_multipart_upload() + .bucket(&bucket) + .key(k) + .send() + .await + .unwrap(); + } + + { + // Default + let r = ctx + .client + .list_multipart_uploads() + .bucket(&bucket) + .send() + .await + .unwrap(); + + assert_eq!(r.uploads.unwrap().len(), 5); + assert!(r.common_prefixes.is_none()); + } + { + // With pagination + let mut next = None; + let mut upnext = None; + let last_idx = KEYS_MULTIPART.len() - 1; + + for i in 0..KEYS_MULTIPART.len() { + let r = ctx + .client + .list_multipart_uploads() + .bucket(&bucket) + .set_key_marker(next) + .set_upload_id_marker(upnext) + .max_uploads(1) + .send() + .await + .unwrap(); + + next = r.next_key_marker; + upnext = r.next_upload_id_marker; + + assert_eq!(r.uploads.unwrap().len(), 1); + assert!(r.common_prefixes.is_none()); + if i != last_idx { + assert!(next.is_some()); + } + } + } + { + // With delimiter + let r = ctx + .client + .list_multipart_uploads() + .bucket(&bucket) + .delimiter("/") + .send() + .await + .unwrap(); + + assert_eq!(r.uploads.unwrap().len(), 3); + assert_eq!(r.common_prefixes.unwrap().len(), 1); + } + { + // With delimiter and pagination + let mut next = None; + let mut upnext = None; + let mut upcnt = 0; + let mut pfxcnt = 0; + let mut loopcnt = 0; + + while loopcnt < KEYS_MULTIPART.len() { + let r = ctx + .client + .list_multipart_uploads() + .bucket(&bucket) + .delimiter("/") + .max_uploads(1) + .set_key_marker(next) + .set_upload_id_marker(upnext) + .send() + .await + .unwrap(); + + next = r.next_key_marker; + upnext = r.next_upload_id_marker; + + loopcnt += 1; + upcnt += r.uploads.unwrap_or(vec![]).len(); + pfxcnt += r.common_prefixes.unwrap_or(vec![]).len(); + + if next.is_none() { + break; + } + } + + assert_eq!(upcnt + pfxcnt, loopcnt); + assert_eq!(upcnt, 3); + assert_eq!(pfxcnt, 1); + } + { + // With prefix + let r = ctx + .client + .list_multipart_uploads() + .bucket(&bucket) + .prefix("c") + .send() + .await + .unwrap(); + + assert_eq!(r.uploads.unwrap().len(), 3); + assert!(r.common_prefixes.is_none()); + } + { + // With prefix and delimiter + let r = ctx + .client + .list_multipart_uploads() + .bucket(&bucket) + .prefix("c") + .delimiter("/") + .send() + .await + .unwrap(); + + assert_eq!(r.uploads.unwrap().len(), 1); + assert_eq!(r.common_prefixes.unwrap().len(), 1); + } + { + // With prefix, delimiter and max keys + let r = ctx + .client + .list_multipart_uploads() + .bucket(&bucket) + .prefix("c") + .delimiter("/") + .max_uploads(1) + .send() + .await + .unwrap(); + + assert_eq!(r.uploads.unwrap().len(), 1); + assert!(r.common_prefixes.is_none()); + } + { + // With starting token before the first element + let r = ctx + .client + .list_multipart_uploads() + .bucket(&bucket) + .key_marker("ZZZZZ") + .send() + .await + .unwrap(); + + assert_eq!(r.uploads.unwrap().len(), 5); + assert!(r.common_prefixes.is_none()); + } + { + // With starting token after the last element + let r = ctx + .client + .list_multipart_uploads() + .bucket(&bucket) + .key_marker("d") + .send() + .await + .unwrap(); + + assert!(r.uploads.is_none()); + assert!(r.common_prefixes.is_none()); + } +} diff --git a/src/garage/tests/multipart.rs b/src/garage/tests/multipart.rs new file mode 100644 index 00000000..095c9d34 --- /dev/null +++ b/src/garage/tests/multipart.rs @@ -0,0 +1,415 @@ +use crate::common; +use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart}; +use aws_sdk_s3::ByteStream; + +const SZ_5MB: usize = 5 * 1024 * 1024; +const SZ_10MB: usize = 10 * 1024 * 1024; + +#[tokio::test] +async fn test_uploadlistpart() { + let ctx = common::context(); + let bucket = ctx.create_bucket("uploadpart"); + + let u1 = vec![0xee; SZ_5MB]; + let u2 = vec![0x11; SZ_5MB]; + + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("a") + .send() + .await + .unwrap(); + let uid = up.upload_id.as_ref().unwrap(); + + assert!(up.upload_id.is_some()); + + { + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .unwrap(); + + assert!(r.parts.is_none()); + } + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(2) + .body(ByteStream::from(u1)) + .send() + .await + .unwrap(); + + { + // ListPart on 1st element + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .unwrap(); + + let ps = r.parts.unwrap(); + assert_eq!(ps.len(), 1); + let fp = ps.iter().find(|x| x.part_number == 2).unwrap(); + assert!(fp.last_modified.is_some()); + assert_eq!( + fp.e_tag.as_ref().unwrap(), + "\"3366bb9dcf710d6801b5926467d02e19\"" + ); + assert_eq!(fp.size, SZ_5MB as i64); + } + + let p2 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(1) + .body(ByteStream::from(u2)) + .send() + .await + .unwrap(); + + { + // ListPart on the 2 elements + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .unwrap(); + + let ps = r.parts.unwrap(); + assert_eq!(ps.len(), 2); + let fp = ps.iter().find(|x| x.part_number == 1).unwrap(); + assert!(fp.last_modified.is_some()); + assert_eq!( + fp.e_tag.as_ref().unwrap(), + "\"3c484266f9315485694556e6c693bfa2\"" + ); + assert_eq!(fp.size, SZ_5MB as i64); + } + + { + // Call pagination + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .max_parts(1) + .send() + .await + .unwrap(); + + assert!(r.part_number_marker.is_none()); + assert!(r.next_part_number_marker.is_some()); + assert_eq!(r.max_parts, 1 as i32); + assert!(r.is_truncated); + assert_eq!(r.key.unwrap(), "a"); + assert_eq!(r.upload_id.unwrap().as_str(), uid.as_str()); + assert_eq!(r.parts.unwrap().len(), 1); + + let r2 = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .max_parts(1) + .part_number_marker(r.next_part_number_marker.as_ref().unwrap()) + .send() + .await + .unwrap(); + + assert_eq!( + r2.part_number_marker.as_ref().unwrap(), + r.next_part_number_marker.as_ref().unwrap() + ); + assert_eq!(r2.max_parts, 1 as i32); + assert!(r2.is_truncated); + assert_eq!(r2.key.unwrap(), "a"); + assert_eq!(r2.upload_id.unwrap().as_str(), uid.as_str()); + assert_eq!(r2.parts.unwrap().len(), 1); + } + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(p2.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .build(); + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + + // The multipart upload must not appear anymore + assert!(ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .is_err()); + + { + // The object must appear as a regular object + let r = ctx + .client + .head_object() + .bucket(&bucket) + .key("a") + .send() + .await + .unwrap(); + + assert_eq!(r.content_length, (SZ_5MB * 2) as i64); + } +} + +#[tokio::test] +async fn test_uploadpartcopy() { + let ctx = common::context(); + let bucket = ctx.create_bucket("uploadpartcopy"); + + let u1 = vec![0x11; SZ_10MB]; + let u2 = vec![0x22; SZ_5MB]; + let u3 = vec![0x33; SZ_5MB]; + let u4 = vec![0x44; SZ_5MB]; + let u5 = vec![0x55; SZ_5MB]; + + let overflow = 5500000 - SZ_5MB; + let mut exp_obj = u3.clone(); + exp_obj.extend(&u4[500..]); + exp_obj.extend(&u5[..overflow + 1]); + exp_obj.extend(&u2); + exp_obj.extend(&u1[500..5500000 + 1]); + + // (setup) Upload a single part object + ctx.client + .put_object() + .bucket(&bucket) + .key("source1") + .body(ByteStream::from(u1)) + .send() + .await + .unwrap(); + + // (setup) Upload a multipart object with 2 parts + { + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("source2") + .send() + .await + .unwrap(); + let uid = up.upload_id.as_ref().unwrap(); + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("source2") + .upload_id(uid) + .part_number(1) + .body(ByteStream::from(u4)) + .send() + .await + .unwrap(); + + let p2 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("source2") + .upload_id(uid) + .part_number(2) + .body(ByteStream::from(u5)) + .send() + .await + .unwrap(); + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(p2.e_tag.unwrap()) + .build(), + ) + .build(); + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("source2") + .upload_id(uid) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + } + + // Our multipart object that does copy + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("target") + .send() + .await + .unwrap(); + let uid = up.upload_id.as_ref().unwrap(); + + let p3 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(3) + .body(ByteStream::from(u2)) + .send() + .await + .unwrap(); + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(1) + .body(ByteStream::from(u3)) + .send() + .await + .unwrap(); + + let p2 = ctx + .client + .upload_part_copy() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(2) + .copy_source("uploadpartcopy/source2") + .copy_source_range("bytes=500-5500000") + .send() + .await + .unwrap(); + + let p4 = ctx + .client + .upload_part_copy() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(4) + .copy_source("uploadpartcopy/source1") + .copy_source_range("bytes=500-5500000") + .send() + .await + .unwrap(); + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(p2.copy_part_result.unwrap().e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(3) + .e_tag(p3.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(4) + .e_tag(p4.copy_part_result.unwrap().e_tag.unwrap()) + .build(), + ) + .build(); + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + + // (check) Get object + + let obj = ctx + .client + .get_object() + .bucket(&bucket) + .key("target") + .send() + .await + .unwrap(); + + let real_obj = obj + .body + .collect() + .await + .expect("Error reading data") + .into_bytes(); + + assert_eq!(real_obj.len(), exp_obj.len()); + assert_eq!(real_obj, exp_obj); +} diff --git a/src/garage/tests/objects.rs b/src/garage/tests/objects.rs new file mode 100644 index 00000000..9086073e --- /dev/null +++ b/src/garage/tests/objects.rs @@ -0,0 +1,266 @@ +use crate::common; +use aws_sdk_s3::model::{Delete, ObjectIdentifier}; +use aws_sdk_s3::ByteStream; + +const STD_KEY: &str = "hello world"; +const CTRL_KEY: &str = "\x00\x01\x02\x00"; +const UTF8_KEY: &str = "\u{211D}\u{1F923}\u{1F44B}"; +const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +#[tokio::test] +async fn test_putobject() { + let ctx = common::context(); + let bucket = ctx.create_bucket("putobject"); + + { + // Send an empty object (can serve as a directory marker) + // with a content type + let etag = "\"d41d8cd98f00b204e9800998ecf8427e\""; + let content_type = "text/csv"; + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .content_type(content_type) + .send() + .await + .unwrap(); + + assert_eq!(r.e_tag.unwrap().as_str(), etag); + // We return a version ID here + // We should check if Amazon is returning one when versioning is not enabled + assert!(r.version_id.is_some()); + + let _version = r.version_id.unwrap(); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, b""); + assert_eq!(o.e_tag.unwrap(), etag); + // We do not return version ID + // We should check if Amazon is returning one when versioning is not enabled + // assert_eq!(o.version_id.unwrap(), _version); + assert_eq!(o.content_type.unwrap(), content_type); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length, 0); + assert_eq!(o.parts_count, 0); + assert_eq!(o.tag_count, 0); + } + + { + // Key with control characters, + // no content type and some data + let etag = "\"49f68a5c8493ec2c0bf489821c21fc3b\""; + let data = ByteStream::from_static(b"hi"); + + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(CTRL_KEY) + .body(data) + .send() + .await + .unwrap(); + + assert_eq!(r.e_tag.unwrap().as_str(), etag); + assert!(r.version_id.is_some()); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(CTRL_KEY) + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, b"hi"); + assert_eq!(o.e_tag.unwrap(), etag); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length, 2); + assert_eq!(o.parts_count, 0); + assert_eq!(o.tag_count, 0); + } + + { + // Key with UTF8 codepoints including emoji + let etag = "\"d41d8cd98f00b204e9800998ecf8427e\""; + + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(UTF8_KEY) + .send() + .await + .unwrap(); + + assert_eq!(r.e_tag.unwrap().as_str(), etag); + assert!(r.version_id.is_some()); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(UTF8_KEY) + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, b""); + assert_eq!(o.e_tag.unwrap(), etag); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length, 0); + assert_eq!(o.parts_count, 0); + assert_eq!(o.tag_count, 0); + } +} + +#[tokio::test] +async fn test_getobject() { + let ctx = common::context(); + let bucket = ctx.create_bucket("getobject"); + + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + let data = ByteStream::from_static(BODY); + + let r = ctx + .client + .put_object() + .bucket(&bucket) + .key(STD_KEY) + .body(data) + .send() + .await + .unwrap(); + + assert_eq!(r.e_tag.unwrap().as_str(), etag); + + { + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .range("bytes=1-9") + .send() + .await + .unwrap(); + + assert_eq!(o.content_range.unwrap().as_str(), "bytes 1-9/62"); + assert_bytes_eq!(o.body, &BODY[1..10]); + } + { + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .range("bytes=9-") + .send() + .await + .unwrap(); + assert_eq!(o.content_range.unwrap().as_str(), "bytes 9-61/62"); + assert_bytes_eq!(o.body, &BODY[9..]); + } + { + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .range("bytes=-5") + .send() + .await + .unwrap(); + assert_eq!(o.content_range.unwrap().as_str(), "bytes 57-61/62"); + assert_bytes_eq!(o.body, &BODY[57..]); + } +} + +#[tokio::test] +async fn test_deleteobject() { + let ctx = common::context(); + let bucket = ctx.create_bucket("deleteobject"); + + let mut to_del = Delete::builder(); + + // add content without data + for i in 0..5 { + let k = format!("k-{}", i); + ctx.client + .put_object() + .bucket(&bucket) + .key(k.to_string()) + .send() + .await + .unwrap(); + if i > 0 { + to_del = to_del.objects(ObjectIdentifier::builder().key(k).build()); + } + } + + // add content with data + for i in 0..5 { + let k = format!("l-{}", i); + let data = ByteStream::from_static(BODY); + ctx.client + .put_object() + .bucket(&bucket) + .key(k.to_string()) + .body(data) + .send() + .await + .unwrap(); + + if i > 0 { + to_del = to_del.objects(ObjectIdentifier::builder().key(k).build()); + } + } + + ctx.client + .delete_object() + .bucket(&bucket) + .key("k-0") + .send() + .await + .unwrap(); + + ctx.client + .delete_object() + .bucket(&bucket) + .key("l-0") + .send() + .await + .unwrap(); + + let r = ctx + .client + .delete_objects() + .bucket(&bucket) + .delete(to_del.build()) + .send() + .await + .unwrap(); + + assert_eq!(r.deleted.unwrap().len(), 8); + + let l = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .send() + .await + .unwrap(); + + assert!(l.contents.is_none()); +} diff --git a/src/garage/tests/website.rs b/src/garage/tests/website.rs new file mode 100644 index 00000000..34093a79 --- /dev/null +++ b/src/garage/tests/website.rs @@ -0,0 +1,342 @@ +use crate::common; +use crate::common::ext::*; +use aws_sdk_s3::{ + model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, + ByteStream, +}; +use http::Request; +use hyper::{ + body::{to_bytes, Body}, + Client, +}; + +const BODY: &[u8; 16] = b"

bonjour

"; +const BODY_ERR: &[u8; 6] = b"erreur"; + +#[tokio::test] +async fn test_website() { + const BCKT_NAME: &str = "my-website"; + let ctx = common::context(); + let bucket = ctx.create_bucket(BCKT_NAME); + + let data = ByteStream::from_static(BODY); + + ctx.client + .put_object() + .bucket(&bucket) + .key("index.html") + .body(data) + .send() + .await + .unwrap(); + + let client = Client::new(); + + let req = || { + Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{}/", + common::garage::DEFAULT_PORT + 2 + )) + .header("Host", format!("{}.web.garage", BCKT_NAME)) + .body(Body::empty()) + .unwrap() + }; + + let mut resp = client.request(req()).await.unwrap(); + + assert_eq!(resp.status(), 404); + assert_ne!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY.as_ref() + ); /* check that we do not leak body */ + + ctx.garage + .command() + .args(["bucket", "website", "--allow", BCKT_NAME]) + .quiet() + .expect_success_status("Could not allow website on bucket"); + + resp = client.request(req()).await.unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY.as_ref() + ); + + ctx.garage + .command() + .args(["bucket", "website", "--deny", BCKT_NAME]) + .quiet() + .expect_success_status("Could not deny website on bucket"); + + resp = client.request(req()).await.unwrap(); + assert_eq!(resp.status(), 404); + assert_ne!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY.as_ref() + ); /* check that we do not leak body */ +} + +#[tokio::test] +async fn test_website_s3_api() { + const BCKT_NAME: &str = "my-cors"; + let ctx = common::context(); + let bucket = ctx.create_bucket(BCKT_NAME); + + let data = ByteStream::from_static(BODY); + + ctx.client + .put_object() + .bucket(&bucket) + .key("site/home.html") + .body(data) + .send() + .await + .unwrap(); + + ctx.client + .put_object() + .bucket(&bucket) + .key("err/error.html") + .body(ByteStream::from_static(BODY_ERR)) + .send() + .await + .unwrap(); + + let conf = WebsiteConfiguration::builder() + .index_document(IndexDocument::builder().suffix("home.html").build()) + .error_document(ErrorDocument::builder().key("err/error.html").build()) + .build(); + + ctx.client + .put_bucket_website() + .bucket(&bucket) + .website_configuration(conf) + .send() + .await + .unwrap(); + + let cors = CorsConfiguration::builder() + .cors_rules( + CorsRule::builder() + .id("main-rule") + .allowed_headers("*") + .allowed_methods("GET") + .allowed_methods("PUT") + .allowed_origins("*") + .build(), + ) + .build(); + + ctx.client + .put_bucket_cors() + .bucket(&bucket) + .cors_configuration(cors) + .send() + .await + .unwrap(); + + { + let cors_res = ctx + .client + .get_bucket_cors() + .bucket(&bucket) + .send() + .await + .unwrap(); + + let main_rule = cors_res.cors_rules().unwrap().iter().next().unwrap(); + + assert_eq!(main_rule.id.as_ref().unwrap(), "main-rule"); + assert_eq!( + main_rule.allowed_headers.as_ref().unwrap(), + &vec!["*".to_string()] + ); + assert_eq!( + main_rule.allowed_origins.as_ref().unwrap(), + &vec!["*".to_string()] + ); + assert_eq!( + main_rule.allowed_methods.as_ref().unwrap(), + &vec!["GET".to_string(), "PUT".to_string()] + ); + } + + let client = Client::new(); + + // Test direct requests with CORS + { + let req = Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{}/site/", + common::garage::DEFAULT_PORT + 2 + )) + .header("Host", format!("{}.web.garage", BCKT_NAME)) + .header("Origin", "https://example.com") + .body(Body::empty()) + .unwrap(); + + let mut resp = client.request(req).await.unwrap(); + + assert_eq!(resp.status(), 200); + assert_eq!( + resp.headers().get("access-control-allow-origin").unwrap(), + "*" + ); + assert_eq!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY.as_ref() + ); + } + + // Test ErrorDocument on 404 + { + let req = Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{}/wrong.html", + common::garage::DEFAULT_PORT + 2 + )) + .header("Host", format!("{}.web.garage", BCKT_NAME)) + .body(Body::empty()) + .unwrap(); + + let mut resp = client.request(req).await.unwrap(); + + assert_eq!(resp.status(), 404); + assert_eq!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY_ERR.as_ref() + ); + } + + // Test CORS with an allowed preflight request + { + let req = Request::builder() + .method("OPTIONS") + .uri(format!( + "http://127.0.0.1:{}/site/", + common::garage::DEFAULT_PORT + 2 + )) + .header("Host", format!("{}.web.garage", BCKT_NAME)) + .header("Origin", "https://example.com") + .header("Access-Control-Request-Method", "PUT") + .body(Body::empty()) + .unwrap(); + + let mut resp = client.request(req).await.unwrap(); + + assert_eq!(resp.status(), 200); + assert_eq!( + resp.headers().get("access-control-allow-origin").unwrap(), + "*" + ); + assert_ne!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY.as_ref() + ); + } + + // Test CORS with a forbidden preflight request + { + let req = Request::builder() + .method("OPTIONS") + .uri(format!( + "http://127.0.0.1:{}/site/", + common::garage::DEFAULT_PORT + 2 + )) + .header("Host", format!("{}.web.garage", BCKT_NAME)) + .header("Origin", "https://example.com") + .header("Access-Control-Request-Method", "DELETE") + .body(Body::empty()) + .unwrap(); + + let mut resp = client.request(req).await.unwrap(); + + assert_eq!(resp.status(), 403); + assert_ne!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY.as_ref() + ); + } + + //@TODO test CORS on the S3 endpoint. We need to handle auth manually to check it. + + // Delete cors + ctx.client + .delete_bucket_cors() + .bucket(&bucket) + .send() + .await + .unwrap(); + + // Check CORS are deleted from the API + // @FIXME check what is the expected behavior when GetBucketCors is called on a bucket without + // any CORS. + assert!(ctx + .client + .get_bucket_cors() + .bucket(&bucket) + .send() + .await + .is_err()); + + // Test CORS are not sent anymore on a previously allowed request + { + let req = Request::builder() + .method("OPTIONS") + .uri(format!( + "http://127.0.0.1:{}/site/", + common::garage::DEFAULT_PORT + 2 + )) + .header("Host", format!("{}.web.garage", BCKT_NAME)) + .header("Origin", "https://example.com") + .header("Access-Control-Request-Method", "PUT") + .body(Body::empty()) + .unwrap(); + + let mut resp = client.request(req).await.unwrap(); + + assert_eq!(resp.status(), 403); + assert_ne!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY.as_ref() + ); + } + + // Disallow website from the API + ctx.client + .delete_bucket_website() + .bucket(&bucket) + .send() + .await + .unwrap(); + + // Check that the website is not served anymore + { + let req = Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{}/site/", + common::garage::DEFAULT_PORT + 2 + )) + .header("Host", format!("{}.web.garage", BCKT_NAME)) + .body(Body::empty()) + .unwrap(); + + let mut resp = client.request(req).await.unwrap(); + + assert_eq!(resp.status(), 404); + assert_ne!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY_ERR.as_ref() + ); + assert_ne!( + to_bytes(resp.body_mut()).await.unwrap().as_ref(), + BODY.as_ref() + ); + } +}