From 2ef60b84178ec0d91c5fa534e63ca2b55e51ad67 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 10 Feb 2022 17:02:30 +0100 Subject: [PATCH] Functional test for multipart endpoints --- script/test-smoke.sh | 103 --------- src/garage/tests/multipart.rs | 414 ++++++++++++++++++++++++++++++++++ 2 files changed, 414 insertions(+), 103 deletions(-) diff --git a/script/test-smoke.sh b/script/test-smoke.sh index ffd0b317..dc9b2d2d 100755 --- a/script/test-smoke.sh +++ b/script/test-smoke.sh @@ -143,109 +143,6 @@ fi # Advanced testing via S3API if [ -z "$SKIP_AWS" ]; then - echo "Test for ListParts" - UPLOAD_ID=$(aws s3api create-multipart-upload --bucket eprouvette --key list-parts | jq -r .UploadId) - aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT - [ $(jq '.Parts | length' $CMDOUT) == 0 ] - [ $(jq -r '.StorageClass' $CMDOUT) == 'STANDARD' ] # check that the result is not empty - ETAG1=$(aws s3api upload-part --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --part-number 1 --body /tmp/garage.2.rnd | jq .ETag) - aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT - [ $(jq '.Parts | length' $CMDOUT) == 1 ] - [ $(jq '.Parts[0].PartNumber' $CMDOUT) == 1 ] - [ $(jq '.Parts[0].Size' $CMDOUT) == 5242880 ] - [ $(jq '.Parts[0].ETag' $CMDOUT) == $ETAG1 ] - - ETAG2=$(aws s3api upload-part --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --part-number 3 --body /tmp/garage.3.rnd | jq .ETag) - ETAG3=$(aws s3api upload-part --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --part-number 2 --body /tmp/garage.2.rnd | jq .ETag) - aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID >$CMDOUT - [ $(jq '.Parts | length' $CMDOUT) == 3 ] - [ $(jq '.Parts[1].ETag' $CMDOUT) == $ETAG3 ] - - aws s3api list-parts --bucket eprouvette --key list-parts --upload-id $UPLOAD_ID --page-size 1 >$CMDOUT - [ $(jq '.Parts | length' $CMDOUT) == 3 ] - [ $(jq '.Parts[1].ETag' $CMDOUT) == $ETAG3 ] - - cat >/tmp/garage.multipart_struct <$CMDOUT - aws s3 rm "s3://eprouvette/list-parts" - - - # @FIXME We do not write tests with --starting-token due to a bug with awscli - # See here: https://github.com/aws/aws-cli/issues/6666 - - echo "Test for UploadPartCopy" - aws s3 cp "/tmp/garage.3.rnd" "s3://eprouvette/copy_part_source" - UPLOAD_ID=$(aws s3api create-multipart-upload --bucket eprouvette --key test_multipart | jq -r .UploadId) - PART1=$(aws s3api upload-part \ - --bucket eprouvette --key test_multipart \ - --upload-id $UPLOAD_ID --part-number 1 \ - --body /tmp/garage.2.rnd | jq .ETag) - PART2=$(aws s3api upload-part-copy \ - --bucket eprouvette --key test_multipart \ - --upload-id $UPLOAD_ID --part-number 2 \ - --copy-source "/eprouvette/copy_part_source" \ - --copy-source-range "bytes=500-5000500" \ - | jq .CopyPartResult.ETag) - PART3=$(aws s3api upload-part \ - --bucket eprouvette --key test_multipart \ - --upload-id $UPLOAD_ID --part-number 3 \ - --body /tmp/garage.3.rnd | jq .ETag) - cat >/tmp/garage.multipart_struct < /tmp/garage.test_multipart_reference - diff /tmp/garage.test_multipart /tmp/garage.test_multipart_reference >/tmp/garage.test_multipart_diff 2>&1 - - aws s3 rm "s3://eprouvette/copy_part_source" - aws s3 rm "s3://eprouvette/test_multipart" - - rm /tmp/garage.multipart_struct - rm /tmp/garage.test_multipart - rm /tmp/garage.test_multipart_reference - rm /tmp/garage.test_multipart_diff - - echo "Test CORS endpoints" garage -c /tmp/config.1.toml bucket website --allow eprouvette aws s3api put-object --bucket eprouvette --key index.html diff --git a/src/garage/tests/multipart.rs b/src/garage/tests/multipart.rs index 8b137891..095c9d34 100644 --- a/src/garage/tests/multipart.rs +++ b/src/garage/tests/multipart.rs @@ -1 +1,415 @@ +use crate::common; +use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart}; +use aws_sdk_s3::ByteStream; +const SZ_5MB: usize = 5 * 1024 * 1024; +const SZ_10MB: usize = 10 * 1024 * 1024; + +#[tokio::test] +async fn test_uploadlistpart() { + let ctx = common::context(); + let bucket = ctx.create_bucket("uploadpart"); + + let u1 = vec![0xee; SZ_5MB]; + let u2 = vec![0x11; SZ_5MB]; + + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("a") + .send() + .await + .unwrap(); + let uid = up.upload_id.as_ref().unwrap(); + + assert!(up.upload_id.is_some()); + + { + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .unwrap(); + + assert!(r.parts.is_none()); + } + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(2) + .body(ByteStream::from(u1)) + .send() + .await + .unwrap(); + + { + // ListPart on 1st element + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .unwrap(); + + let ps = r.parts.unwrap(); + assert_eq!(ps.len(), 1); + let fp = ps.iter().find(|x| x.part_number == 2).unwrap(); + assert!(fp.last_modified.is_some()); + assert_eq!( + fp.e_tag.as_ref().unwrap(), + "\"3366bb9dcf710d6801b5926467d02e19\"" + ); + assert_eq!(fp.size, SZ_5MB as i64); + } + + let p2 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .part_number(1) + .body(ByteStream::from(u2)) + .send() + .await + .unwrap(); + + { + // ListPart on the 2 elements + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .unwrap(); + + let ps = r.parts.unwrap(); + assert_eq!(ps.len(), 2); + let fp = ps.iter().find(|x| x.part_number == 1).unwrap(); + assert!(fp.last_modified.is_some()); + assert_eq!( + fp.e_tag.as_ref().unwrap(), + "\"3c484266f9315485694556e6c693bfa2\"" + ); + assert_eq!(fp.size, SZ_5MB as i64); + } + + { + // Call pagination + let r = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .max_parts(1) + .send() + .await + .unwrap(); + + assert!(r.part_number_marker.is_none()); + assert!(r.next_part_number_marker.is_some()); + assert_eq!(r.max_parts, 1 as i32); + assert!(r.is_truncated); + assert_eq!(r.key.unwrap(), "a"); + assert_eq!(r.upload_id.unwrap().as_str(), uid.as_str()); + assert_eq!(r.parts.unwrap().len(), 1); + + let r2 = ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .max_parts(1) + .part_number_marker(r.next_part_number_marker.as_ref().unwrap()) + .send() + .await + .unwrap(); + + assert_eq!( + r2.part_number_marker.as_ref().unwrap(), + r.next_part_number_marker.as_ref().unwrap() + ); + assert_eq!(r2.max_parts, 1 as i32); + assert!(r2.is_truncated); + assert_eq!(r2.key.unwrap(), "a"); + assert_eq!(r2.upload_id.unwrap().as_str(), uid.as_str()); + assert_eq!(r2.parts.unwrap().len(), 1); + } + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(p2.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .build(); + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + + // The multipart upload must not appear anymore + assert!(ctx + .client + .list_parts() + .bucket(&bucket) + .key("a") + .upload_id(uid) + .send() + .await + .is_err()); + + { + // The object must appear as a regular object + let r = ctx + .client + .head_object() + .bucket(&bucket) + .key("a") + .send() + .await + .unwrap(); + + assert_eq!(r.content_length, (SZ_5MB * 2) as i64); + } +} + +#[tokio::test] +async fn test_uploadpartcopy() { + let ctx = common::context(); + let bucket = ctx.create_bucket("uploadpartcopy"); + + let u1 = vec![0x11; SZ_10MB]; + let u2 = vec![0x22; SZ_5MB]; + let u3 = vec![0x33; SZ_5MB]; + let u4 = vec![0x44; SZ_5MB]; + let u5 = vec![0x55; SZ_5MB]; + + let overflow = 5500000 - SZ_5MB; + let mut exp_obj = u3.clone(); + exp_obj.extend(&u4[500..]); + exp_obj.extend(&u5[..overflow + 1]); + exp_obj.extend(&u2); + exp_obj.extend(&u1[500..5500000 + 1]); + + // (setup) Upload a single part object + ctx.client + .put_object() + .bucket(&bucket) + .key("source1") + .body(ByteStream::from(u1)) + .send() + .await + .unwrap(); + + // (setup) Upload a multipart object with 2 parts + { + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("source2") + .send() + .await + .unwrap(); + let uid = up.upload_id.as_ref().unwrap(); + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("source2") + .upload_id(uid) + .part_number(1) + .body(ByteStream::from(u4)) + .send() + .await + .unwrap(); + + let p2 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("source2") + .upload_id(uid) + .part_number(2) + .body(ByteStream::from(u5)) + .send() + .await + .unwrap(); + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(p2.e_tag.unwrap()) + .build(), + ) + .build(); + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("source2") + .upload_id(uid) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + } + + // Our multipart object that does copy + let up = ctx + .client + .create_multipart_upload() + .bucket(&bucket) + .key("target") + .send() + .await + .unwrap(); + let uid = up.upload_id.as_ref().unwrap(); + + let p3 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(3) + .body(ByteStream::from(u2)) + .send() + .await + .unwrap(); + + let p1 = ctx + .client + .upload_part() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(1) + .body(ByteStream::from(u3)) + .send() + .await + .unwrap(); + + let p2 = ctx + .client + .upload_part_copy() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(2) + .copy_source("uploadpartcopy/source2") + .copy_source_range("bytes=500-5500000") + .send() + .await + .unwrap(); + + let p4 = ctx + .client + .upload_part_copy() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .part_number(4) + .copy_source("uploadpartcopy/source1") + .copy_source_range("bytes=500-5500000") + .send() + .await + .unwrap(); + + let cmp = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(p1.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(p2.copy_part_result.unwrap().e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(3) + .e_tag(p3.e_tag.unwrap()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(4) + .e_tag(p4.copy_part_result.unwrap().e_tag.unwrap()) + .build(), + ) + .build(); + + ctx.client + .complete_multipart_upload() + .bucket(&bucket) + .key("target") + .upload_id(uid) + .multipart_upload(cmp) + .send() + .await + .unwrap(); + + // (check) Get object + + let obj = ctx + .client + .get_object() + .bucket(&bucket) + .key("target") + .send() + .await + .unwrap(); + + let real_obj = obj + .body + .collect() + .await + .expect("Error reading data") + .into_bytes(); + + assert_eq!(real_obj.len(), exp_obj.len()); + assert_eq!(real_obj, exp_obj); +}