Add smoke test for UploadPartCopy & fix some bugs

script/dev-cluster.sh

@@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
 FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m")
 
 export RUST_BACKTRACE=1 
-export RUST_LOG=garage=info,garage_api=debug
+export RUST_LOG=garage=info,garage_api=debug,garage_api::signature=trace
 MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
 
 WHICH_GARAGE=$(which garage || exit 1)

script/test-smoke.sh

@@ -52,6 +52,57 @@ if [ -z "$SKIP_AWS" ]; then
   done
 fi
 
+# AWS advanced testing
+if [ -z "$SKIP_AWS" ]; then
+    aws s3 cp "/tmp/garage.3.rnd" "s3://eprouvette/copy_part_source"
+	UPLOAD_ID=$(aws s3api create-multipart-upload --bucket eprouvette --key test_multipart | jq -r .UploadId)
+	PART1=$(aws s3api upload-part \
+		--bucket eprouvette --key test_multipart \
+		--upload-id $UPLOAD_ID --part-number 1 \
+		--body /tmp/garage.2.rnd | jq .ETag)
+	PART2=$(aws s3api upload-part-copy \
+		--bucket eprouvette --key test_multipart \
+		--upload-id $UPLOAD_ID --part-number 2 \
+		--copy-source "/eprouvette/copy_part_source" \
+		--copy-source-range "bytes=500-5000500" \
+		| jq .CopyPartResult.ETag)
+	PART3=$(aws s3api upload-part \
+		--bucket eprouvette --key test_multipart \
+		--upload-id $UPLOAD_ID --part-number 3 \
+		--body /tmp/garage.3.rnd | jq .ETag)
+	cat >/tmp/garage.multipart_struct <<EOF
+{
+  "Parts": [
+    {
+      "ETag": $PART1,
+      "PartNumber": 1
+    },
+    {
+      "ETag": $PART2,
+      "PartNumber": 2
+    },
+    {
+      "ETag": $PART3,
+      "PartNumber": 3
+    }
+  ]
+}
+EOF
+	aws s3api complete-multipart-upload \
+		--bucket eprouvette --key test_multipart --upload-id $UPLOAD_ID \
+		--multipart-upload file:///tmp/garage.multipart_struct
+
+	aws s3 cp "s3://eprouvette/test_multipart" /tmp/garage.test_multipart
+	cat /tmp/garage.2.rnd <(tail -c +501 /tmp/garage.3.rnd | head -c 5000001) /tmp/garage.3.rnd > /tmp/garage.test_multipart_reference
+	diff /tmp/garage.test_multipart /tmp/garage.test_multipart_reference >/tmp/garage.test_multipart_diff 2>&1
+
+	aws s3 rm "s3://eprouvette/copy_part_source"
+	aws s3 rm "s3://eprouvette/test_multipart"
+
+	rm /tmp/garage.multipart_struct
+	rm /tmp/garage.test_multipart
+fi
+
 # S3CMD
 if [ -z "$SKIP_S3CMD" ]; then
   echo "🛠️ Testing with s3cmd"

shell.nix

@@ -83,6 +83,7 @@ function refresh_toolchain {
      pkgs.which
      pkgs.openssl
      pkgs.curl
+     pkgs.jq
     ] else [])
    ++
    (if release then [ 

src/api/s3_copy.rs

@@ -537,6 +537,7 @@ impl CopyPreconditionHeaders {
 			}
 			(None, None, Some(inm), None) => !inm.iter().any(|x| x == etag || x == "*"),
 			(None, None, None, Some(ims)) => v_date > *ims,
+			(None, None, None, None) => true,
 			_ => {
 				return Err(Error::BadRequest(
 					"Invalid combination of x-amz-copy-source-if-xxxxx headers".into(),

src/api/signature.rs

@@ -266,10 +266,13 @@ fn canonical_header_string(headers: &HashMap<String, String>, signed_headers: &s
 	let mut items = headers
 		.iter()
 		.filter(|(key, _)| signed_headers_vec.contains(&key.as_str()))
-		.map(|(key, value)| key.to_lowercase() + ":" + value.trim())
 		.collect::<Vec<_>>();
-	items.sort();
-	items.join("\n")
+	items.sort_by(|(k1, _), (k2, _)| k1.cmp(k2));
+	items
+		.iter()
+		.map(|(key, value)| key.to_lowercase() + ":" + value.trim())
+		.collect::<Vec<_>>()
+		.join("\n")
 }
 
 fn canonical_query_string(uri: &hyper::Uri) -> String {