From 601ae25ad27d99c524691d5284e56b2e61545979 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 29 Nov 2020 16:21:28 +0100 Subject: [PATCH 1/7] Small refactorings --- src/api/s3_put.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index a528720d..6906d758 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -51,12 +51,7 @@ pub async fn handle_put( let md5sum_arr = md5sum.finalize(); let md5sum_hex = hex::encode(md5sum_arr); - let mut sha256sum = Sha256::new(); - sha256sum.input(&first_block[..]); - let sha256sum_arr = sha256sum.result(); - let mut hash = [0u8; 32]; - hash.copy_from_slice(&sha256sum_arr[..]); - let sha256sum_hash = Hash::from(hash); + let sha256sum_hash = hash(&first_block[..]); ensure_checksum_matches( md5sum_arr.as_slice(), @@ -282,7 +277,6 @@ pub fn put_response(version_uuid: UUID, etag: String) -> Response { Response::builder() .header("x-amz-version-id", hex::encode(version_uuid)) .header("ETag", etag) - // TODO ETag .body(Body::from(vec![])) .unwrap() } @@ -369,7 +363,7 @@ pub async fn handle_put_part( } // Copy block to store - let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]); + let version = Version::new(version_uuid, bucket, key, false, vec![]); let first_block_hash = hash(&first_block[..]); let (_, md5sum_arr, sha256sum) = read_and_put_blocks( &garage, From fed97f37e1f0cc2ed8e06f4b76ed0cfcf4a24c97 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 29 Nov 2020 16:38:01 +0100 Subject: [PATCH 2/7] ETag patch --- src/api/s3_put.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index 6906d758..09c3cdbe 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -273,10 +273,10 @@ impl BodyChunker { } } -pub fn put_response(version_uuid: UUID, etag: String) -> Response { +pub fn put_response(version_uuid: UUID, md5sum_hex: String) -> Response { Response::builder() .header("x-amz-version-id", hex::encode(version_uuid)) - .header("ETag", etag) + .header("ETag", format!("\"{}\"", md5sum_hex)) .body(Body::from(vec![])) .unwrap() } @@ -382,7 +382,11 @@ pub async fn handle_put_part( content_sha256, )?; - Ok(Response::new(Body::from(vec![]))) + let response = Response::builder() + .header("ETag", format!("\"{}\"", hex::encode(md5sum_arr))) + .body(Body::from(vec![])) + .unwrap(); + Ok(response) } pub async fn handle_complete_multipart_upload( From 07e87595f8c12a6e60c90b3eb11ab89d7f719420 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 29 Nov 2020 16:48:49 +0100 Subject: [PATCH 3/7] S3 does not support accentuated buckets + add a script to clean tmp --- script/dev-bucket.sh | 6 +++--- script/dev-clean.sh | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100755 script/dev-clean.sh diff --git a/script/dev-bucket.sh b/script/dev-bucket.sh index f07263f5..8c0ef4e4 100755 --- a/script/dev-bucket.sh +++ b/script/dev-bucket.sh @@ -6,11 +6,11 @@ GARAGE_DEBUG="${REPO_FOLDER}/target/debug/" GARAGE_RELEASE="${REPO_FOLDER}/target/release/" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH" -garage bucket create éprouvette +garage bucket create eprouvette KEY_INFO=`garage key new --name opérateur` ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'` SECRET_KEY=`echo $KEY_INFO|grep -Po 'secret_key: "[a-f0-9]+'|grep -Po '[a-f0-9]+$'` -garage bucket allow éprouvette --read --write --key $ACCESS_KEY +garage bucket allow eprouvette --read --write --key $ACCESS_KEY echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3 -echo "Bucket s3://éprouvette created. Credentials stored in /tmp/garage.s3." +echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3." diff --git a/script/dev-clean.sh b/script/dev-clean.sh new file mode 100755 index 00000000..151c5547 --- /dev/null +++ b/script/dev-clean.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -ex + +killall -9 garage || echo "garage is not running" +rm -rf /tmp/garage* +rm -rf /tmp/config.*.toml From 3f18aa6f1def52d61a4fc3bbb667d1f0911665ed Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 29 Nov 2020 17:03:08 +0100 Subject: [PATCH 4/7] Add a smoke test script --- script/dev-cluster.sh | 2 +- script/dev-configure.sh | 5 +++++ script/test-smoke.sh | 22 ++++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100755 script/test-smoke.sh diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh index cfe9be0d..2826d226 100755 --- a/script/dev-cluster.sh +++ b/script/dev-cluster.sh @@ -10,7 +10,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH" FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m") export RUST_BACKTRACE=1 -export RUST_LOG=garage=info +export RUST_LOG=garage=trace MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m" WHICH_GARAGE=$(which garage || exit 1) diff --git a/script/dev-configure.sh b/script/dev-configure.sh index 8b7392c6..698c7ed9 100755 --- a/script/dev-configure.sh +++ b/script/dev-configure.sh @@ -6,6 +6,11 @@ GARAGE_DEBUG="${REPO_FOLDER}/target/debug/" GARAGE_RELEASE="${REPO_FOLDER}/target/release/" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH" +until garage status 2>&1|grep -q Healthy ; do + echo "cluster starting..." + sleep 1 +done + garage status \ | grep UNCONFIGURED \ | grep -Po '^[0-9a-f]+' \ diff --git a/script/test-smoke.sh b/script/test-smoke.sh new file mode 100755 index 00000000..7b462b00 --- /dev/null +++ b/script/test-smoke.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -ex +shopt -s expand_aliases + +SCRIPT_FOLDER="`dirname \"$0\"`" +REPO_FOLDER="${SCRIPT_FOLDER}/../" + +cargo build +${SCRIPT_FOLDER}/dev-clean.sh +${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 & +${SCRIPT_FOLDER}/dev-configure.sh +${SCRIPT_FOLDER}/dev-bucket.sh +source ${SCRIPT_FOLDER}/dev-env.sh + +dd if=/dev/urandom of=/tmp/garage.rnd bs=1M count=10 + +s3grg put /tmp/garage.rnd s3://eprouvette/ +s3grg ls s3://eprouvette +s3grg get s3://eprouvette/garage.rnd /tmp/garage.dl + +diff /tmp/garage.rnd /tmp/garage.dl From d54f15b2c6794eacee82d3900c50dbfb086f8c79 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 29 Nov 2020 17:06:55 +0100 Subject: [PATCH 5/7] Small optimisation --- src/api/s3_put.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index 09c3cdbe..9c4d625c 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -248,7 +248,7 @@ impl BodyChunker { body, read_all: false, block_size, - buf: VecDeque::new(), + buf: VecDeque::with_capacity(2 * block_size), } } async fn next(&mut self) -> Result>, GarageError> { From 13d1b66ba40bd533e53484f24dde6decbec6512e Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 29 Nov 2020 17:07:12 +0100 Subject: [PATCH 6/7] Rollback logging on dev-cluster --- script/dev-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh index 2826d226..cfe9be0d 100755 --- a/script/dev-cluster.sh +++ b/script/dev-cluster.sh @@ -10,7 +10,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH" FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m") export RUST_BACKTRACE=1 -export RUST_LOG=garage=trace +export RUST_LOG=garage=info MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m" WHICH_GARAGE=$(which garage || exit 1) From cee6c3a821fb2ef0f5254f9ad1c0e47dd47047d4 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 29 Nov 2020 17:15:49 +0100 Subject: [PATCH 7/7] A fix for s3cmd --- script/dev-env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/dev-env.sh b/script/dev-env.sh index 7e8ffc50..f5e71004 100755 --- a/script/dev-env.sh +++ b/script/dev-env.sh @@ -11,6 +11,7 @@ SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` alias s3grg="s3cmd \ --host 127.0.0.1:3900 \ + --host-bucket 127.0.0.1:3900 \ --access_key=$ACCESS_KEY \ --secret_key=$SECRET_KEY \ --region=garage \