garage/script/test-smoke.sh

#!/usr/bin/env bash

set -ex

export LC_ALL=C.UTF-8
export LANG=C.UTF-8
SCRIPT_FOLDER="`dirname \"$0\"`"
REPO_FOLDER="${SCRIPT_FOLDER}/../"
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
NIX_RELEASE="${REPO_FOLDER}/result/bin/:${REPO_FOLDER}/result-bin/bin/"
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
CMDOUT=/tmp/garage.cmd.tmp

# @FIXME Duck is not ready for testing, we have a bug
SKIP_DUCK=1

echo "⏳ Setup"
${SCRIPT_FOLDER}/dev-clean.sh
${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
sleep 6
${SCRIPT_FOLDER}/dev-configure.sh
${SCRIPT_FOLDER}/dev-bucket.sh

which garage
garage -c /tmp/config.1.toml status
garage -c /tmp/config.1.toml key list
garage -c /tmp/config.1.toml bucket list

dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline storage (< INLINE_THRESHOLD = 3072 bytes)
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB

dd if=/dev/urandom of=/tmp/garage.part1.rnd bs=1M count=5
dd if=/dev/urandom of=/tmp/garage.part2.rnd bs=1M count=5
dd if=/dev/urandom of=/tmp/garage.part3.rnd bs=1M count=5
dd if=/dev/urandom of=/tmp/garage.part4.rnd bs=1M count=5

# data of lower entropy, to test compression
dd if=/dev/urandom bs=1k count=2  | base64 -w0 > /tmp/garage.1.b64
dd if=/dev/urandom bs=1M count=5  | base64 -w0 > /tmp/garage.2.b64
dd if=/dev/urandom bs=1M count=10 | base64 -w0 > /tmp/garage.3.b64

echo "🧪 S3 API testing..."

# AWS
if [ -z "$SKIP_AWS" ]; then
  echo "🛠️ Testing with awscli (aws s3)"
  source ${SCRIPT_FOLDER}/dev-env-aws.sh
  aws s3 ls
  for idx in {1..3}.{rnd,b64}; do
    aws s3 cp "/tmp/garage.$idx" "s3://eprouvette/&+-é\"/garage.$idx.aws"
    aws s3 ls s3://eprouvette
    aws s3 cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
    diff /tmp/garage.$idx /tmp/garage.$idx.dl
    rm /tmp/garage.$idx.dl
    aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
  done

  echo "🛠️ Testing multipart uploads with awscli (aws s3api)"
  UPLOAD=$(aws s3api create-multipart-upload --bucket eprouvette --key 'upload' | jq -r ".UploadId")
  echo "Upload ID: $UPLOAD"
  ETAG3=$(aws s3api upload-part --bucket eprouvette --key 'upload' \
      --part-number 3 --body "/tmp/garage.part1.rnd" --upload-id "$UPLOAD" \
      | jq -r ".ETag")
  ETAG2=$(aws s3api upload-part --bucket eprouvette --key 'upload' \
      --part-number 2 --body "/tmp/garage.part2.rnd" --upload-id "$UPLOAD" \
      | jq -r ".ETag")
  ETAG3=$(aws s3api upload-part --bucket eprouvette --key 'upload' \
      --part-number 3 --body "/tmp/garage.part3.rnd" --upload-id "$UPLOAD" \
      | jq -r ".ETag")
  ETAG6=$(aws s3api upload-part --bucket eprouvette --key 'upload' \
      --part-number 6 --body "/tmp/garage.part4.rnd" --upload-id "$UPLOAD" \
      | jq -r ".ETag")
  MPU="{\"Parts\":[{\"PartNumber\":2,\"ETag\":$ETAG2}, {\"PartNumber\":3,\"ETag\":$ETAG3}, {\"PartNumber\":6,\"ETag\":$ETAG6}]}"
  echo $MPU > /tmp/garage.mpu.json
  aws s3api complete-multipart-upload --multipart-upload file:///tmp/garage.mpu.json \
      --bucket eprouvette --key 'upload' --upload-id "$UPLOAD"
  aws s3api get-object --bucket eprouvette --key upload /tmp/garage.mpu.get
  if [ "$(md5sum /tmp/garage.mpu.get | cut -d ' ' -f 1)" != "$(cat /tmp/garage.part{2,3,4}.rnd | md5sum | cut -d ' ' -f 1)" ]; then
    echo "Invalid multipart upload"
    exit 1
  fi
  aws s3api delete-object --bucket eprouvette --key upload

  echo "🛠️ Test SSE-C with awscli (aws s3)"
  SSEC_KEY="u8zCfnEyt5Imo/krN+sxA1DQXxLWtPJavU6T6gOVj1Y="
  SSEC_KEY_MD5="jMGbs3GyZkYjJUP6q5jA7g=="
  echo "$SSEC_KEY" |  base64 -d  > /tmp/garage.ssec-key
  for idx in {1,2}.rnd; do
    aws s3 cp --sse-c AES256 --sse-c-key fileb:///tmp/garage.ssec-key \
      "/tmp/garage.$idx" "s3://eprouvette/garage.$idx.aws.sse-c"
    aws s3 cp --sse-c AES256 --sse-c-key fileb:///tmp/garage.ssec-key \
      "s3://eprouvette/garage.$idx.aws.sse-c" "/tmp/garage.$idx.dl.sse-c"
    diff "/tmp/garage.$idx" "/tmp/garage.$idx.dl.sse-c"
    aws s3api delete-object --bucket eprouvette --key "garage.$idx.aws.sse-c"
  done
fi

# S3CMD
if [ -z "$SKIP_S3CMD" ]; then
  echo "🛠️ Testing with s3cmd"
  source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
  s3cmd ls
  for idx in {1..3}.{rnd,b64}; do
    s3cmd put "/tmp/garage.$idx" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
    s3cmd ls s3://eprouvette
    s3cmd get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
    diff /tmp/garage.$idx /tmp/garage.$idx.dl
    rm /tmp/garage.$idx.dl
    s3cmd rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
  done
fi

# Minio Client
if [ -z "$SKIP_MC" ]; then
  echo "🛠️ Testing with mc (minio client)"
  source ${SCRIPT_FOLDER}/dev-env-mc.sh
  mc ls garage/
  for idx in {1..3}.{rnd,b64}; do
    mc cp "/tmp/garage.$idx" "garage/eprouvette/&+-é\"/garage.$idx.mc"
    mc ls garage/eprouvette
    mc cp "garage/eprouvette/&+-é\"/garage.$idx.mc" "/tmp/garage.$idx.dl"
    diff /tmp/garage.$idx /tmp/garage.$idx.dl
    rm /tmp/garage.$idx.dl
    mc rm "garage/eprouvette/&+-é\"/garage.$idx.mc"
  done
fi

# RClone
if [ -z "$SKIP_RCLONE" ]; then
  echo "🛠️ Testing with rclone"
  source ${SCRIPT_FOLDER}/dev-env-rclone.sh
  rclone lsd garage:
  for idx in {1..3}.{rnd,b64}; do
    cp /tmp/garage.$idx /tmp/garage.$idx.dl
    rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/"
    rm /tmp/garage.$idx.dl
    rclone ls garage:eprouvette
    rclone copy "garage:eprouvette/&+-é\"/garage.$idx.dl" "/tmp/"
    diff /tmp/garage.$idx /tmp/garage.$idx.dl
    rm /tmp/garage.$idx.dl
    rclone delete "garage:eprouvette/&+-é\"/garage.$idx.dl"
  done
fi

# Duck (aka Cyberduck CLI)
if [ -z "$SKIP_DUCK" ]; then
  echo "🛠️ Testing with duck (aka cyberduck cli)"
  source ${SCRIPT_FOLDER}/dev-env-duck.sh
  duck --list garage:/
  duck --mkdir "garage:/eprouvette/duck"
  for idx in {1..3}.{rnd,b64}; do
    duck --verbose --upload "garage:/eprouvette/duck/" "/tmp/garage.$idx"
    duck --list garage:/eprouvette/duck/
    duck --download "garage:/eprouvette/duck/garage.$idx" "/tmp/garage.$idx.dl"
    diff /tmp/garage.$idx /tmp/garage.$idx.dl
    rm /tmp/garage.$idx.dl
    duck --delete "garage:/eprouvette/duck/garage.$idx.dk"
  done
fi

if [ -z "$SKIP_WINSCP" ]; then
  echo "🛠️ Testing with winscp"
  source ${SCRIPT_FOLDER}/dev-env-winscp.sh
  winscp <<EOF
open $WINSCP_URL
ls
mkdir eprouvette/winscp
EOF
  for idx in {1..3}.{rnd,b64}; do
    winscp <<EOF
open $WINSCP_URL
put Z:\\tmp\\garage.$idx eprouvette/winscp/garage.$idx.winscp
ls eprouvette/winscp/
get eprouvette/winscp/garage.$idx.winscp Z:\\tmp\\garage.$idx.dl
rm eprouvette/winscp/garage.$idx.winscp
EOF
    diff /tmp/garage.$idx /tmp/garage.$idx.dl
    rm /tmp/garage.$idx.dl
  done
  winscp <<EOF
open $WINSCP_URL
rm eprouvette/winscp
EOF
fi

rm /tmp/garage.part{1..4}.rnd
rm /tmp/garage.{1..3}.{rnd,b64}

echo "🏁 Teardown"
AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
garage -c /tmp/config.1.toml bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID
garage -c /tmp/config.1.toml bucket delete --yes eprouvette
garage -c /tmp/config.1.toml key delete --yes $AWS_ACCESS_KEY_ID
exec 3>&-

echo "✅ Success"