forked from Deuxfleurs/garage
Add minio & rclone to our functional tests
It is now possible to configure which clients you do not want to test with the env variable SKIP_XXX=1, XXX being the client name. eg. SKIP_S3CMD=1 ./script/test-smoke.sh
This commit is contained in:
parent
a981244f11
commit
ef4d6e782a
9 changed files with 155 additions and 75 deletions
|
@ -37,7 +37,7 @@ steps:
|
|||
- nonexistent_skip_this_step
|
||||
|
||||
- name: build
|
||||
image: lxpz/garage_builder_amd64:1
|
||||
image: superboum/garage_builder_amd64:3
|
||||
volumes:
|
||||
- name: cargo_home
|
||||
path: /drone/cargo
|
||||
|
@ -49,7 +49,7 @@ steps:
|
|||
- cargo build
|
||||
|
||||
- name: cargo-test
|
||||
image: lxpz/garage_builder_amd64:1
|
||||
image: superboum/garage_builder_amd64:3
|
||||
volumes:
|
||||
- name: cargo_home
|
||||
path: /drone/cargo
|
||||
|
@ -85,7 +85,7 @@ steps:
|
|||
- nonexistent_skip_this_step
|
||||
|
||||
- name: smoke-test
|
||||
image: lxpz/garage_builder_amd64:1
|
||||
image: superboum/garage_builder_amd64:3
|
||||
volumes:
|
||||
- name: cargo_home
|
||||
path: /drone/cargo
|
||||
|
@ -129,6 +129,6 @@ steps:
|
|||
|
||||
---
|
||||
kind: signature
|
||||
hmac: de82026387bd09e547dbc9cc5d232fd865204b4f393d32508c50b58f8e60611d
|
||||
hmac: d584c2a15ede6d5702fbe27ae5ae2b2bf7a04461ae7aed2d53cbda83b7fd503e
|
||||
|
||||
...
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
FROM rust:buster
|
||||
RUN apt-get update && \
|
||||
apt-get install --yes libsodium-dev awscli python-pip && \
|
||||
apt-get install --yes libsodium-dev awscli python-pip wget rclone openssl socat && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
|
||||
RUN rustup component add rustfmt
|
||||
RUN pip install s3cmd
|
||||
|
|
|
@ -3,5 +3,6 @@
|
|||
set -ex
|
||||
|
||||
killall -9 garage || echo "garage is not running"
|
||||
killall -9 socat || echo "socat is not running"
|
||||
rm -rf /tmp/garage*
|
||||
rm -rf /tmp/config.*.toml
|
||||
|
|
|
@ -47,6 +47,21 @@ EOF
|
|||
|
||||
echo -en "$LABEL configuration written to $CONF_PATH\n"
|
||||
|
||||
if [ -z "$SKIP_HTTPS" ]; then
|
||||
echo -en "$LABEL Starting dummy HTTPS reverse proxy\n"
|
||||
mkdir -p /tmp/garagessl
|
||||
openssl req \
|
||||
-new \
|
||||
-x509 \
|
||||
-keyout /tmp/garagessl/test.key \
|
||||
-out /tmp/garagessl/test.crt \
|
||||
-nodes \
|
||||
-subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=localhost/emailAddress=X@X.XX" \
|
||||
-addext "subjectAltName = DNS:localhost, IP:127.0.0.1"
|
||||
cat /tmp/garagessl/test.key /tmp/garagessl/test.crt > /tmp/garagessl/test.pem
|
||||
socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
|
||||
fi
|
||||
|
||||
(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
|
||||
done
|
||||
|
||||
|
|
|
@ -1,14 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
|
||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
||||
|
||||
export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
||||
export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
||||
export AWS_DEFAULT_REGION='garage'
|
||||
|
||||
alias awsgrg="aws s3 \
|
||||
--endpoint-url http://127.0.0.1:3911"
|
||||
function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; }
|
||||
|
||||
aws --version
|
||||
|
|
22
script/dev-env-mc.sh
Normal file
22
script/dev-env-mc.sh
Normal file
|
@ -0,0 +1,22 @@
|
|||
ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
||||
SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
||||
|
||||
mkdir -p /tmp/garage.mc/certs/CAs
|
||||
|
||||
cat > /tmp/garage.mc/config.json <<EOF
|
||||
{
|
||||
"version": "10",
|
||||
"aliases": {
|
||||
"garage": {
|
||||
"url": "https://localhost:4443",
|
||||
"accessKey": "$ACCESS_KEY",
|
||||
"secretKey": "$SECRET_KEY",
|
||||
"api": "S3v4",
|
||||
"path": "auto"
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
function mc { command mc --insecure --config-dir /tmp/garage.mc $@ ; }
|
||||
mc --version
|
21
script/dev-env-rclone.sh
Normal file
21
script/dev-env-rclone.sh
Normal file
|
@ -0,0 +1,21 @@
|
|||
ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
||||
SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
||||
|
||||
cat > /tmp/garage.rclone.conf <<EOF
|
||||
[garage]
|
||||
type = s3
|
||||
provider = Other
|
||||
env_auth = false
|
||||
access_key_id = $ACCESS_KEY
|
||||
secret_access_key = $SECRET_KEY
|
||||
endpoint = http://127.0.0.1:3911
|
||||
bucket_acl = private
|
||||
force_path_style = true
|
||||
region = garage
|
||||
no_check_bucket = true
|
||||
EOF
|
||||
# It seems that region is mandatory as rclone does not support redirection
|
||||
|
||||
|
||||
function rclone { command rclone --config /tmp/garage.rclone.conf $@ ; }
|
||||
rclone --version
|
|
@ -1,19 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
|
||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
||||
|
||||
ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
||||
SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
||||
|
||||
alias s3grg="s3cmd \
|
||||
--host 127.0.0.1:3911 \
|
||||
--host-bucket 127.0.0.1:3911 \
|
||||
--access_key=$ACCESS_KEY \
|
||||
--secret_key=$SECRET_KEY \
|
||||
--region=garage \
|
||||
--no-ssl"
|
||||
cat > /tmp/garage.s3cmd.cfg <<EOF
|
||||
[default]
|
||||
access_key = $ACCESS_KEY
|
||||
secret_key = $SECRET_KEY
|
||||
host_base = 127.0.0.1:3911
|
||||
host_bucket = 127.0.0.1:3911
|
||||
use_https = False
|
||||
EOF
|
||||
|
||||
function s3cmd { command s3cmd --config=/tmp/garage.s3cmd.cfg $@ ; }
|
||||
s3cmd --version
|
||||
python3 --version
|
||||
|
|
|
@ -1,22 +1,23 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
shopt -s expand_aliases
|
||||
|
||||
export LC_ALL=C.UTF-8
|
||||
export LANG=C.UTF-8
|
||||
SCRIPT_FOLDER="`dirname \"$0\"`"
|
||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
||||
|
||||
echo "setup"
|
||||
echo "⏳ Setup"
|
||||
cargo build
|
||||
${SCRIPT_FOLDER}/dev-clean.sh
|
||||
${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
|
||||
${SCRIPT_FOLDER}/dev-configure.sh
|
||||
${SCRIPT_FOLDER}/dev-bucket.sh
|
||||
source ${SCRIPT_FOLDER}/dev-env-aws.sh
|
||||
source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
|
||||
|
||||
which garage
|
||||
garage status
|
||||
garage key list
|
||||
garage bucket list
|
||||
|
@ -25,58 +26,86 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline sto
|
|||
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
|
||||
dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB
|
||||
|
||||
echo "s3 api testing..."
|
||||
awsgrg --version
|
||||
s3cmd --version
|
||||
python3 --version
|
||||
echo "🧪 S3 API testing..."
|
||||
|
||||
for idx in $(seq 1 3); do
|
||||
# AWS sends
|
||||
awsgrg cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
# AWS
|
||||
if [ -z "$SKIP_AWS" ]; then
|
||||
echo "🛠️ Testing with awscli"
|
||||
source ${SCRIPT_FOLDER}/dev-env-aws.sh
|
||||
for idx in $(seq 1 3); do
|
||||
aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
aws s3 ls s3://eprouvette
|
||||
aws s3 cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
done
|
||||
fi
|
||||
|
||||
awsgrg ls s3://eprouvette
|
||||
# S3CMD
|
||||
if [ -z "$SKIP_S3CMD" ]; then
|
||||
echo "🛠️ Testing with s3cmd"
|
||||
source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
|
||||
for idx in $(seq 1 3); do
|
||||
s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
s3cmd ls s3://eprouvette
|
||||
s3cmd get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
s3cmd rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
done
|
||||
fi
|
||||
|
||||
awsgrg cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
# Minio Client
|
||||
if [ -z "$SKIP_MC" ]; then
|
||||
echo "🛠️ Testing with mc (minio client)"
|
||||
source ${SCRIPT_FOLDER}/dev-env-mc.sh
|
||||
for idx in $(seq 1 3); do
|
||||
mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc"
|
||||
mc ls garage/eprouvette
|
||||
mc cp "garage/eprouvette/&+-é\"/garage.$idx.mc" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
mc rm "garage/eprouvette/&+-é\"/garage.$idx.mc"
|
||||
done
|
||||
fi
|
||||
|
||||
s3grg get "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
# RClone
|
||||
if [ -z "$SKIP_RCLONE" ]; then
|
||||
echo "🛠️ Testing with rclone"
|
||||
source ${SCRIPT_FOLDER}/dev-env-rclone.sh
|
||||
for idx in $(seq 1 3); do
|
||||
cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/"
|
||||
rm /tmp/garage.$idx.dl
|
||||
rclone ls garage:eprouvette
|
||||
rclone copy "garage:eprouvette/&+-é\"/garage.$idx.dl" "/tmp/"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
rclone delete "garage:eprouvette/&+-é\"/garage.$idx.dl"
|
||||
done
|
||||
fi
|
||||
|
||||
awsgrg rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
|
||||
# S3CMD sends
|
||||
s3grg put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
|
||||
s3grg ls s3://eprouvette
|
||||
|
||||
s3grg get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
|
||||
awsgrg cp "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
|
||||
s3grg rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
done
|
||||
rm /tmp/garage.{1,2,3}.rnd
|
||||
|
||||
echo "website testing"
|
||||
echo "<h1>hello world</h1>" > /tmp/garage-index.html
|
||||
awsgrg cp /tmp/garage-index.html s3://eprouvette/index.html
|
||||
[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
|
||||
garage bucket website --allow eprouvette
|
||||
[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 200 ]
|
||||
garage bucket website --deny eprouvette
|
||||
[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
|
||||
awsgrg rm s3://eprouvette/index.html
|
||||
rm /tmp/garage-index.html
|
||||
if [ -z "$SKIP_AWS" ]; then
|
||||
echo "🧪 Website Testing"
|
||||
echo "<h1>hello world</h1>" > /tmp/garage-index.html
|
||||
aws s3 cp /tmp/garage-index.html s3://eprouvette/index.html
|
||||
[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
|
||||
garage bucket website --allow eprouvette
|
||||
[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 200 ]
|
||||
garage bucket website --deny eprouvette
|
||||
[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
|
||||
aws s3 rm s3://eprouvette/index.html
|
||||
rm /tmp/garage-index.html
|
||||
fi
|
||||
|
||||
echo "teardown"
|
||||
echo "🏁 Teardown"
|
||||
AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
||||
AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
||||
garage bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID
|
||||
garage bucket delete --yes eprouvette
|
||||
garage key delete --yes $AWS_ACCESS_KEY_ID
|
||||
|
||||
echo "success"
|
||||
echo "✅ Success"
|
||||
|
|
Loading…
Reference in a new issue