aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorQuentin Dufour <quentin@deuxfleurs.fr>2021-04-28 11:35:10 +0200
committerQuentin Dufour <quentin@deuxfleurs.fr>2021-05-02 14:59:58 +0200
commitef4d6e782a67bd422489d643e8807eaf7fb853c4 (patch)
treeafb0e2c60837e52e808002fc643a2c87ed2c4c93
parenta981244f11af13f3dcebb9f399ab18d7e484014f (diff)
downloadgarage-test/more-clients.tar.gz
garage-test/more-clients.zip
Add minio & rclone to our functional teststest/more-clients
It is now possible to configure which clients you do not want to test with the env variable SKIP_XXX=1, XXX being the client name. eg. SKIP_S3CMD=1 ./script/test-smoke.sh
-rw-r--r--.drone.yml8
-rw-r--r--script/builder_image/Dockerfile3
-rwxr-xr-xscript/dev-clean.sh1
-rwxr-xr-xscript/dev-cluster.sh15
-rw-r--r--script/dev-env-aws.sh11
-rw-r--r--script/dev-env-mc.sh22
-rw-r--r--script/dev-env-rclone.sh21
-rw-r--r--script/dev-env-s3cmd.sh28
-rwxr-xr-xscript/test-smoke.sh133
9 files changed, 161 insertions, 81 deletions
diff --git a/.drone.yml b/.drone.yml
index b0b54792..1c2362ff 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -37,7 +37,7 @@ steps:
- nonexistent_skip_this_step
- name: build
- image: lxpz/garage_builder_amd64:1
+ image: superboum/garage_builder_amd64:3
volumes:
- name: cargo_home
path: /drone/cargo
@@ -49,7 +49,7 @@ steps:
- cargo build
- name: cargo-test
- image: lxpz/garage_builder_amd64:1
+ image: superboum/garage_builder_amd64:3
volumes:
- name: cargo_home
path: /drone/cargo
@@ -85,7 +85,7 @@ steps:
- nonexistent_skip_this_step
- name: smoke-test
- image: lxpz/garage_builder_amd64:1
+ image: superboum/garage_builder_amd64:3
volumes:
- name: cargo_home
path: /drone/cargo
@@ -129,6 +129,6 @@ steps:
---
kind: signature
-hmac: de82026387bd09e547dbc9cc5d232fd865204b4f393d32508c50b58f8e60611d
+hmac: d584c2a15ede6d5702fbe27ae5ae2b2bf7a04461ae7aed2d53cbda83b7fd503e
...
diff --git a/script/builder_image/Dockerfile b/script/builder_image/Dockerfile
index 292d551d..00dea191 100644
--- a/script/builder_image/Dockerfile
+++ b/script/builder_image/Dockerfile
@@ -1,6 +1,7 @@
FROM rust:buster
RUN apt-get update && \
- apt-get install --yes libsodium-dev awscli python-pip && \
+ apt-get install --yes libsodium-dev awscli python-pip wget rclone openssl socat && \
rm -rf /var/lib/apt/lists/*
+RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
RUN rustup component add rustfmt
RUN pip install s3cmd
diff --git a/script/dev-clean.sh b/script/dev-clean.sh
index 151c5547..60065a3d 100755
--- a/script/dev-clean.sh
+++ b/script/dev-clean.sh
@@ -3,5 +3,6 @@
set -ex
killall -9 garage || echo "garage is not running"
+killall -9 socat || echo "socat is not running"
rm -rf /tmp/garage*
rm -rf /tmp/config.*.toml
diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh
index addbf4b4..96147ca0 100755
--- a/script/dev-cluster.sh
+++ b/script/dev-cluster.sh
@@ -47,6 +47,21 @@ EOF
echo -en "$LABEL configuration written to $CONF_PATH\n"
+if [ -z "$SKIP_HTTPS" ]; then
+ echo -en "$LABEL Starting dummy HTTPS reverse proxy\n"
+ mkdir -p /tmp/garagessl
+ openssl req \
+ -new \
+ -x509 \
+ -keyout /tmp/garagessl/test.key \
+ -out /tmp/garagessl/test.crt \
+ -nodes \
+ -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=localhost/emailAddress=X@X.XX" \
+ -addext "subjectAltName = DNS:localhost, IP:127.0.0.1"
+ cat /tmp/garagessl/test.key /tmp/garagessl/test.crt > /tmp/garagessl/test.pem
+ socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
+fi
+
(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
done
diff --git a/script/dev-env-aws.sh b/script/dev-env-aws.sh
index c9a57660..864fd4c2 100644
--- a/script/dev-env-aws.sh
+++ b/script/dev-env-aws.sh
@@ -1,14 +1,9 @@
#!/bin/bash
-SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
-REPO_FOLDER="${SCRIPT_FOLDER}/../"
-GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
-GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
-PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
-
export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
export AWS_DEFAULT_REGION='garage'
-alias awsgrg="aws s3 \
- --endpoint-url http://127.0.0.1:3911"
+function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; }
+
+aws --version
diff --git a/script/dev-env-mc.sh b/script/dev-env-mc.sh
new file mode 100644
index 00000000..98b63047
--- /dev/null
+++ b/script/dev-env-mc.sh
@@ -0,0 +1,22 @@
+ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
+SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
+
+mkdir -p /tmp/garage.mc/certs/CAs
+
+cat > /tmp/garage.mc/config.json <<EOF
+{
+ "version": "10",
+ "aliases": {
+ "garage": {
+ "url": "https://localhost:4443",
+ "accessKey": "$ACCESS_KEY",
+ "secretKey": "$SECRET_KEY",
+ "api": "S3v4",
+ "path": "auto"
+ }
+ }
+}
+EOF
+
+function mc { command mc --insecure --config-dir /tmp/garage.mc $@ ; }
+mc --version
diff --git a/script/dev-env-rclone.sh b/script/dev-env-rclone.sh
new file mode 100644
index 00000000..87933476
--- /dev/null
+++ b/script/dev-env-rclone.sh
@@ -0,0 +1,21 @@
+ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
+SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
+
+cat > /tmp/garage.rclone.conf <<EOF
+[garage]
+type = s3
+provider = Other
+env_auth = false
+access_key_id = $ACCESS_KEY
+secret_access_key = $SECRET_KEY
+endpoint = http://127.0.0.1:3911
+bucket_acl = private
+force_path_style = true
+region = garage
+no_check_bucket = true
+EOF
+# It seems that region is mandatory as rclone does not support redirection
+
+
+function rclone { command rclone --config /tmp/garage.rclone.conf $@ ; }
+rclone --version
diff --git a/script/dev-env-s3cmd.sh b/script/dev-env-s3cmd.sh
index 88d2941f..e97c35e2 100644
--- a/script/dev-env-s3cmd.sh
+++ b/script/dev-env-s3cmd.sh
@@ -1,19 +1,15 @@
-#!/bin/bash
-
-SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
-REPO_FOLDER="${SCRIPT_FOLDER}/../"
-GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
-GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
-PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
-
ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
-alias s3grg="s3cmd \
- --host 127.0.0.1:3911 \
- --host-bucket 127.0.0.1:3911 \
- --access_key=$ACCESS_KEY \
- --secret_key=$SECRET_KEY \
- --region=garage \
- --no-ssl"
-
+cat > /tmp/garage.s3cmd.cfg <<EOF
+[default]
+access_key = $ACCESS_KEY
+secret_key = $SECRET_KEY
+host_base = 127.0.0.1:3911
+host_bucket = 127.0.0.1:3911
+use_https = False
+EOF
+
+function s3cmd { command s3cmd --config=/tmp/garage.s3cmd.cfg $@ ; }
+s3cmd --version
+python3 --version
diff --git a/script/test-smoke.sh b/script/test-smoke.sh
index 16ab3807..e8d41ed3 100755
--- a/script/test-smoke.sh
+++ b/script/test-smoke.sh
@@ -1,22 +1,23 @@
#!/bin/bash
set -ex
-shopt -s expand_aliases
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
SCRIPT_FOLDER="`dirname \"$0\"`"
REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
-echo "setup"
+echo "⏳ Setup"
cargo build
${SCRIPT_FOLDER}/dev-clean.sh
${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
${SCRIPT_FOLDER}/dev-configure.sh
${SCRIPT_FOLDER}/dev-bucket.sh
-source ${SCRIPT_FOLDER}/dev-env-aws.sh
-source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
+which garage
garage status
garage key list
garage bucket list
@@ -25,58 +26,86 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline sto
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB
-echo "s3 api testing..."
-awsgrg --version
-s3cmd --version
-python3 --version
+echo "🧪 S3 API testing..."
+
+# AWS
+if [ -z "$SKIP_AWS" ]; then
+ echo "🛠️ Testing with awscli"
+ source ${SCRIPT_FOLDER}/dev-env-aws.sh
+ for idx in $(seq 1 3); do
+ aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
+ aws s3 ls s3://eprouvette
+ aws s3 cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
+ diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+ rm /tmp/garage.$idx.dl
+ aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
+ done
+fi
+
+# S3CMD
+if [ -z "$SKIP_S3CMD" ]; then
+ echo "🛠️ Testing with s3cmd"
+ source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
+ for idx in $(seq 1 3); do
+ s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
+ s3cmd ls s3://eprouvette
+ s3cmd get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
+ diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+ rm /tmp/garage.$idx.dl
+ s3cmd rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
+ done
+fi
+
+# Minio Client
+if [ -z "$SKIP_MC" ]; then
+ echo "🛠️ Testing with mc (minio client)"
+ source ${SCRIPT_FOLDER}/dev-env-mc.sh
+ for idx in $(seq 1 3); do
+ mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc"
+ mc ls garage/eprouvette
+ mc cp "garage/eprouvette/&+-é\"/garage.$idx.mc" "/tmp/garage.$idx.dl"
+ diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+ rm /tmp/garage.$idx.dl
+ mc rm "garage/eprouvette/&+-é\"/garage.$idx.mc"
+ done
+fi
+
+# RClone
+if [ -z "$SKIP_RCLONE" ]; then
+ echo "🛠️ Testing with rclone"
+ source ${SCRIPT_FOLDER}/dev-env-rclone.sh
+ for idx in $(seq 1 3); do
+ cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+ rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/"
+ rm /tmp/garage.$idx.dl
+ rclone ls garage:eprouvette
+ rclone copy "garage:eprouvette/&+-é\"/garage.$idx.dl" "/tmp/"
+ diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+ rm /tmp/garage.$idx.dl
+ rclone delete "garage:eprouvette/&+-é\"/garage.$idx.dl"
+ done
+fi
-for idx in $(seq 1 3); do
- # AWS sends
- awsgrg cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
-
- awsgrg ls s3://eprouvette
-
- awsgrg cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
- diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
- rm /tmp/garage.$idx.dl
-
- s3grg get "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
- diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
- rm /tmp/garage.$idx.dl
-
- awsgrg rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
-
- # S3CMD sends
- s3grg put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
-
- s3grg ls s3://eprouvette
-
- s3grg get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
- diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
- rm /tmp/garage.$idx.dl
-
- awsgrg cp "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
- diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
- rm /tmp/garage.$idx.dl
-
- s3grg rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
-done
rm /tmp/garage.{1,2,3}.rnd
-echo "website testing"
-echo "<h1>hello world</h1>" > /tmp/garage-index.html
-awsgrg cp /tmp/garage-index.html s3://eprouvette/index.html
-[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
-garage bucket website --allow eprouvette
-[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 200 ]
-garage bucket website --deny eprouvette
-[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
-awsgrg rm s3://eprouvette/index.html
-rm /tmp/garage-index.html
-
-echo "teardown"
+if [ -z "$SKIP_AWS" ]; then
+ echo "🧪 Website Testing"
+ echo "<h1>hello world</h1>" > /tmp/garage-index.html
+ aws s3 cp /tmp/garage-index.html s3://eprouvette/index.html
+ [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
+ garage bucket website --allow eprouvette
+ [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 200 ]
+ garage bucket website --deny eprouvette
+ [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ]
+ aws s3 rm s3://eprouvette/index.html
+ rm /tmp/garage-index.html
+fi
+
+echo "🏁 Teardown"
+AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
+AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
garage bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID
garage bucket delete --yes eprouvette
garage key delete --yes $AWS_ACCESS_KEY_ID
-echo "success"
+echo "✅ Success"