diff options
| -rw-r--r-- | .drone.yml | 8 | ||||
| -rw-r--r-- | script/builder_image/Dockerfile | 3 | ||||
| -rwxr-xr-x | script/dev-clean.sh | 1 | ||||
| -rwxr-xr-x | script/dev-cluster.sh | 15 | ||||
| -rw-r--r-- | script/dev-env-aws.sh | 11 | ||||
| -rw-r--r-- | script/dev-env-mc.sh | 22 | ||||
| -rw-r--r-- | script/dev-env-rclone.sh | 21 | ||||
| -rw-r--r-- | script/dev-env-s3cmd.sh | 28 | ||||
| -rwxr-xr-x | script/test-smoke.sh | 133 |
9 files changed, 161 insertions, 81 deletions
@@ -37,7 +37,7 @@ steps: - nonexistent_skip_this_step - name: build - image: lxpz/garage_builder_amd64:1 + image: superboum/garage_builder_amd64:3 volumes: - name: cargo_home path: /drone/cargo @@ -49,7 +49,7 @@ steps: - cargo build - name: cargo-test - image: lxpz/garage_builder_amd64:1 + image: superboum/garage_builder_amd64:3 volumes: - name: cargo_home path: /drone/cargo @@ -85,7 +85,7 @@ steps: - nonexistent_skip_this_step - name: smoke-test - image: lxpz/garage_builder_amd64:1 + image: superboum/garage_builder_amd64:3 volumes: - name: cargo_home path: /drone/cargo @@ -129,6 +129,6 @@ steps: --- kind: signature -hmac: de82026387bd09e547dbc9cc5d232fd865204b4f393d32508c50b58f8e60611d +hmac: d584c2a15ede6d5702fbe27ae5ae2b2bf7a04461ae7aed2d53cbda83b7fd503e ... diff --git a/script/builder_image/Dockerfile b/script/builder_image/Dockerfile index 292d551d..00dea191 100644 --- a/script/builder_image/Dockerfile +++ b/script/builder_image/Dockerfile @@ -1,6 +1,7 @@ FROM rust:buster RUN apt-get update && \ - apt-get install --yes libsodium-dev awscli python-pip && \ + apt-get install --yes libsodium-dev awscli python-pip wget rclone openssl socat && \ rm -rf /var/lib/apt/lists/* +RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc RUN rustup component add rustfmt RUN pip install s3cmd diff --git a/script/dev-clean.sh b/script/dev-clean.sh index 151c5547..60065a3d 100755 --- a/script/dev-clean.sh +++ b/script/dev-clean.sh @@ -3,5 +3,6 @@ set -ex killall -9 garage || echo "garage is not running" +killall -9 socat || echo "socat is not running" rm -rf /tmp/garage* rm -rf /tmp/config.*.toml diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh index addbf4b4..96147ca0 100755 --- a/script/dev-cluster.sh +++ b/script/dev-cluster.sh @@ -47,6 +47,21 @@ EOF echo -en "$LABEL configuration written to $CONF_PATH\n" +if [ -z "$SKIP_HTTPS" ]; then + echo -en "$LABEL Starting dummy HTTPS reverse proxy\n" + mkdir -p /tmp/garagessl + openssl req \ + -new \ + -x509 \ + -keyout /tmp/garagessl/test.key \ + -out /tmp/garagessl/test.crt \ + -nodes \ + -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=localhost/emailAddress=X@X.XX" \ + -addext "subjectAltName = DNS:localhost, IP:127.0.0.1" + cat /tmp/garagessl/test.key /tmp/garagessl/test.crt > /tmp/garagessl/test.pem + socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 & +fi + (garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) & done diff --git a/script/dev-env-aws.sh b/script/dev-env-aws.sh index c9a57660..864fd4c2 100644 --- a/script/dev-env-aws.sh +++ b/script/dev-env-aws.sh @@ -1,14 +1,9 @@ #!/bin/bash -SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`" -REPO_FOLDER="${SCRIPT_FOLDER}/../" -GARAGE_DEBUG="${REPO_FOLDER}/target/debug/" -GARAGE_RELEASE="${REPO_FOLDER}/target/release/" -PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH" - export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1` export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` export AWS_DEFAULT_REGION='garage' -alias awsgrg="aws s3 \ - --endpoint-url http://127.0.0.1:3911" +function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; } + +aws --version diff --git a/script/dev-env-mc.sh b/script/dev-env-mc.sh new file mode 100644 index 00000000..98b63047 --- /dev/null +++ b/script/dev-env-mc.sh @@ -0,0 +1,22 @@ +ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1` +SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` + +mkdir -p /tmp/garage.mc/certs/CAs + +cat > /tmp/garage.mc/config.json <<EOF +{ + "version": "10", + "aliases": { + "garage": { + "url": "https://localhost:4443", + "accessKey": "$ACCESS_KEY", + "secretKey": "$SECRET_KEY", + "api": "S3v4", + "path": "auto" + } + } +} +EOF + +function mc { command mc --insecure --config-dir /tmp/garage.mc $@ ; } +mc --version diff --git a/script/dev-env-rclone.sh b/script/dev-env-rclone.sh new file mode 100644 index 00000000..87933476 --- /dev/null +++ b/script/dev-env-rclone.sh @@ -0,0 +1,21 @@ +ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1` +SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` + +cat > /tmp/garage.rclone.conf <<EOF +[garage] +type = s3 +provider = Other +env_auth = false +access_key_id = $ACCESS_KEY +secret_access_key = $SECRET_KEY +endpoint = http://127.0.0.1:3911 +bucket_acl = private +force_path_style = true +region = garage +no_check_bucket = true +EOF +# It seems that region is mandatory as rclone does not support redirection + + +function rclone { command rclone --config /tmp/garage.rclone.conf $@ ; } +rclone --version diff --git a/script/dev-env-s3cmd.sh b/script/dev-env-s3cmd.sh index 88d2941f..e97c35e2 100644 --- a/script/dev-env-s3cmd.sh +++ b/script/dev-env-s3cmd.sh @@ -1,19 +1,15 @@ -#!/bin/bash - -SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`" -REPO_FOLDER="${SCRIPT_FOLDER}/../" -GARAGE_DEBUG="${REPO_FOLDER}/target/debug/" -GARAGE_RELEASE="${REPO_FOLDER}/target/release/" -PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH" - ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1` SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` -alias s3grg="s3cmd \ - --host 127.0.0.1:3911 \ - --host-bucket 127.0.0.1:3911 \ - --access_key=$ACCESS_KEY \ - --secret_key=$SECRET_KEY \ - --region=garage \ - --no-ssl" - +cat > /tmp/garage.s3cmd.cfg <<EOF +[default] +access_key = $ACCESS_KEY +secret_key = $SECRET_KEY +host_base = 127.0.0.1:3911 +host_bucket = 127.0.0.1:3911 +use_https = False +EOF + +function s3cmd { command s3cmd --config=/tmp/garage.s3cmd.cfg $@ ; } +s3cmd --version +python3 --version diff --git a/script/test-smoke.sh b/script/test-smoke.sh index 16ab3807..e8d41ed3 100755 --- a/script/test-smoke.sh +++ b/script/test-smoke.sh @@ -1,22 +1,23 @@ #!/bin/bash set -ex -shopt -s expand_aliases export LC_ALL=C.UTF-8 export LANG=C.UTF-8 SCRIPT_FOLDER="`dirname \"$0\"`" REPO_FOLDER="${SCRIPT_FOLDER}/../" +GARAGE_DEBUG="${REPO_FOLDER}/target/debug/" +GARAGE_RELEASE="${REPO_FOLDER}/target/release/" +PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH" -echo "setup" +echo "⏳ Setup" cargo build ${SCRIPT_FOLDER}/dev-clean.sh ${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 & ${SCRIPT_FOLDER}/dev-configure.sh ${SCRIPT_FOLDER}/dev-bucket.sh -source ${SCRIPT_FOLDER}/dev-env-aws.sh -source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh +which garage garage status garage key list garage bucket list @@ -25,58 +26,86 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline sto dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB -echo "s3 api testing..." -awsgrg --version -s3cmd --version -python3 --version +echo "🧪 S3 API testing..." + +# AWS +if [ -z "$SKIP_AWS" ]; then + echo "🛠️ Testing with awscli" + source ${SCRIPT_FOLDER}/dev-env-aws.sh + for idx in $(seq 1 3); do + aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws" + aws s3 ls s3://eprouvette + aws s3 cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl" + diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl + rm /tmp/garage.$idx.dl + aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws" + done +fi + +# S3CMD +if [ -z "$SKIP_S3CMD" ]; then + echo "🛠️ Testing with s3cmd" + source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh + for idx in $(seq 1 3); do + s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" + s3cmd ls s3://eprouvette + s3cmd get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl" + diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl + rm /tmp/garage.$idx.dl + s3cmd rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" + done +fi + +# Minio Client +if [ -z "$SKIP_MC" ]; then + echo "🛠️ Testing with mc (minio client)" + source ${SCRIPT_FOLDER}/dev-env-mc.sh + for idx in $(seq 1 3); do + mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc" + mc ls garage/eprouvette + mc cp "garage/eprouvette/&+-é\"/garage.$idx.mc" "/tmp/garage.$idx.dl" + diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl + rm /tmp/garage.$idx.dl + mc rm "garage/eprouvette/&+-é\"/garage.$idx.mc" + done +fi + +# RClone +if [ -z "$SKIP_RCLONE" ]; then + echo "🛠️ Testing with rclone" + source ${SCRIPT_FOLDER}/dev-env-rclone.sh + for idx in $(seq 1 3); do + cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl + rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/" + rm /tmp/garage.$idx.dl + rclone ls garage:eprouvette + rclone copy "garage:eprouvette/&+-é\"/garage.$idx.dl" "/tmp/" + diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl + rm /tmp/garage.$idx.dl + rclone delete "garage:eprouvette/&+-é\"/garage.$idx.dl" + done +fi -for idx in $(seq 1 3); do - # AWS sends - awsgrg cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws" - - awsgrg ls s3://eprouvette - - awsgrg cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl" - diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl - rm /tmp/garage.$idx.dl - - s3grg get "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl" - diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl - rm /tmp/garage.$idx.dl - - awsgrg rm "s3://eprouvette/&+-é\"/garage.$idx.aws" - - # S3CMD sends - s3grg put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" - - s3grg ls s3://eprouvette - - s3grg get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl" - diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl - rm /tmp/garage.$idx.dl - - awsgrg cp "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl" - diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl - rm /tmp/garage.$idx.dl - - s3grg rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" -done rm /tmp/garage.{1,2,3}.rnd -echo "website testing" -echo "<h1>hello world</h1>" > /tmp/garage-index.html -awsgrg cp /tmp/garage-index.html s3://eprouvette/index.html -[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ] -garage bucket website --allow eprouvette -[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 200 ] -garage bucket website --deny eprouvette -[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ] -awsgrg rm s3://eprouvette/index.html -rm /tmp/garage-index.html - -echo "teardown" +if [ -z "$SKIP_AWS" ]; then + echo "🧪 Website Testing" + echo "<h1>hello world</h1>" > /tmp/garage-index.html + aws s3 cp /tmp/garage-index.html s3://eprouvette/index.html + [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ] + garage bucket website --allow eprouvette + [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 200 ] + garage bucket website --deny eprouvette + [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld" http://127.0.0.1:3923/ ` == 404 ] + aws s3 rm s3://eprouvette/index.html + rm /tmp/garage-index.html +fi + +echo "🏁 Teardown" +AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1` +AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` garage bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID garage bucket delete --yes eprouvette garage key delete --yes $AWS_ACCESS_KEY_ID -echo "success" +echo "✅ Success" |