aboutsummaryrefslogtreecommitdiff
path: root/script
diff options
context:
space:
mode:
Diffstat (limited to 'script')
-rwxr-xr-xscript/dev-bucket.sh17
-rwxr-xr-xscript/dev-cluster.sh20
-rwxr-xr-xscript/dev-configure.sh33
-rw-r--r--script/test-renumbering.sh138
-rw-r--r--script/test-skip-part.sh103
-rwxr-xr-xscript/test-smoke.sh36
-rwxr-xr-xscript/test-upgrade.sh75
7 files changed, 403 insertions, 19 deletions
diff --git a/script/dev-bucket.sh b/script/dev-bucket.sh
index 9bcf5bda..708c2c43 100755
--- a/script/dev-bucket.sh
+++ b/script/dev-bucket.sh
@@ -9,11 +9,22 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
-garage -c /tmp/config.1.toml bucket create eprouvette
-KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur)
+if [ -z "$GARAGE_BIN" ]; then
+ GARAGE_BIN=$(which garage || exit 1)
+ echo -en "Found garage at: ${GARAGE_BIN}\n"
+else
+ echo -en "Using garage binary at: ${GARAGE_BIN}\n"
+fi
+
+$GARAGE_BIN -c /tmp/config.1.toml bucket create eprouvette
+if [ "$GARAGE_08" = "1" ]; then
+ KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key new --name opérateur)
+else
+ KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key create opérateur)
+fi
ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
-garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY
+$GARAGE_BIN -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY
echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."
diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh
index c7fbe08d..6b39255a 100755
--- a/script/dev-cluster.sh
+++ b/script/dev-cluster.sh
@@ -11,11 +11,16 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m")
export RUST_BACKTRACE=1
-export RUST_LOG=garage=info,garage_api=debug,netapp=trace
+export RUST_LOG=garage=info,garage_api=debug
MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
-WHICH_GARAGE=$(which garage || exit 1)
-echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n"
+if [ -z "$GARAGE_BIN" ]; then
+ GARAGE_BIN=$(which garage || exit 1)
+ echo -en "${MAIN_LABEL} Found garage at: ${GARAGE_BIN}\n"
+else
+ echo -en "${MAIN_LABEL} Using garage binary at: ${GARAGE_BIN}\n"
+fi
+$GARAGE_BIN --version
NETWORK_SECRET="$(openssl rand -hex 32)"
@@ -28,6 +33,7 @@ LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
cat > $CONF_PATH <<EOF
block_size = 1048576 # objects are split in blocks of maximum this number of bytes
metadata_dir = "/tmp/garage-meta-$count"
+db_engine = "lmdb"
data_dir = "/tmp/garage-data-$count"
rpc_bind_addr = "0.0.0.0:$((3900+$count))" # the port other Garage nodes will use to talk to this node
rpc_public_addr = "127.0.0.1:$((3900+$count))"
@@ -51,7 +57,7 @@ EOF
echo -en "$LABEL configuration written to $CONF_PATH\n"
-(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
+($GARAGE_BIN -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
done
# >>>>>>>>>>>>>>>> END FOR LOOP ON NODES
@@ -73,14 +79,14 @@ fi
sleep 3
# Establish connections between nodes
for count in $(seq 1 3); do
- NODE=$(garage -c /tmp/config.$count.toml node id -q)
+ NODE=$($GARAGE_BIN -c /tmp/config.$count.toml node id -q)
for count2 in $(seq 1 3); do
- garage -c /tmp/config.$count2.toml node connect $NODE
+ $GARAGE_BIN -c /tmp/config.$count2.toml node connect $NODE
done
done
RETRY=120
-until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
+until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
(( RETRY-- ))
if (( RETRY <= 0 )); then
echo -en "${MAIN_LABEL} Garage did not start"
diff --git a/script/dev-configure.sh b/script/dev-configure.sh
index f0a7843d..0649cdbe 100755
--- a/script/dev-configure.sh
+++ b/script/dev-configure.sh
@@ -9,9 +9,17 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
+if [ -z "$GARAGE_BIN" ]; then
+ GARAGE_BIN=$(which garage || exit 1)
+ echo -en "Found garage at: ${GARAGE_BIN}\n"
+else
+ echo -en "Using garage binary at: ${GARAGE_BIN}\n"
+fi
+$GARAGE_BIN --version
+
sleep 5
RETRY=120
-until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
+until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
(( RETRY-- ))
if (( RETRY <= 0 )); then
echo "garage did not start in time, failing."
@@ -21,11 +29,20 @@ until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
sleep 1
done
-garage -c /tmp/config.1.toml status \
- | grep 'NO ROLE' \
- | grep -Po '^[0-9a-f]+' \
- | while read id; do
- garage -c /tmp/config.1.toml layout assign $id -z dc1 -c 1
- done
+if [ "$GARAGE_08" = "1" ]; then
+ $GARAGE_BIN -c /tmp/config.1.toml status \
+ | grep 'NO ROLE' \
+ | grep -Po '^[0-9a-f]+' \
+ | while read id; do
+ $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1
+ done
+else
+ $GARAGE_BIN -c /tmp/config.1.toml status \
+ | grep 'NO ROLE' \
+ | grep -Po '^[0-9a-f]+' \
+ | while read id; do
+ $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1G
+ done
+fi
-garage -c /tmp/config.1.toml layout apply --version 1
+$GARAGE_BIN -c /tmp/config.1.toml layout apply --version 1
diff --git a/script/test-renumbering.sh b/script/test-renumbering.sh
new file mode 100644
index 00000000..0b1bd320
--- /dev/null
+++ b/script/test-renumbering.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+: '
+ This script tests part renumbering on an S3 remote (here configured for Minio).
+
+ On Minio:
+
+ The results confirm that if I upload parts with number 1, 4, 5 and 6,
+ they are renumbered to 1, 2, 3 and 4 after CompleteMultipartUpload.
+ Thus, specifying partNumber=4 on a GetObject/HeadObject should return
+ information on the part I originally uploaded with part number
+
+ On S3: not tested
+
+ Sample output (on Minio):
+
+ f07e1404cc527d494242824ded3a616b part1
+ 78974cd4d0f622eb3426ea7cd22f5a1c part4
+ f9cc379f8baa61645558d9ba7e6351fa part5
+ 1bd2383eebbac1f8e7143575ba5b1f4a part6
+ Upload ID: 6838b813-d0ca-400b-9d28-ec8b2b5cd004
+ PART 1 ETag: "f07e1404cc527d494242824ded3a616b"
+ PART 4 ETag: "78974cd4d0f622eb3426ea7cd22f5a1c"
+ PART 5 ETag: "f9cc379f8baa61645558d9ba7e6351fa"
+ PART 6 ETag: "1bd2383eebbac1f8e7143575ba5b1f4a"
+ ======================================== LIST ====
+ {
+ "Parts": [
+ {
+ "PartNumber": 1,
+ "LastModified": "2023-04-25T10:21:54.350000+00:00",
+ "ETag": "\"f07e1404cc527d494242824ded3a616b\"",
+ "Size": 20971520
+ },
+ {
+ "PartNumber": 4,
+ "LastModified": "2023-04-25T10:21:54.350000+00:00",
+ "ETag": "\"78974cd4d0f622eb3426ea7cd22f5a1c\"",
+ "Size": 20971520
+ },
+ {
+ "PartNumber": 5,
+ "LastModified": "2023-04-25T10:21:54.350000+00:00",
+ "ETag": "\"f9cc379f8baa61645558d9ba7e6351fa\"",
+ "Size": 20971520
+ },
+ {
+ "PartNumber": 6,
+ "LastModified": "2023-04-25T10:21:54.350000+00:00",
+ "ETag": "\"1bd2383eebbac1f8e7143575ba5b1f4a\"",
+ "Size": 20971520
+ }
+ ],
+ "ChecksumAlgorithm": "",
+ "Initiator": {
+ "ID": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4",
+ "DisplayName": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4"
+ },
+ "Owner": {
+ "DisplayName": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4",
+ "ID": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4"
+ },
+ "StorageClass": "STANDARD"
+ }
+ ======================================== COMPLETE ====
+ {
+ "Location": "http://localhost:9000/test/upload",
+ "Bucket": "test",
+ "Key": "upload",
+ "ETag": "\"8e817c8ccd442f9a79c77b58fe808c43-4\""
+ }
+ ======================================== LIST ====
+
+ An error occurred (NoSuchUpload) when calling the ListParts operation: The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.
+ ======================================== GET PART 4 ====
+ {
+ "AcceptRanges": "bytes",
+ "LastModified": "2023-04-25T10:21:59+00:00",
+ "ContentLength": 20971520,
+ "ETag": "\"8e817c8ccd442f9a79c77b58fe808c43-4\"",
+ "ContentRange": "bytes 62914560-83886079/83886080",
+ "ContentType": "binary/octet-stream",
+ "Metadata": {},
+ "PartsCount": 4
+ }
+ 1bd2383eebbac1f8e7143575ba5b1f4a get-part4
+
+
+ Conclusions:
+
+ - Parts are indeed renumbered with consecutive numbers
+ - ListParts only applies to multipart uploads in progress,
+ it cannot be used once the multipart upload has been completed
+'
+
+export AWS_ACCESS_KEY_ID=1D8Pk2k4oQSoh1BU
+export AWS_SECRET_ACCESS_KEY=4B46SR8U7FUgY0raB8Zuxg1NLyLTvbNV
+
+function aws { command aws --endpoint-url http://localhost:9000 $@ ; }
+
+aws --version
+
+aws s3 mb s3://test
+
+for NUM in 1 4 5 6; do
+ dd if=/dev/urandom of=part$NUM bs=1M count=10
+done
+md5sum part*
+
+UPLOAD=$(aws s3api create-multipart-upload --bucket test --key 'upload' | jq -r ".UploadId")
+echo "Upload ID: $UPLOAD"
+
+PARTS=""
+
+for NUM in 1 4 5 6; do
+ ETAG=$(aws s3api upload-part --bucket test --key 'upload' --part-number $NUM \
+ --body "part$NUM" --upload-id "$UPLOAD" | jq -r ".ETag")
+ echo "PART $NUM ETag: $ETAG"
+ if [ -n "$PARTS" ]; then
+ PARTS="$PARTS,"
+ fi
+ PARTS="$PARTS {\"ETag\":$ETAG,\"PartNumber\":$NUM}"
+done
+
+echo "======================================== LIST ===="
+aws s3api list-parts --bucket test --key upload --upload-id "$UPLOAD" | jq
+
+echo "======================================== COMPLETE ===="
+echo "{\"Parts\":[$PARTS]}" > mpu
+aws s3api complete-multipart-upload --multipart-upload file://mpu \
+ --bucket test --key 'upload' --upload-id "$UPLOAD"
+
+echo "======================================== LIST ===="
+aws s3api list-parts --bucket test --key upload --upload-id "$UPLOAD" | jq
+
+echo "======================================== GET PART 4 ===="
+aws s3api get-object --bucket test --key upload --part-number 4 get-part4
+md5sum get-part4
diff --git a/script/test-skip-part.sh b/script/test-skip-part.sh
new file mode 100644
index 00000000..20ae017d
--- /dev/null
+++ b/script/test-skip-part.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+: '
+ This script tests whether uploaded parts can be skipped in a
+ CompleteMultipartUpoad
+
+ On Minio: yes, parts can be skipped
+
+ On S3: not tested
+
+ Sample output (on Minio):
+
+ f23911bcd1230f5ebe8887cbf5bc396e part1
+ a2657143167eaf647c40473e78a091dc part4
+ 72f72c02c5163bc81024b28ac818c5e0 part5
+ e29cf500d20498218904b8df8806caa2 part6
+ Upload ID: e8fe7b83-9800-46fb-ae90-9d7ccd42fe76
+ PART 1 ETag: "f23911bcd1230f5ebe8887cbf5bc396e"
+ PART 4 ETag: "a2657143167eaf647c40473e78a091dc"
+ PART 5 ETag: "72f72c02c5163bc81024b28ac818c5e0"
+ PART 6 ETag: "e29cf500d20498218904b8df8806caa2"
+ ======================================== COMPLETE ====
+ {
+ "Location": "http://localhost:9000/test/upload",
+ "Bucket": "test",
+ "Key": "upload",
+ "ETag": "\"48246e44d4b38bdc2f3c10ee25b1af17-3\""
+ }
+ ======================================== GET FULL ====
+ {
+ "AcceptRanges": "bytes",
+ "LastModified": "2023-04-25T10:54:35+00:00",
+ "ContentLength": 31457280,
+ "ETag": "\"48246e44d4b38bdc2f3c10ee25b1af17-3\"",
+ "ContentType": "binary/octet-stream",
+ "Metadata": {}
+ }
+ 97fb904da7ad310699a6afab0eb6e061 get-full
+ 97fb904da7ad310699a6afab0eb6e061 -
+ ======================================== GET PART 3 ====
+ {
+ "AcceptRanges": "bytes",
+ "LastModified": "2023-04-25T10:54:35+00:00",
+ "ContentLength": 10485760,
+ "ETag": "\"48246e44d4b38bdc2f3c10ee25b1af17-3\"",
+ "ContentRange": "bytes 20971520-31457279/31457280",
+ "ContentType": "binary/octet-stream",
+ "Metadata": {},
+ "PartsCount": 3
+ }
+ e29cf500d20498218904b8df8806caa2 get-part3
+
+ Conclusions:
+
+ - Skipping a part in a CompleteMultipartUpoad call is OK
+ - The part is simply not included in the stored object
+ - Sequential part renumbering counts only non-skipped parts
+'
+
+export AWS_ACCESS_KEY_ID=1D8Pk2k4oQSoh1BU
+export AWS_SECRET_ACCESS_KEY=4B46SR8U7FUgY0raB8Zuxg1NLyLTvbNV
+
+function aws { command aws --endpoint-url http://localhost:9000 $@ ; }
+
+aws --version
+
+aws s3 mb s3://test
+
+for NUM in 1 4 5 6; do
+ dd if=/dev/urandom of=part$NUM bs=1M count=10
+done
+md5sum part*
+
+UPLOAD=$(aws s3api create-multipart-upload --bucket test --key 'upload' | jq -r ".UploadId")
+echo "Upload ID: $UPLOAD"
+
+PARTS=""
+
+for NUM in 1 4 5 6; do
+ ETAG=$(aws s3api upload-part --bucket test --key 'upload' --part-number $NUM \
+ --body "part$NUM" --upload-id "$UPLOAD" | jq -r ".ETag")
+ echo "PART $NUM ETag: $ETAG"
+ if [ "$NUM" != "5" ]; then
+ if [ -n "$PARTS" ]; then
+ PARTS="$PARTS,"
+ fi
+ PARTS="$PARTS {\"ETag\":$ETAG,\"PartNumber\":$NUM}"
+ fi
+done
+
+echo "======================================== COMPLETE ===="
+echo "{\"Parts\":[$PARTS]}" > mpu
+aws s3api complete-multipart-upload --multipart-upload file://mpu \
+ --bucket test --key 'upload' --upload-id "$UPLOAD"
+
+echo "======================================== GET FULL ===="
+aws s3api get-object --bucket test --key upload get-full
+md5sum get-full
+cat part1 part4 part6 | md5sum
+
+echo "======================================== GET PART 3 ===="
+aws s3api get-object --bucket test --key upload --part-number 3 get-part3
+md5sum get-part3
diff --git a/script/test-smoke.sh b/script/test-smoke.sh
index eababf38..6965c0f3 100755
--- a/script/test-smoke.sh
+++ b/script/test-smoke.sh
@@ -31,6 +31,11 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline sto
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB
+dd if=/dev/urandom of=/tmp/garage.part1.rnd bs=1M count=5
+dd if=/dev/urandom of=/tmp/garage.part2.rnd bs=1M count=5
+dd if=/dev/urandom of=/tmp/garage.part3.rnd bs=1M count=5
+dd if=/dev/urandom of=/tmp/garage.part4.rnd bs=1M count=5
+
# data of lower entropy, to test compression
dd if=/dev/urandom bs=1k count=2 | base64 -w0 > /tmp/garage.1.b64
dd if=/dev/urandom bs=1M count=5 | base64 -w0 > /tmp/garage.2.b64
@@ -40,7 +45,7 @@ echo "🧪 S3 API testing..."
# AWS
if [ -z "$SKIP_AWS" ]; then
- echo "🛠️ Testing with awscli"
+ echo "🛠️ Testing with awscli (aws s3)"
source ${SCRIPT_FOLDER}/dev-env-aws.sh
aws s3 ls
for idx in {1..3}.{rnd,b64}; do
@@ -51,8 +56,36 @@ if [ -z "$SKIP_AWS" ]; then
rm /tmp/garage.$idx.dl
aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
done
+
+ echo "🛠️ Testing multipart uploads with awscli (aws s3api)"
+ UPLOAD=$(aws s3api create-multipart-upload --bucket eprouvette --key 'upload' | jq -r ".UploadId")
+ echo "Upload ID: $UPLOAD"
+ ETAG3=$(aws s3api upload-part --bucket eprouvette --key 'upload' \
+ --part-number 3 --body "/tmp/garage.part1.rnd" --upload-id "$UPLOAD" \
+ | jq -r ".ETag")
+ ETAG2=$(aws s3api upload-part --bucket eprouvette --key 'upload' \
+ --part-number 2 --body "/tmp/garage.part2.rnd" --upload-id "$UPLOAD" \
+ | jq -r ".ETag")
+ ETAG3=$(aws s3api upload-part --bucket eprouvette --key 'upload' \
+ --part-number 3 --body "/tmp/garage.part3.rnd" --upload-id "$UPLOAD" \
+ | jq -r ".ETag")
+ ETAG6=$(aws s3api upload-part --bucket eprouvette --key 'upload' \
+ --part-number 6 --body "/tmp/garage.part4.rnd" --upload-id "$UPLOAD" \
+ | jq -r ".ETag")
+ MPU="{\"Parts\":[{\"PartNumber\":2,\"ETag\":$ETAG2}, {\"PartNumber\":3,\"ETag\":$ETAG3}, {\"PartNumber\":6,\"ETag\":$ETAG6}]}"
+ echo $MPU > /tmp/garage.mpu.json
+ aws s3api complete-multipart-upload --multipart-upload file:///tmp/garage.mpu.json \
+ --bucket eprouvette --key 'upload' --upload-id "$UPLOAD"
+ aws s3api get-object --bucket eprouvette --key upload /tmp/garage.mpu.get
+ if [ "$(md5sum /tmp/garage.mpu.get | cut -d ' ' -f 1)" != "$(cat /tmp/garage.part{2,3,4}.rnd | md5sum | cut -d ' ' -f 1)" ]; then
+ echo "Invalid multipart upload"
+ exit 1
+ fi
fi
+echo "OK!!"
+exit 0
+
# S3CMD
if [ -z "$SKIP_S3CMD" ]; then
echo "🛠️ Testing with s3cmd"
@@ -141,6 +174,7 @@ rm eprouvette/winscp
EOF
fi
+rm /tmp/garage.part{1..4}.rnd
rm /tmp/garage.{1..3}.{rnd,b64}
echo "🏁 Teardown"
diff --git a/script/test-upgrade.sh b/script/test-upgrade.sh
new file mode 100755
index 00000000..dc25e7c6
--- /dev/null
+++ b/script/test-upgrade.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+set -ex
+
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+SCRIPT_FOLDER="`dirname \"$0\"`"
+REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+NIX_RELEASE="${REPO_FOLDER}/result/bin/:${REPO_FOLDER}/result-bin/bin/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
+
+OLD_VERSION="$1"
+ARCH="$2"
+
+
+echo "Downloading old garage binary..."
+curl https://garagehq.deuxfleurs.fr/_releases/$OLD_VERSION/$ARCH/garage > /tmp/old_garage
+chmod +x /tmp/old_garage
+
+echo "============= insert data into old version cluster ================="
+
+export GARAGE_BIN=/tmp/old_garage
+if echo $OLD_VERSION | grep 'v0\.8\.'; then
+ echo "Detected Garage v0.8.x"
+ export GARAGE_08=1
+fi
+
+echo "⏳ Setup cluster using old version"
+$GARAGE_BIN --version
+${SCRIPT_FOLDER}/dev-clean.sh
+${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
+sleep 6
+${SCRIPT_FOLDER}/dev-configure.sh
+${SCRIPT_FOLDER}/dev-bucket.sh
+
+echo "🛠️ Inserting data in old cluster"
+source ${SCRIPT_FOLDER}/dev-env-rclone.sh
+rclone copy "${SCRIPT_FOLDER}/../.git/" garage:eprouvette/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line
+
+echo "🏁 Stopping old cluster"
+killall -INT old_garage
+sleep 2
+killall -9 old_garage || true
+
+echo "🏁 Removing old garage version"
+rm -rv $GARAGE_BIN
+export -n GARAGE_BIN
+export -n GARAGE_08
+
+echo "================ read data from new cluster ==================="
+
+echo "⏳ Setup cluster using new version"
+pwd
+ls
+export GARAGE_BIN=$(which garage)
+$GARAGE_BIN --version
+${SCRIPT_FOLDER}/dev-cluster.sh >> /tmp/garage.log 2>&1 &
+sleep 3
+
+echo "🛠️ Retrieving data from old cluster"
+rclone copy garage:eprouvette/test_dotgit /tmp/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line --fast-list
+
+if ! diff <(find "${SCRIPT_FOLDER}/../.git" -type f | xargs md5sum | cut -d ' ' -f 1 | sort) <(find /tmp/test_dotgit -type f | xargs md5sum | cut -d ' ' -f 1 | sort); then
+ echo "TEST FAILURE: directories are different"
+ exit 1
+fi
+rm -r /tmp/test_dotgit
+
+echo "🏁 Teardown"
+rm -rf /tmp/garage-{data,meta}-*
+rm -rf /tmp/config.*.toml
+
+echo "✅ Success"