diff options
Diffstat (limited to 'cluster/prod/app/backup')
| -rw-r--r-- | cluster/prod/app/backup/build/backup-garage/Dockerfile | 7 | ||||
| -rw-r--r-- | cluster/prod/app/backup/build/backup-garage/do-backup.sh | 84 | ||||
| -rw-r--r-- | cluster/prod/app/backup/deploy/backup-daily.hcl | 51 |
3 files changed, 139 insertions, 3 deletions
diff --git a/cluster/prod/app/backup/build/backup-garage/Dockerfile b/cluster/prod/app/backup/build/backup-garage/Dockerfile new file mode 100644 index 0000000..ea42331 --- /dev/null +++ b/cluster/prod/app/backup/build/backup-garage/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3.17 + +RUN apk add rclone btrfs-progs curl bash jq + +COPY do-backup.sh /do-backup.sh + +CMD bash /do-backup.sh diff --git a/cluster/prod/app/backup/build/backup-garage/do-backup.sh b/cluster/prod/app/backup/build/backup-garage/do-backup.sh new file mode 100644 index 0000000..36ba2f2 --- /dev/null +++ b/cluster/prod/app/backup/build/backup-garage/do-backup.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# DEPENDENCIES: btrfs-progs curl rclone jq + +# PARAMETERS (environmenet variables) +# $BACKUP_BASEDIR => where to store backups and btrfs snapshots +# $GARAGE_ADMIN_TOKEN => Garage administration access token +# $GARAGE_ACCESS_KEY => Garage access key +# $GARAGE_SECRET_KEY => Garage secret key + +if [ -z "$BACKUP_BASEDIR" -o -z "$GARAGE_ACCESS_KEY" -o -z "$GARAGE_ADMIN_TOKEN" ]; then + echo "Missing parameters" +fi + +if [ ! -d "$BACKUP_BASEDIR/buckets" ]; then + btrfs subvolume create "$BACKUP_BASEDIR/buckets" +fi + + +function gcurl { + curl -s -H "Authorization: Bearer $GARAGE_ADMIN_TOKEN" $@ +} + +BUCKETS=$(gcurl "http://localhost:3903/v0/bucket" | jq -r '.[].id') + +for BUCKET in $BUCKETS; do + echo "==== BUCKET $BUCKET ====" + + gcurl "http://localhost:3903/v0/bucket?id=$BUCKET" > "$BACKUP_BASEDIR/buckets/$BUCKET.json" + + ALIASES=$(jq -r '.globalAliases[]' < "$BACKUP_BASEDIR/buckets/$BUCKET.json") + echo "(aka. $ALIASES)" + + case $ALIASES in + *backup*) + echo "Skipping $BUCKET (not doing backup of backup)" + ;; + *cache*) + echo "Skipping $BUCKET (not doing backup of cache)" + ;; + *) + echo "Backing up $BUCKET" + + if [ ! -d "$BACKUP_BASEDIR/buckets/$BUCKET" ]; then + mkdir "$BACKUP_BASEDIR/buckets/$BUCKET" + fi + + gcurl -X POST -H "Content-Type: application/json" --data @- "http://localhost:3903/v0/bucket/allow" >/dev/null <<EOF + { + "bucketId": "$BUCKET", + "accessKeyId": "$GARAGE_ACCESS_KEY", + "permissions": {"read": true} + } +EOF + + rclone sync --s3-endpoint http://localhost:3900 \ + --s3-access-key-id $GARAGE_ACCESS_KEY \ + --s3-secret-access-key $GARAGE_SECRET_KEY \ + --s3-region garage \ + --s3-force-path-style \ + --transfers 32 \ + --fast-list \ + --stats-one-line \ + --stats 10s \ + --stats-log-level NOTICE \ + ":s3:$BUCKET" "$BACKUP_BASEDIR/buckets/$BUCKET" 2>&1 + ;; + esac +done + +echo "========= DONE SYNCHRONIZING ==========" + +if [ ! -d "$BACKUP_BASEDIR/snapshots" ]; then + mkdir "$BACKUP_BASEDIR/snapshots" +fi + +SNAPSHOT="$BACKUP_BASEDIR/snapshots/buckets-$(date +%F)" +if [ ! -e "$SNAPSHOT" ]; then + echo "Making snapshot: $SNAPSHOT" + btrfs subvolume snapshot "$BACKUP_BASEDIR/buckets" "$SNAPSHOT" + btrfs prop set "$SNAPSHOT" ro true +fi + + diff --git a/cluster/prod/app/backup/deploy/backup-daily.hcl b/cluster/prod/app/backup/deploy/backup-daily.hcl index f3da8aa..96e97bc 100644 --- a/cluster/prod/app/backup/deploy/backup-daily.hcl +++ b/cluster/prod/app/backup/deploy/backup-daily.hcl @@ -1,5 +1,5 @@ job "backup_daily" { - datacenters = ["orion", "neptune"] + datacenters = ["orion", "neptune", "scorpio"] type = "batch" priority = "60" @@ -44,7 +44,7 @@ EOH resources { cpu = 500 memory = 100 - memory_max = 300 + memory_max = 1000 } restart { @@ -90,7 +90,7 @@ EOH resources { cpu = 500 memory = 100 - memory_max = 300 + memory_max = 1000 } restart { @@ -227,7 +227,52 @@ EOH resources { cpu = 500 + memory = 100 + memory_max = 1000 + } + + restart { + attempts = 2 + interval = "30m" + delay = "15s" + mode = "fail" + } + } + } + + group "backup-garage" { + constraint { + attribute = "${attr.unique.hostname}" + operator = "=" + value = "abricot" + } + + task "main" { + driver = "docker" + + config { + image = "lxpz/backup_garage:4" + network_mode = "host" + volumes = [ + "/mnt/storage/backup/garage.deuxfleurs.fr:/backup" + ] + } + + template { + data = <<EOH +BACKUP_BASEDIR=/backup +GARAGE_ADMIN_TOKEN={{ key "secrets/garage/admin_token" }} +GARAGE_ACCESS_KEY={{ key "secrets/backup/garage/s3_access_key_id" }} +GARAGE_SECRET_KEY={{ key "secrets/backup/garage/s3_secret_access_key" }} +EOH + destination = "secrets/env_vars" + env = true + } + + resources { + cpu = 500 memory = 200 + memory_max = 4000 } restart { |