From b7c4f94ebd9afdf06338e1410f93c7c5ee99f561 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Mon, 20 Mar 2023 16:47:22 +0100
Subject: Add Garage backup script running on Abricot

---
 .../prod/app/backup/build/backup-garage/Dockerfile |  7 ++
 .../app/backup/build/backup-garage/do-backup.sh    | 84 ++++++++++++++++++++++
 cluster/prod/app/backup/deploy/backup-daily.hcl    | 53 ++++++++++++--
 3 files changed, 140 insertions(+), 4 deletions(-)
 create mode 100644 cluster/prod/app/backup/build/backup-garage/Dockerfile
 create mode 100644 cluster/prod/app/backup/build/backup-garage/do-backup.sh

(limited to 'cluster/prod/app/backup')

diff --git a/cluster/prod/app/backup/build/backup-garage/Dockerfile b/cluster/prod/app/backup/build/backup-garage/Dockerfile
new file mode 100644
index 0000000..ea42331
--- /dev/null
+++ b/cluster/prod/app/backup/build/backup-garage/Dockerfile
@@ -0,0 +1,7 @@
+FROM alpine:3.17
+
+RUN apk add rclone btrfs-progs curl bash jq
+
+COPY do-backup.sh /do-backup.sh
+
+CMD bash /do-backup.sh
diff --git a/cluster/prod/app/backup/build/backup-garage/do-backup.sh b/cluster/prod/app/backup/build/backup-garage/do-backup.sh
new file mode 100644
index 0000000..36ba2f2
--- /dev/null
+++ b/cluster/prod/app/backup/build/backup-garage/do-backup.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# DEPENDENCIES: btrfs-progs curl rclone jq
+
+# PARAMETERS (environmenet variables)
+#       $BACKUP_BASEDIR         => where to store backups and btrfs snapshots
+#       $GARAGE_ADMIN_TOKEN     => Garage administration access token
+#       $GARAGE_ACCESS_KEY      => Garage access key
+#       $GARAGE_SECRET_KEY      => Garage secret key
+
+if [ -z "$BACKUP_BASEDIR" -o -z "$GARAGE_ACCESS_KEY" -o -z "$GARAGE_ADMIN_TOKEN" ]; then
+        echo "Missing parameters"
+fi
+
+if [ ! -d "$BACKUP_BASEDIR/buckets" ]; then
+        btrfs subvolume create "$BACKUP_BASEDIR/buckets"
+fi
+
+
+function gcurl {
+        curl -s -H "Authorization: Bearer $GARAGE_ADMIN_TOKEN" $@
+}
+
+BUCKETS=$(gcurl "http://localhost:3903/v0/bucket" | jq -r '.[].id')
+
+for BUCKET in $BUCKETS; do
+        echo "==== BUCKET $BUCKET ===="
+
+        gcurl "http://localhost:3903/v0/bucket?id=$BUCKET" > "$BACKUP_BASEDIR/buckets/$BUCKET.json"
+
+        ALIASES=$(jq -r '.globalAliases[]' < "$BACKUP_BASEDIR/buckets/$BUCKET.json")
+        echo "(aka. $ALIASES)"
+
+        case $ALIASES in
+                *backup*)
+                        echo "Skipping $BUCKET (not doing backup of backup)"
+                        ;;
+                *cache*)
+                        echo "Skipping $BUCKET (not doing backup of cache)"
+                        ;;
+                *)
+                        echo "Backing up $BUCKET"
+
+                        if [ ! -d "$BACKUP_BASEDIR/buckets/$BUCKET" ]; then
+                                mkdir "$BACKUP_BASEDIR/buckets/$BUCKET"
+                        fi
+
+                        gcurl -X POST -H "Content-Type: application/json"  --data @- "http://localhost:3903/v0/bucket/allow" >/dev/null <<EOF
+                                {
+                                        "bucketId": "$BUCKET",
+                                        "accessKeyId": "$GARAGE_ACCESS_KEY",
+                                        "permissions": {"read": true}
+                                }
+EOF
+
+                        rclone sync --s3-endpoint http://localhost:3900 \
+                                --s3-access-key-id $GARAGE_ACCESS_KEY \
+                                --s3-secret-access-key $GARAGE_SECRET_KEY \
+                                --s3-region garage \
+                                --s3-force-path-style \
+                                --transfers 32 \
+                                --fast-list \
+                                --stats-one-line \
+                                --stats 10s \
+                                --stats-log-level NOTICE \
+                                ":s3:$BUCKET" "$BACKUP_BASEDIR/buckets/$BUCKET" 2>&1
+                        ;;
+        esac
+done
+
+echo "========= DONE SYNCHRONIZING =========="
+
+if [ ! -d "$BACKUP_BASEDIR/snapshots" ]; then
+        mkdir "$BACKUP_BASEDIR/snapshots"
+fi
+
+SNAPSHOT="$BACKUP_BASEDIR/snapshots/buckets-$(date +%F)"
+if [ ! -e "$SNAPSHOT" ]; then
+        echo "Making snapshot: $SNAPSHOT"
+        btrfs subvolume snapshot "$BACKUP_BASEDIR/buckets" "$SNAPSHOT"
+        btrfs prop set "$SNAPSHOT" ro true
+fi
+
+
diff --git a/cluster/prod/app/backup/deploy/backup-daily.hcl b/cluster/prod/app/backup/deploy/backup-daily.hcl
index f3da8aa..fb301e1 100644
--- a/cluster/prod/app/backup/deploy/backup-daily.hcl
+++ b/cluster/prod/app/backup/deploy/backup-daily.hcl
@@ -1,5 +1,5 @@
 job "backup_daily" {
-  datacenters = ["orion", "neptune"]
+  datacenters = ["orion", "neptune", "scorpio"]
   type = "batch"
 
   priority = "60"
@@ -44,7 +44,7 @@ EOH
       resources {
         cpu = 500
         memory = 100
-	memory_max = 300
+		memory_max = 1000
       }
 
       restart {
@@ -90,7 +90,7 @@ EOH
       resources {
         cpu = 500
         memory = 100
-	memory_max = 300
+		memory_max = 1000
       }
 
       restart {
@@ -227,7 +227,52 @@ EOH
 
       resources {
         cpu = 500
-        memory = 200
+        memory = 100
+		memory_max = 1000
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
+    }
+  }
+
+  group "backup-garage" {
+    constraint {
+      attribute = "${attr.unique.hostname}"
+      operator = "="
+      value = "abricot"
+    }
+
+    task "main" {
+      driver = "docker"
+
+      config {
+        image = "lxpz/backup_garage:4"
+		network_mode = "host"
+        volumes = [
+          "/mnt/storage/backup/garage.deuxfleurs.fr:/backup"
+        ]
+      }
+
+      template {
+        data = <<EOH
+BACKUP_BASEDIR=/backup
+GARAGE_ADMIN_TOKEN={{ key "secrets/garage/admin_token" }}
+GARAGE_ACCESS_KEY={{ key "secrets/backup/garage/s3_access_key_id" }}
+GARAGE_SECRET_KEY={{ key "secrets/backup/garage/s3_secret_access_key" }}
+EOH
+         destination = "secrets/env_vars"
+         env = true
+      }
+
+      resources {
+        cpu = 500
+        memory = 100
+		memory_max = 1000
       }
 
       restart {
-- 
cgit v1.2.3


From 8e29ee3b0b2f1855a26bc45e5dc502f8aad8e266 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Fri, 24 Mar 2023 11:28:59 +0100
Subject: backup memory

---
 cluster/prod/app/backup/deploy/backup-daily.hcl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cluster/prod/app/backup')

diff --git a/cluster/prod/app/backup/deploy/backup-daily.hcl b/cluster/prod/app/backup/deploy/backup-daily.hcl
index fb301e1..96e97bc 100644
--- a/cluster/prod/app/backup/deploy/backup-daily.hcl
+++ b/cluster/prod/app/backup/deploy/backup-daily.hcl
@@ -271,8 +271,8 @@ EOH
 
       resources {
         cpu = 500
-        memory = 100
-		memory_max = 1000
+        memory = 200
+		memory_max = 4000
       }
 
       restart {
-- 
cgit v1.2.3