From af82308e8463e34c8f75e469f4d1a5df18f50710 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Thu, 20 Apr 2023 12:10:07 +0200
Subject: Garage backup to SFTP target hosted by Max

---
 .../prod/app/backup/build/backup-garage/Dockerfile |  2 +-
 .../app/backup/build/backup-garage/do-backup.sh    | 71 +++++++++++----------
 cluster/prod/app/backup/deploy/backup-daily.hcl    | 44 -------------
 cluster/prod/app/backup/deploy/backup-garage.hcl   | 72 ++++++++++++++++++++++
 4 files changed, 108 insertions(+), 81 deletions(-)
 create mode 100644 cluster/prod/app/backup/deploy/backup-garage.hcl

(limited to 'cluster')

diff --git a/cluster/prod/app/backup/build/backup-garage/Dockerfile b/cluster/prod/app/backup/build/backup-garage/Dockerfile
index ea42331..ffb558c 100644
--- a/cluster/prod/app/backup/build/backup-garage/Dockerfile
+++ b/cluster/prod/app/backup/build/backup-garage/Dockerfile
@@ -1,6 +1,6 @@
 FROM alpine:3.17
 
-RUN apk add rclone btrfs-progs curl bash jq
+RUN apk add rclone curl bash jq
 
 COPY do-backup.sh /do-backup.sh
 
diff --git a/cluster/prod/app/backup/build/backup-garage/do-backup.sh b/cluster/prod/app/backup/build/backup-garage/do-backup.sh
index 36ba2f2..c8e01aa 100644
--- a/cluster/prod/app/backup/build/backup-garage/do-backup.sh
+++ b/cluster/prod/app/backup/build/backup-garage/do-backup.sh
@@ -1,34 +1,53 @@
 #!/usr/bin/env bash
 
-# DEPENDENCIES: btrfs-progs curl rclone jq
-
+# DESCRIPTION:
+# 		Script to backup all buckets on a Garage cluster using rclone.
+#
+# REQUIREMENTS:
+# 		An access key for the backup script must be created in Garage beforehand.
+# 		This script will use the Garage administration API to grant read access
+# 		to this key on all buckets.
+#
+# 		A rclone configuration file is expected to be located at `/etc/secrets/rclone.conf`,
+# 		which contains credentials to the following two remotes:
+# 			garage:		the Garage server, for read access (using the backup access key)
+# 			backup:		the backup location
+#
+# DEPENDENCIES: (see Dockerfile)
+# 		curl
+# 		jq
+# 		rclone
+#
 # PARAMETERS (environmenet variables)
-#       $BACKUP_BASEDIR         => where to store backups and btrfs snapshots
+#       $GARAGE_ADMIN_API_URL   => Garage administration API URL (e.g. http://localhost:3903)
 #       $GARAGE_ADMIN_TOKEN     => Garage administration access token
-#       $GARAGE_ACCESS_KEY      => Garage access key
-#       $GARAGE_SECRET_KEY      => Garage secret key
+#       $GARAGE_ACCESS_KEY      => Garage access key ID
+#       $TARGET_BACKUP_DIR      => Folder on the backup remote where to store buckets
 
-if [ -z "$BACKUP_BASEDIR" -o -z "$GARAGE_ACCESS_KEY" -o -z "$GARAGE_ADMIN_TOKEN" ]; then
+if [ -z "$GARAGE_ACCESS_KEY" -o -z "$GARAGE_ADMIN_TOKEN" -o -z "$GARAGE_ADMIN_API_URL" ]; then
         echo "Missing parameters"
 fi
 
-if [ ! -d "$BACKUP_BASEDIR/buckets" ]; then
-        btrfs subvolume create "$BACKUP_BASEDIR/buckets"
-fi
-
+# copy potentially immutable file to a mutable location,
+# otherwise rclone complains
+mkdir -p /root/.config/rclone
+cp /etc/secrets/rclone.conf /root/.config/rclone/rclone.conf
 
 function gcurl {
         curl -s -H "Authorization: Bearer $GARAGE_ADMIN_TOKEN" $@
 }
 
-BUCKETS=$(gcurl "http://localhost:3903/v0/bucket" | jq -r '.[].id')
+BUCKETS=$(gcurl "$GARAGE_ADMIN_API_URL/v0/bucket" | jq -r '.[].id')
+
+mkdir -p /tmp/buckets-info
 
 for BUCKET in $BUCKETS; do
         echo "==== BUCKET $BUCKET ===="
 
-        gcurl "http://localhost:3903/v0/bucket?id=$BUCKET" > "$BACKUP_BASEDIR/buckets/$BUCKET.json"
+        gcurl "http://localhost:3903/v0/bucket?id=$BUCKET" > "/tmp/buckets-info/$BUCKET.json"
+		rclone copy "/tmp/buckets-info/$BUCKET.json" "backup:$TARGET_BACKUP_DIR/" 2>&1
 
-        ALIASES=$(jq -r '.globalAliases[]' < "$BACKUP_BASEDIR/buckets/$BUCKET.json")
+        ALIASES=$(jq -r '.globalAliases[]' < "/tmp/buckets-info/$BUCKET.json")
         echo "(aka. $ALIASES)"
 
         case $ALIASES in
@@ -41,10 +60,6 @@ for BUCKET in $BUCKETS; do
                 *)
                         echo "Backing up $BUCKET"
 
-                        if [ ! -d "$BACKUP_BASEDIR/buckets/$BUCKET" ]; then
-                                mkdir "$BACKUP_BASEDIR/buckets/$BUCKET"
-                        fi
-
                         gcurl -X POST -H "Content-Type: application/json"  --data @- "http://localhost:3903/v0/bucket/allow" >/dev/null <<EOF
                                 {
                                         "bucketId": "$BUCKET",
@@ -53,32 +68,16 @@ for BUCKET in $BUCKETS; do
                                 }
 EOF
 
-                        rclone sync --s3-endpoint http://localhost:3900 \
-                                --s3-access-key-id $GARAGE_ACCESS_KEY \
-                                --s3-secret-access-key $GARAGE_SECRET_KEY \
-                                --s3-region garage \
-                                --s3-force-path-style \
-                                --transfers 32 \
+                        rclone sync \
+                        		--transfers 32 \
                                 --fast-list \
                                 --stats-one-line \
                                 --stats 10s \
                                 --stats-log-level NOTICE \
-                                ":s3:$BUCKET" "$BACKUP_BASEDIR/buckets/$BUCKET" 2>&1
+								"garage:$BUCKET" "backup:$TARGET_BACKUP_DIR/$BUCKET" 2>&1
                         ;;
         esac
 done
 
 echo "========= DONE SYNCHRONIZING =========="
 
-if [ ! -d "$BACKUP_BASEDIR/snapshots" ]; then
-        mkdir "$BACKUP_BASEDIR/snapshots"
-fi
-
-SNAPSHOT="$BACKUP_BASEDIR/snapshots/buckets-$(date +%F)"
-if [ ! -e "$SNAPSHOT" ]; then
-        echo "Making snapshot: $SNAPSHOT"
-        btrfs subvolume snapshot "$BACKUP_BASEDIR/buckets" "$SNAPSHOT"
-        btrfs prop set "$SNAPSHOT" ro true
-fi
-
-
diff --git a/cluster/prod/app/backup/deploy/backup-daily.hcl b/cluster/prod/app/backup/deploy/backup-daily.hcl
index 96e97bc..a7a3dcf 100644
--- a/cluster/prod/app/backup/deploy/backup-daily.hcl
+++ b/cluster/prod/app/backup/deploy/backup-daily.hcl
@@ -239,48 +239,4 @@ EOH
       }
     }
   }
-
-  group "backup-garage" {
-    constraint {
-      attribute = "${attr.unique.hostname}"
-      operator = "="
-      value = "abricot"
-    }
-
-    task "main" {
-      driver = "docker"
-
-      config {
-        image = "lxpz/backup_garage:4"
-		network_mode = "host"
-        volumes = [
-          "/mnt/storage/backup/garage.deuxfleurs.fr:/backup"
-        ]
-      }
-
-      template {
-        data = <<EOH
-BACKUP_BASEDIR=/backup
-GARAGE_ADMIN_TOKEN={{ key "secrets/garage/admin_token" }}
-GARAGE_ACCESS_KEY={{ key "secrets/backup/garage/s3_access_key_id" }}
-GARAGE_SECRET_KEY={{ key "secrets/backup/garage/s3_secret_access_key" }}
-EOH
-         destination = "secrets/env_vars"
-         env = true
-      }
-
-      resources {
-        cpu = 500
-        memory = 200
-		memory_max = 4000
-      }
-
-      restart {
-        attempts = 2
-        interval = "30m"
-        delay = "15s"
-        mode = "fail"
-      }
-    }
-  }
 }
diff --git a/cluster/prod/app/backup/deploy/backup-garage.hcl b/cluster/prod/app/backup/deploy/backup-garage.hcl
new file mode 100644
index 0000000..f9d64d1
--- /dev/null
+++ b/cluster/prod/app/backup/deploy/backup-garage.hcl
@@ -0,0 +1,72 @@
+job "backup-garage" {
+  datacenters = ["neptune", "bespin"]
+  type = "batch"
+
+  priority = "60"
+
+  periodic {
+    cron = "@daily"
+    // Do not allow overlapping runs.
+    prohibit_overlap = true
+  }
+
+  group "backup-garage" {
+    task "main" {
+      driver = "docker"
+
+      config {
+        image = "lxpz/backup_garage:9"
+		network_mode = "host"
+        volumes = [
+		  "secrets/rclone.conf:/etc/secrets/rclone.conf"
+        ]
+      }
+
+      template {
+        data = <<EOH
+GARAGE_ADMIN_TOKEN={{ key "secrets/garage/admin_token" }}
+GARAGE_ADMIN_API_URL=http://localhost:3903
+GARAGE_ACCESS_KEY={{ key "secrets/backup/garage/s3_access_key_id" }}
+TARGET_BACKUP_DIR={{ key "secrets/backup/garage/target_sftp_directory" }}
+EOH
+         destination = "secrets/env_vars"
+         env = true
+      }
+
+      template {
+        data = <<EOH
+[garage]
+type = s3
+provider = Other
+env_auth = false
+access_key_id = {{ key "secrets/backup/garage/s3_access_key_id" }}
+secret_access_key = {{ key "secrets/backup/garage/s3_secret_access_key" }}
+endpoint = http://localhost:3900
+region = garage
+
+[backup]
+type = sftp
+host = {{ key "secrets/backup/garage/target_sftp_host" }}
+user = {{ key "secrets/backup/garage/target_sftp_user" }}
+port = {{ key "secrets/backup/garage/target_sftp_port" }}
+key_pem = {{ key "secrets/backup/garage/target_sftp_key_pem" | replaceAll "\n" "\\n" }}
+shell_type = unix
+EOH
+		destination = "secrets/rclone.conf"
+      }
+
+      resources {
+        cpu = 500
+        memory = 200
+		memory_max = 4000
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
+    }
+  }
+}
-- 
cgit v1.2.3