aboutsummaryrefslogtreecommitdiff
path: root/cluster/prod/app
diff options
context:
space:
mode:
authorAlex Auvolat <alex@adnab.me>2023-04-21 10:31:05 +0200
committerAlex Auvolat <alex@adnab.me>2023-04-21 10:31:05 +0200
commit529480b133c5ee9197c6ca3f400a22d4579dfd17 (patch)
tree6d1135933de74d2e06edd29ffc7654b87efc04e1 /cluster/prod/app
parentb4e82e37e4e1718dfffa70cd0c6222c1b34fc997 (diff)
parentaf82308e8463e34c8f75e469f4d1a5df18f50710 (diff)
downloadnixcfg-529480b133c5ee9197c6ca3f400a22d4579dfd17.tar.gz
nixcfg-529480b133c5ee9197c6ca3f400a22d4579dfd17.zip
Merge branch 'main' into simplify-network-config
Diffstat (limited to 'cluster/prod/app')
-rw-r--r--cluster/prod/app/backup/build/backup-garage/Dockerfile2
-rw-r--r--cluster/prod/app/backup/build/backup-garage/do-backup.sh71
-rw-r--r--cluster/prod/app/backup/deploy/backup-daily.hcl44
-rw-r--r--cluster/prod/app/backup/deploy/backup-garage.hcl72
4 files changed, 108 insertions, 81 deletions
diff --git a/cluster/prod/app/backup/build/backup-garage/Dockerfile b/cluster/prod/app/backup/build/backup-garage/Dockerfile
index ea42331..ffb558c 100644
--- a/cluster/prod/app/backup/build/backup-garage/Dockerfile
+++ b/cluster/prod/app/backup/build/backup-garage/Dockerfile
@@ -1,6 +1,6 @@
FROM alpine:3.17
-RUN apk add rclone btrfs-progs curl bash jq
+RUN apk add rclone curl bash jq
COPY do-backup.sh /do-backup.sh
diff --git a/cluster/prod/app/backup/build/backup-garage/do-backup.sh b/cluster/prod/app/backup/build/backup-garage/do-backup.sh
index 36ba2f2..c8e01aa 100644
--- a/cluster/prod/app/backup/build/backup-garage/do-backup.sh
+++ b/cluster/prod/app/backup/build/backup-garage/do-backup.sh
@@ -1,34 +1,53 @@
#!/usr/bin/env bash
-# DEPENDENCIES: btrfs-progs curl rclone jq
-
+# DESCRIPTION:
+# Script to backup all buckets on a Garage cluster using rclone.
+#
+# REQUIREMENTS:
+# An access key for the backup script must be created in Garage beforehand.
+# This script will use the Garage administration API to grant read access
+# to this key on all buckets.
+#
+# A rclone configuration file is expected to be located at `/etc/secrets/rclone.conf`,
+# which contains credentials to the following two remotes:
+# garage: the Garage server, for read access (using the backup access key)
+# backup: the backup location
+#
+# DEPENDENCIES: (see Dockerfile)
+# curl
+# jq
+# rclone
+#
# PARAMETERS (environmenet variables)
-# $BACKUP_BASEDIR => where to store backups and btrfs snapshots
+# $GARAGE_ADMIN_API_URL => Garage administration API URL (e.g. http://localhost:3903)
# $GARAGE_ADMIN_TOKEN => Garage administration access token
-# $GARAGE_ACCESS_KEY => Garage access key
-# $GARAGE_SECRET_KEY => Garage secret key
+# $GARAGE_ACCESS_KEY => Garage access key ID
+# $TARGET_BACKUP_DIR => Folder on the backup remote where to store buckets
-if [ -z "$BACKUP_BASEDIR" -o -z "$GARAGE_ACCESS_KEY" -o -z "$GARAGE_ADMIN_TOKEN" ]; then
+if [ -z "$GARAGE_ACCESS_KEY" -o -z "$GARAGE_ADMIN_TOKEN" -o -z "$GARAGE_ADMIN_API_URL" ]; then
echo "Missing parameters"
fi
-if [ ! -d "$BACKUP_BASEDIR/buckets" ]; then
- btrfs subvolume create "$BACKUP_BASEDIR/buckets"
-fi
-
+# copy potentially immutable file to a mutable location,
+# otherwise rclone complains
+mkdir -p /root/.config/rclone
+cp /etc/secrets/rclone.conf /root/.config/rclone/rclone.conf
function gcurl {
curl -s -H "Authorization: Bearer $GARAGE_ADMIN_TOKEN" $@
}
-BUCKETS=$(gcurl "http://localhost:3903/v0/bucket" | jq -r '.[].id')
+BUCKETS=$(gcurl "$GARAGE_ADMIN_API_URL/v0/bucket" | jq -r '.[].id')
+
+mkdir -p /tmp/buckets-info
for BUCKET in $BUCKETS; do
echo "==== BUCKET $BUCKET ===="
- gcurl "http://localhost:3903/v0/bucket?id=$BUCKET" > "$BACKUP_BASEDIR/buckets/$BUCKET.json"
+ gcurl "http://localhost:3903/v0/bucket?id=$BUCKET" > "/tmp/buckets-info/$BUCKET.json"
+ rclone copy "/tmp/buckets-info/$BUCKET.json" "backup:$TARGET_BACKUP_DIR/" 2>&1
- ALIASES=$(jq -r '.globalAliases[]' < "$BACKUP_BASEDIR/buckets/$BUCKET.json")
+ ALIASES=$(jq -r '.globalAliases[]' < "/tmp/buckets-info/$BUCKET.json")
echo "(aka. $ALIASES)"
case $ALIASES in
@@ -41,10 +60,6 @@ for BUCKET in $BUCKETS; do
*)
echo "Backing up $BUCKET"
- if [ ! -d "$BACKUP_BASEDIR/buckets/$BUCKET" ]; then
- mkdir "$BACKUP_BASEDIR/buckets/$BUCKET"
- fi
-
gcurl -X POST -H "Content-Type: application/json" --data @- "http://localhost:3903/v0/bucket/allow" >/dev/null <<EOF
{
"bucketId": "$BUCKET",
@@ -53,32 +68,16 @@ for BUCKET in $BUCKETS; do
}
EOF
- rclone sync --s3-endpoint http://localhost:3900 \
- --s3-access-key-id $GARAGE_ACCESS_KEY \
- --s3-secret-access-key $GARAGE_SECRET_KEY \
- --s3-region garage \
- --s3-force-path-style \
- --transfers 32 \
+ rclone sync \
+ --transfers 32 \
--fast-list \
--stats-one-line \
--stats 10s \
--stats-log-level NOTICE \
- ":s3:$BUCKET" "$BACKUP_BASEDIR/buckets/$BUCKET" 2>&1
+ "garage:$BUCKET" "backup:$TARGET_BACKUP_DIR/$BUCKET" 2>&1
;;
esac
done
echo "========= DONE SYNCHRONIZING =========="
-if [ ! -d "$BACKUP_BASEDIR/snapshots" ]; then
- mkdir "$BACKUP_BASEDIR/snapshots"
-fi
-
-SNAPSHOT="$BACKUP_BASEDIR/snapshots/buckets-$(date +%F)"
-if [ ! -e "$SNAPSHOT" ]; then
- echo "Making snapshot: $SNAPSHOT"
- btrfs subvolume snapshot "$BACKUP_BASEDIR/buckets" "$SNAPSHOT"
- btrfs prop set "$SNAPSHOT" ro true
-fi
-
-
diff --git a/cluster/prod/app/backup/deploy/backup-daily.hcl b/cluster/prod/app/backup/deploy/backup-daily.hcl
index 96e97bc..a7a3dcf 100644
--- a/cluster/prod/app/backup/deploy/backup-daily.hcl
+++ b/cluster/prod/app/backup/deploy/backup-daily.hcl
@@ -239,48 +239,4 @@ EOH
}
}
}
-
- group "backup-garage" {
- constraint {
- attribute = "${attr.unique.hostname}"
- operator = "="
- value = "abricot"
- }
-
- task "main" {
- driver = "docker"
-
- config {
- image = "lxpz/backup_garage:4"
- network_mode = "host"
- volumes = [
- "/mnt/storage/backup/garage.deuxfleurs.fr:/backup"
- ]
- }
-
- template {
- data = <<EOH
-BACKUP_BASEDIR=/backup
-GARAGE_ADMIN_TOKEN={{ key "secrets/garage/admin_token" }}
-GARAGE_ACCESS_KEY={{ key "secrets/backup/garage/s3_access_key_id" }}
-GARAGE_SECRET_KEY={{ key "secrets/backup/garage/s3_secret_access_key" }}
-EOH
- destination = "secrets/env_vars"
- env = true
- }
-
- resources {
- cpu = 500
- memory = 200
- memory_max = 4000
- }
-
- restart {
- attempts = 2
- interval = "30m"
- delay = "15s"
- mode = "fail"
- }
- }
- }
}
diff --git a/cluster/prod/app/backup/deploy/backup-garage.hcl b/cluster/prod/app/backup/deploy/backup-garage.hcl
new file mode 100644
index 0000000..f9d64d1
--- /dev/null
+++ b/cluster/prod/app/backup/deploy/backup-garage.hcl
@@ -0,0 +1,72 @@
+job "backup-garage" {
+ datacenters = ["neptune", "bespin"]
+ type = "batch"
+
+ priority = "60"
+
+ periodic {
+ cron = "@daily"
+ // Do not allow overlapping runs.
+ prohibit_overlap = true
+ }
+
+ group "backup-garage" {
+ task "main" {
+ driver = "docker"
+
+ config {
+ image = "lxpz/backup_garage:9"
+ network_mode = "host"
+ volumes = [
+ "secrets/rclone.conf:/etc/secrets/rclone.conf"
+ ]
+ }
+
+ template {
+ data = <<EOH
+GARAGE_ADMIN_TOKEN={{ key "secrets/garage/admin_token" }}
+GARAGE_ADMIN_API_URL=http://localhost:3903
+GARAGE_ACCESS_KEY={{ key "secrets/backup/garage/s3_access_key_id" }}
+TARGET_BACKUP_DIR={{ key "secrets/backup/garage/target_sftp_directory" }}
+EOH
+ destination = "secrets/env_vars"
+ env = true
+ }
+
+ template {
+ data = <<EOH
+[garage]
+type = s3
+provider = Other
+env_auth = false
+access_key_id = {{ key "secrets/backup/garage/s3_access_key_id" }}
+secret_access_key = {{ key "secrets/backup/garage/s3_secret_access_key" }}
+endpoint = http://localhost:3900
+region = garage
+
+[backup]
+type = sftp
+host = {{ key "secrets/backup/garage/target_sftp_host" }}
+user = {{ key "secrets/backup/garage/target_sftp_user" }}
+port = {{ key "secrets/backup/garage/target_sftp_port" }}
+key_pem = {{ key "secrets/backup/garage/target_sftp_key_pem" | replaceAll "\n" "\\n" }}
+shell_type = unix
+EOH
+ destination = "secrets/rclone.conf"
+ }
+
+ resources {
+ cpu = 500
+ memory = 200
+ memory_max = 4000
+ }
+
+ restart {
+ attempts = 2
+ interval = "30m"
+ delay = "15s"
+ mode = "fail"
+ }
+ }
+ }
+}