Plume backup + WIP consul

13 files changed, 141 insertions, 87 deletions

diff --git a/app/backup/deploy/backup-daily.hcl b/app/backup/deploy/backup-daily.hcl
index a1c2f89..ef9b219 100644
--- a/app/backup/deploy/backup-daily.hcl
+++ b/app/backup/deploy/backup-daily.hcl
@@ -1,52 +1,167 @@
 job "backup_daily" {
   datacenters = ["dc1"]
   type = "batch"
+
+  priority = "60"
+
   periodic {
     cron = "@daily"
     // Do not allow overlapping runs.
     prohibit_overlap = true
   }
 
-  task "backup-dovecot" {
+  group "backup-dovecot" {
     constraint {
       attribute = "${attr.unique.hostname}"
       operator = "="
       value = "digitale"
     }
 
-    driver = "docker"
+    task "main" {
+      driver = "docker"
 
-    config {
-      image = "restic/restic:0.12.1"
-      entrypoint = [ "/bin/sh", "-c" ]
-      args = [ "restic backup /mail && restic forget --keep-within 1m1d --keep-within-weekly 3m --keep-within-monthly 1y ; restic prune --max-unused 50% --max-repack-size 2G ; restic check" ]
-      volumes = [
-        "/mnt/ssd/mail:/mail"
-      ]
-    }
+      config {
+        image = "restic/restic:0.12.1"
+        entrypoint = [ "/bin/sh", "-c" ]
+        args = [ "restic backup /mail && restic forget --keep-within 1m1d --keep-within-weekly 3m --keep-within-monthly 1y ; restic prune --max-unused 50% --max-repack-size 2G ; restic check" ]
+        volumes = [
+          "/mnt/ssd/mail:/mail"
+        ]
+      }
 
-
-    template {
-      data = <<EOH
+      template {
+        data = <<EOH
 AWS_ACCESS_KEY_ID={{ key "secrets/email/dovecot/backup_aws_access_key_id" }}
 AWS_SECRET_ACCESS_KEY={{ key "secrets/email/dovecot/backup_aws_secret_access_key" }}
 RESTIC_REPOSITORY={{ key "secrets/email/dovecot/backup_restic_repository" }}
 RESTIC_PASSWORD={{ key "secrets/email/dovecot/backup_restic_password" }}
 EOH
 
-       destination = "secrets/env_vars"
-       env = true
+         destination = "secrets/env_vars"
+         env = true
+      }
+
+      resources {
+        cpu = 500
+        memory = 200
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
     }
+  }
 
-    resources {
-      memory = 200
+  group "backup-plume" {
+    constraint {
+      attribute = "${attr.unique.hostname}"
+      operator = "="
+      value = "digitale"
     }
 
-    restart {
-      attempts = 2
-      interval = "30m"
-      delay = "15s"
-      mode = "fail"
+    task "main" {
+      driver = "docker"
+
+      config {
+        image = "restic/restic:0.12.1"
+        entrypoint = [ "/bin/sh", "-c" ]
+        args = [ "restic backup /plume && restic forget --keep-within 1m1d --keep-within-weekly 3m --keep-within-monthly 1y ; restic prune --max-unused 50% --max-repack-size 2G ; restic check" ]
+        volumes = [
+          "/mnt/ssd/plume/media:/plume"
+        ]
+      }
+
+      template {
+        data = <<EOH
+AWS_ACCESS_KEY_ID={{ key "secrets/plume/backup_aws_access_key_id" }}
+AWS_SECRET_ACCESS_KEY={{ key "secrets/plume/backup_aws_secret_access_key" }}
+RESTIC_REPOSITORY={{ key "secrets/plume/backup_restic_repository" }}
+RESTIC_PASSWORD={{ key "secrets/plume/backup_restic_password" }}
+EOH
+
+         destination = "secrets/env_vars"
+         env = true
+      }
+
+      resources {
+        cpu = 500
+        memory = 200
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
+    }
+  }
+
+  group "backup-consul" {
+    task "export-kv-store" {
+      driver = "docker"
+
+      lifecycle {      
+        hook = "prestart"      
+        sidecar = false    
+      }
+
+      config {
+        image = "consul:1.11.2"
+        entrypoint = [ "/bin/sh", "-c" ]
+        NETWORK MODE HOST
+        args = [ "/bin/consul kv export > $NOMAD_ALLOC_DIR/consul.json" ]
+      }
+
+      resources {
+        cpu = 200
+        memory = 200
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
+    }
+
+    task "upload-kv-store" {
+      driver = "docker"
+
+      config {
+        image = "restic/restic:0.12.1"
+        entrypoint = [ "/bin/sh", "-c" ]
+        args = [ "restic backup $NOMAD_ALLOC_DIR && restic forget --keep-within 1m1d --keep-within-weekly 3m --keep-within-monthly 1y ; restic prune --max-unused 50% --max-repack-size 2G ; restic check" ]
+      }
+
+
+      template {
+        data = <<EOH
+AWS_ACCESS_KEY_ID={{ key "secrets/plume/backup_aws_access_key_id" }}
+AWS_SECRET_ACCESS_KEY={{ key "secrets/plume/backup_aws_secret_access_key" }}
+RESTIC_REPOSITORY={{ key "secrets/plume/backup_restic_repository" }}
+RESTIC_PASSWORD={{ key "secrets/plume/backup_restic_password" }}
+EOH
+
+         destination = "secrets/env_vars"
+         env = true
+      }
+
+      resources {
+        cpu = 200
+        memory = 200
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
     }
   }
 }
diff --git a/app/backup/deploy/backup-matrix.hcl b/app/backup/deploy/backup-matrix.hcl
deleted file mode 100644
index cb28c90..0000000
--- a/app/backup/deploy/backup-matrix.hcl
+++ /dev/null
@@ -1,62 +0,0 @@
-job "backup_manual_matrix" {
-	datacenters = ["dc1"]
-
-	type = "batch"
-
-	task "backup-matrix" {
-		driver = "docker"
-
-			config {
-				image = "superboum/backup_matrix:4"
-				volumes = [
-					"secrets/id_ed25519:/root/.ssh/id_ed25519",
-					"secrets/id_ed25519.pub:/root/.ssh/id_ed25519.pub",
-					"secrets/known_hosts:/root/.ssh/known_hosts",
-					"/mnt/glusterfs/chat/matrix/synapse/media:/mnt/glusterfs/chat/matrix/synapse/media"
-				]
-				network_mode = "host"
-			}
-
-		env {
-			CONSUL_HTTP_ADDR = "http://consul.service.2.cluster.deuxfleurs.fr:8500"
-		}
-
-		template {
-			data = <<EOH
-TARGET_SSH_USER={{ key "secrets/backup/target_ssh_user" }}
-TARGET_SSH_PORT={{ key "secrets/backup/target_ssh_port" }}
-TARGET_SSH_HOST={{ key "secrets/backup/target_ssh_host" }}
-TARGET_SSH_DIR={{ key "secrets/backup/target_ssh_dir" }}
-REPL_PSQL_USER={{ key "secrets/postgres/keeper/pg_repl_username" }}
-REPL_PSQL_PWD={{ key "secrets/postgres/keeper/pg_repl_pwd" }}
-EOH
-
-			destination = "secrets/env_vars"
-			env = true
-		}
-
-		template {
-			data = "{{ key \"secrets/backup/id_ed25519\" }}"
-			destination = "secrets/id_ed25519"
-		}
-		template {
-			data = "{{ key \"secrets/backup/id_ed25519.pub\" }}"
-			destination = "secrets/id_ed25519.pub"
-		}
-		template {
-			data = "{{ key \"secrets/backup/target_ssh_fingerprint\" }}"
-			destination = "secrets/known_hosts"
-		}
-
-		resources {
-			memory = 200
-		}
-
-		restart {
-			attempts = 2
-			interval = "30m"
-			delay = "15s"
-			mode = "fail"
-		}
-	}
-}
diff --git a/app/backup/secrets/email/dovecot/backup_aws_access_key_id b/app/email/secrets/email/dovecot/backup_aws_access_key_id
index 9ae6adf..9ae6adf 100644
--- a/app/backup/secrets/email/dovecot/backup_aws_access_key_id
+++ b/app/email/secrets/email/dovecot/backup_aws_access_key_id
diff --git a/app/backup/secrets/email/dovecot/backup_aws_secret_access_key b/app/email/secrets/email/dovecot/backup_aws_secret_access_key
index ac95906..ac95906 100644
--- a/app/backup/secrets/email/dovecot/backup_aws_secret_access_key
+++ b/app/email/secrets/email/dovecot/backup_aws_secret_access_key
diff --git a/app/backup/secrets/email/dovecot/backup_restic_password b/app/email/secrets/email/dovecot/backup_restic_password
index c19a4a3..c19a4a3 100644
--- a/app/backup/secrets/email/dovecot/backup_restic_password
+++ b/app/email/secrets/email/dovecot/backup_restic_password
diff --git a/app/backup/secrets/email/dovecot/backup_restic_repository b/app/email/secrets/email/dovecot/backup_restic_repository
index 0434a15..0434a15 100644
--- a/app/backup/secrets/email/dovecot/backup_restic_repository
+++ b/app/email/secrets/email/dovecot/backup_restic_repository
diff --git a/app/plume/secrets/plume/aws_access_key_id b/app/plume/secrets/plume/aws_access_key_id
deleted file mode 100644
index 8129980..0000000
--- a/app/plume/secrets/plume/aws_access_key_id
+++ /dev/null
@@ -1 +0,0 @@
-USER AWS access key ID, eg. GKxxxx
diff --git a/app/plume/secrets/plume/aws_endpoint b/app/plume/secrets/plume/aws_endpoint
deleted file mode 100644
index 9868a5a..0000000
--- a/app/plume/secrets/plume/aws_endpoint
+++ /dev/null
@@ -1 +0,0 @@
-USER AWS Endpoint, eg. s3.garage.tld
diff --git a/app/plume/secrets/plume/aws_secret_access_key b/app/plume/secrets/plume/aws_secret_access_key
deleted file mode 100644
index 03a5225..0000000
--- a/app/plume/secrets/plume/aws_secret_access_key
+++ /dev/null
@@ -1 +0,0 @@
-USER AWS Secret Access Key
diff --git a/app/plume/secrets/plume/backup_aws_access_key_id b/app/plume/secrets/plume/backup_aws_access_key_id
new file mode 100644
index 0000000..9235e53
--- /dev/null
+++ b/app/plume/secrets/plume/backup_aws_access_key_id
@@ -0,0 +1 @@
+USER Backup AWS access key ID
diff --git a/app/plume/secrets/plume/backup_aws_secret_access_key b/app/plume/secrets/plume/backup_aws_secret_access_key
new file mode 100644
index 0000000..f34677e
--- /dev/null
+++ b/app/plume/secrets/plume/backup_aws_secret_access_key
@@ -0,0 +1 @@
+USER Backup AWS secret access key
diff --git a/app/plume/secrets/plume/backup_restic_password b/app/plume/secrets/plume/backup_restic_password
new file mode 100644
index 0000000..fbaa5fa
--- /dev/null
+++ b/app/plume/secrets/plume/backup_restic_password
@@ -0,0 +1 @@
+USER Restic password to encrypt backups
diff --git a/app/plume/secrets/plume/backup_restic_repository b/app/plume/secrets/plume/backup_restic_repository
new file mode 100644
index 0000000..3f6cb93
--- /dev/null
+++ b/app/plume/secrets/plume/backup_restic_repository
@@ -0,0 +1 @@
+USER Restic repository, eg. s3:https://s3.garage.tld