diff options
| -rw-r--r-- | app/backup/build/backup-matrix/Dockerfile | 22 | ||||
| -rwxr-xr-x | app/backup/build/backup-matrix/do_backup.sh | 40 | ||||
| -rw-r--r-- | app/backup/build/backup-psql/default.nix | 16 | ||||
| -rwxr-xr-x | app/backup/build/backup-psql/do_backup.py | 101 | ||||
| -rw-r--r-- | op_guide/stolon/nomad_full_backup.md | 26 |
5 files changed, 143 insertions, 62 deletions
diff --git a/app/backup/build/backup-matrix/Dockerfile b/app/backup/build/backup-matrix/Dockerfile deleted file mode 100644 index 34b6040..0000000 --- a/app/backup/build/backup-matrix/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM golang:buster as builder - -WORKDIR /root -RUN git clone https://filippo.io/age && cd age/cmd/age && go build -o age . - -FROM amd64/debian:buster - -COPY --from=builder /root/age/cmd/age/age /usr/local/bin/age - -RUN apt-get update && \ - apt-get -qq -y full-upgrade && \ - apt-get install -y rsync wget openssh-client postgresql-client && \ - apt-get clean && \ - rm -f /var/lib/apt/lists/*_* - -RUN mkdir -p /root/.ssh -WORKDIR /root - -COPY do_backup.sh /root/do_backup.sh - -CMD "/root/do_backup.sh" - diff --git a/app/backup/build/backup-matrix/do_backup.sh b/app/backup/build/backup-matrix/do_backup.sh deleted file mode 100755 index 7461409..0000000 --- a/app/backup/build/backup-matrix/do_backup.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh - -set -x -e - -cd /root - -chmod 0600 .ssh/id_ed25519 - -cat > .ssh/config <<EOF -Host backuphost - HostName $TARGET_SSH_HOST - Port $TARGET_SSH_PORT - User $TARGET_SSH_USER -EOF - -echo "export sql" -export PGPASSWORD=$REPL_PSQL_PWD -pg_basebackup \ - --pgdata=- \ - --format=tar \ - --max-rate=1M \ - --no-slot \ - --wal-method=none \ - --gzip \ - --compress=8 \ - --checkpoint=spread \ - --progress \ - --verbose \ - --status-interval=10 \ - --username=$REPL_PSQL_USER \ - --port=5432 \ - --host=psql-proxy.service.2.cluster.deuxfleurs.fr | \ - age -r "$(cat /root/.ssh/id_ed25519.pub)" | \ - ssh backuphost "cat > $TARGET_SSH_DIR/matrix/db-$(date --iso-8601=minute).gz.age" - -MATRIX_MEDIA="/mnt/glusterfs/chat/matrix/synapse/media" -echo "export local_content" -tar -vzcf - ${MATRIX_MEDIA} | \ - age -r "$(cat /root/.ssh/id_ed25519.pub)" | \ - ssh backuphost "cat > $TARGET_SSH_DIR/matrix/media-$(date --iso-8601=minute).gz.age" diff --git a/app/backup/build/backup-psql/default.nix b/app/backup/build/backup-psql/default.nix new file mode 100644 index 0000000..94dd4e1 --- /dev/null +++ b/app/backup/build/backup-psql/default.nix @@ -0,0 +1,16 @@ +{ pkgs ? import <nixpkgs> {} }: +let + python-with-my-packages = pkgs.python3.withPackages (p: with p; [ + minio + ]); +in +pkgs.mkShell { + buildInputs = [ + python-with-my-packages + pkgs.age + pkgs.postgresql_14 + ]; + shellHook = '' + PYTHONPATH=${python-with-my-packages}/${python-with-my-packages.sitePackages} + ''; +} diff --git a/app/backup/build/backup-psql/do_backup.py b/app/backup/build/backup-psql/do_backup.py new file mode 100755 index 0000000..fa0b94e --- /dev/null +++ b/app/backup/build/backup-psql/do_backup.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +import shutil,sys,os,datetime,minio + +working_directory = "." +if 'CACHE_DIR' in os.environ: working_directory = os.environ['CACHE_DIR'] +required_space_in_bytes = 20 * 1024 * 1024 * 1024 +bucket = os.environ['AWS_BUCKET'] +key = os.environ['AWS_ACCESS_KEY_ID'] +secret = os.environ['AWS_SECRET_ACCESS_KEY'] +endpoint = os.environ['AWS_ENDPOINT'] +pubkey = os.environ['CRYPT_PUBLIC_KEY'] +psql_host = os.environ['PSQL_HOST'] +psql_user = os.environ['PSQL_USER'] +s3_prefix = str(datetime.datetime.now()) +files = [ "backup_manifest", "base.tar.gz", "pg_wal.tar.gz" ] +clear_paths = [ os.path.join(working_directory, f) for f in files ] +crypt_paths = [ os.path.join(working_directory, f) + ".age" for f in files ] +s3_keys = [ s3_prefix + "/" + f for f in files ] + +def abort(msg): + for p in clear_paths + crypt_paths: + if os.path.exists(p): + print(f"Remove {p}") + os.remove(p) + + if msg: sys.exit(msg) + else: print("success") + +# Check we have enough space on disk +if shutil.disk_usage(working_directory).free < required_space_in_bytes: + abort(f"Not enough space on disk at path {working_directory} to perform a backup, aborting") + +# Check postgres password is set +if 'PGPASSWORD' not in os.environ: + abort(f"You must pass postgres' password through the environment variable PGPASSWORD") + +# Check our working directory is empty +if len(os.listdir(working_directory)) != 0: + abort(f"Working directory {working_directory} is not empty, aborting") + +# Check Minio +client = minio.Minio(endpoint, key, secret) +if not client.bucket_exists(bucket): + abort(f"Bucket {bucket} does not exist or its access is forbidden, aborting") + +# Perform the backup locally +ret = os.system(f""" +pg_basebackup \ + --host={psql_host} \ + --username={psql_user} \ + --pgdata={working_directory} \ + --format=tar \ + --wal-method=stream \ + --gzip \ + --compress=6 \ + --progress \ + --max-rate=5M +""") +if ret != 0: + abort(f"pg_baseckup exit code is {ret}, 0 expected. aborting") + +# Check that the expected files are here +for p in clear_paths: + print(f"Checking that {p} exists locally") + if not os.path.exists(p): + abort(f"File {p} expected but not found, aborting") + +# Cipher them +for c, e in zip(clear_paths, crypt_paths): + print(f"Ciphering {c} to {e}") + ret = os.system(f"age -r {pubkey} -o {e} {c}") + if ret != 0: + abort(f"age exit code is {ret}, 0 expected. aborting") + +# Upload the backup to S3 +for p, k in zip(crypt_paths, s3_keys): + try: + print(f"Uploading {p} to {k}") + result = client.fput_object(bucket, k, p) + print( + "created {0} object; etag: {1}, version-id: {2}".format( + result.object_name, result.etag, result.version_id, + ), + ) + except Exception as e: + abort(f"Exception {e} occured while upload {p}. aborting") + +# Check that the files have been uploaded +for k in s3_keys: + try: + print(f"Checking that {k} exists remotely") + result = client.stat_object(bucket, k) + print( + "last-modified: {0}, size: {1}".format( + result.last_modified, result.size, + ), + ) + except Exception as e: + abort(f"{k} not found on S3. {e}. aborting") + +abort(None) diff --git a/op_guide/stolon/nomad_full_backup.md b/op_guide/stolon/nomad_full_backup.md new file mode 100644 index 0000000..574043a --- /dev/null +++ b/op_guide/stolon/nomad_full_backup.md @@ -0,0 +1,26 @@ +Start by following ../backup-minio + +## Garbage collect old backups + +``` +mc ilm import deuxfleurs/${BUCKET_NAME} <<EOF +{ + "Rules": [ + { + "Expiration": { + "Days": 62 + }, + "ID": "PurgeOldBackups", + "Status": "Enabled" + } + ] +} +EOF +``` + +Check that it has been activated: + +``` + mc ilm ls deuxfleurs/${BUCKET_NAME} +``` + |