From c74dc92febd1841c8ea5ff31caab0f941d57527d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sat, 16 Jan 2021 17:07:01 +0100 Subject: Proposal: reorganize app/ folder by modules --- app/postgres/build/postgres/Dockerfile | 19 +++ app/postgres/build/postgres/README.md | 4 + app/postgres/build/postgres/postgresql.conf | 25 ++++ app/postgres/build/postgres/start.sh | 22 ++++ app/postgres/config/keeper/env.tpl | 3 + app/postgres/deploy/postgres.hcl | 134 +++++++++++++++++++++ .../secrets/postgres/keeper/pg_repl_pwd.sample | 0 .../postgres/keeper/pg_repl_username.sample | 0 .../secrets/postgres/keeper/pg_su_pwd.sample | 0 9 files changed, 207 insertions(+) create mode 100644 app/postgres/build/postgres/Dockerfile create mode 100644 app/postgres/build/postgres/README.md create mode 100644 app/postgres/build/postgres/postgresql.conf create mode 100755 app/postgres/build/postgres/start.sh create mode 100644 app/postgres/config/keeper/env.tpl create mode 100644 app/postgres/deploy/postgres.hcl create mode 100644 app/postgres/secrets/postgres/keeper/pg_repl_pwd.sample create mode 100644 app/postgres/secrets/postgres/keeper/pg_repl_username.sample create mode 100644 app/postgres/secrets/postgres/keeper/pg_su_pwd.sample (limited to 'app/postgres') diff --git a/app/postgres/build/postgres/Dockerfile b/app/postgres/build/postgres/Dockerfile new file mode 100644 index 0000000..bb018b8 --- /dev/null +++ b/app/postgres/build/postgres/Dockerfile @@ -0,0 +1,19 @@ +FROM amd64/debian:stretch + +RUN echo "deb http://deb.debian.org/debian stretch-backports main contrib non-free # available after stretch release" > /etc/apt/sources.list.d/stretch-backports.list && \ + apt-get update && \ + apt-get -qq -y full-upgrade && \ + apt-get install -y postgresql-all golang-1.11 git && \ + export GOPATH=/usr/local/go && \ + mkdir -p /usr/local/go/src/github.com/sorintlab && \ + cd /usr/local/go/src/github.com/sorintlab && \ + git clone --depth=1 https://github.com/sorintlab/stolon && \ + ln -s /usr/lib/go-1.11/bin/go /usr/bin/go && \ + ln -s /usr/lib/go-1.11/bin/gofmt /usr/bin/gofmt && \ + cd ./stolon && \ + ./build && \ + mv /usr/local/go/src/github.com/sorintlab/stolon/bin/* /usr/local/bin/ && \ + rm -rf /usr/local/go + +USER postgres + diff --git a/app/postgres/build/postgres/README.md b/app/postgres/build/postgres/README.md new file mode 100644 index 0000000..d2f7a12 --- /dev/null +++ b/app/postgres/build/postgres/README.md @@ -0,0 +1,4 @@ +``` +docker build -t superboum/arm32v7_postgres . +docker build -t superboum/amd64_postgres:v2 . +``` diff --git a/app/postgres/build/postgres/postgresql.conf b/app/postgres/build/postgres/postgresql.conf new file mode 100644 index 0000000..8e0af2b --- /dev/null +++ b/app/postgres/build/postgres/postgresql.conf @@ -0,0 +1,25 @@ +data_directory = '/var/lib/postgresql/9.6/main' # use data in another directory +hba_file = '/etc/postgresql/9.6/main/pg_hba.conf' # host-based authentication file +ident_file = '/etc/postgresql/9.6/main/pg_ident.conf' # ident configuration file +external_pid_file = '/var/run/postgresql/9.6-main.pid' # write an extra PID file +listen_addresses = '*' #listen on every ip / interfaces +port = 5432 # (change requires restart) +max_connections = 100 # (change requires restart) +unix_socket_directories = '/var/run/postgresql' # comma-separated list of directories +ssl = true # (change requires restart) +ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' # (change requires restart) +ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' # (change requires restart) +shared_buffers = 128MB # min 128kB +dynamic_shared_memory_type = posix # the default is the first option +log_line_prefix = '%m [%p] %q%u@%d ' # special values: +log_timezone = 'UTC' +cluster_name = '9.6/main' # added to process titles if nonempty +stats_temp_directory = '/var/run/postgresql/9.6-main.pg_stat_tmp' +datestyle = 'iso, mdy' +timezone = 'UTC' +lc_messages = 'C.UTF-8' # locale for system error message +lc_monetary = 'C.UTF-8' # locale for monetary formatting +lc_numeric = 'C.UTF-8' # locale for number formatting +lc_time = 'C.UTF-8' # locale for time formatting +default_text_search_config = 'pg_catalog.english' + diff --git a/app/postgres/build/postgres/start.sh b/app/postgres/build/postgres/start.sh new file mode 100755 index 0000000..f1d493f --- /dev/null +++ b/app/postgres/build/postgres/start.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ -f /local/pg_hba.conf ]; then + echo "Copying Nomad configuration..." + cp /local/pg_hba.conf /etc/postgresql/9.6/main/ + echo "Done" +fi + + +if [ -z "$(ls -A /var/lib/postgresql/9.6/main)" ]; then + echo "Copying base" + cp -r /var/lib/postgresql/9.6/base/* /var/lib/postgresql/9.6/main + echo "Done" +fi + +chmod -R 700 /var/lib/postgresql/9.6/main +chown -R postgres /var/lib/postgresql/9.6/main + +echo "Starting postgres..." +. /usr/share/postgresql-common/init.d-functions +start 9.6 +tail -f /var/log/postgresql/postgresql-9.6-main.log diff --git a/app/postgres/config/keeper/env.tpl b/app/postgres/config/keeper/env.tpl new file mode 100644 index 0000000..7831aad --- /dev/null +++ b/app/postgres/config/keeper/env.tpl @@ -0,0 +1,3 @@ +PG_SU_PWD={{ key "secrets/postgres/keeper/pg_su_pwd" | trimSpace }} +PG_REPL_USER={{ key "secrets/postgres/keeper/pg_repl_username" | trimSpace }} +PG_REPL_PWD={{ key "secrets/postgres/keeper/pg_repl_pwd" | trimSpace }} diff --git a/app/postgres/deploy/postgres.hcl b/app/postgres/deploy/postgres.hcl new file mode 100644 index 0000000..f5eec51 --- /dev/null +++ b/app/postgres/deploy/postgres.hcl @@ -0,0 +1,134 @@ +job "postgres" { + datacenters = ["dc1"] + type = "system" + priority = 90 + + update { + max_parallel = 1 + stagger = "2m" + } + + group "postgres" { + network { + port "psql_proxy_port" { static = 5432 } + port "psql_port" { static = 5433 } + } + + task "sentinel" { + driver = "docker" + + config { + image = "superboum/amd64_postgres:v3" + network_mode = "host" + readonly_rootfs = false + command = "/usr/local/bin/stolon-sentinel" + args = [ + "--cluster-name", "pissenlit", + "--store-backend", "consul", + "--store-endpoints", "http://consul.service.2.cluster.deuxfleurs.fr:8500", + ] + } + resources { + memory = 100 + } + } + + task "proxy" { + driver = "docker" + + config { + image = "superboum/amd64_postgres:v3" + network_mode = "host" + readonly_rootfs = false + command = "/usr/local/bin/stolon-proxy" + args = [ + "--cluster-name", "pissenlit", + "--store-backend", "consul", + "--store-endpoints", "http://consul.service.2.cluster.deuxfleurs.fr:8500", + "--port", "${NOMAD_PORT_psql_proxy_port}", + "--listen-address", "0.0.0.0" + ] + ports = [ "psql_proxy_port" ] + } + + resources { + memory = 100 + } + + service { + tags = ["sql"] + port = "psql_proxy_port" + address_mode = "host" + name = "psql-proxy" + check { + type = "tcp" + port = "psql_proxy_port" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "10m" + ignore_warnings = false + } + } + } + } + + task "keeper" { + driver = "docker" + + config { + image = "superboum/amd64_postgres:v3" + network_mode = "host" + readonly_rootfs = false + command = "/usr/local/bin/stolon-keeper" + args = [ + "--cluster-name", "pissenlit", + "--store-backend", "consul", + "--store-endpoints", "http://consul.service.2.cluster.deuxfleurs.fr:8500", + "--data-dir", "/mnt/persist", + "--pg-su-password", "${PG_SU_PWD}", + "--pg-repl-username", "${PG_REPL_USER}", + "--pg-repl-password", "${PG_REPL_PWD}", + "--pg-listen-address", "${attr.unique.network.ip-address}", + "--pg-port", "${NOMAD_PORT_psql_port}", + "--pg-bin-path", "/usr/lib/postgresql/9.6/bin/" + ] + ports = [ "psql_port" ] + volumes = [ + "/mnt/ssd/postgres:/mnt/persist" + ] + } + + template { + data = file("../config/keeper/env.tpl") + destination = "secrets/env" + env = true + } + + resources { + memory = 500 + } + + service { + tags = ["sql"] + port = "psql_port" + address_mode = "host" + name = "keeper" + check { + type = "tcp" + port = "psql_port" + interval = "60s" + timeout = "5s" + + check_restart { + limit = 3 + grace = "60m" + ignore_warnings = false + } + } + } + } + } +} + diff --git a/app/postgres/secrets/postgres/keeper/pg_repl_pwd.sample b/app/postgres/secrets/postgres/keeper/pg_repl_pwd.sample new file mode 100644 index 0000000..e69de29 diff --git a/app/postgres/secrets/postgres/keeper/pg_repl_username.sample b/app/postgres/secrets/postgres/keeper/pg_repl_username.sample new file mode 100644 index 0000000..e69de29 diff --git a/app/postgres/secrets/postgres/keeper/pg_su_pwd.sample b/app/postgres/secrets/postgres/keeper/pg_su_pwd.sample new file mode 100644 index 0000000..e69de29 -- cgit v1.2.3 From d4d0b100ad39bf7ae560c2f714b75fdcf47e9a87 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sat, 16 Jan 2021 17:37:34 +0100 Subject: Document secrets and add stub utility to manage them --- app/postgres/secrets/postgres/keeper/pg_repl_pwd | 1 + app/postgres/secrets/postgres/keeper/pg_repl_pwd.sample | 0 app/postgres/secrets/postgres/keeper/pg_repl_username | 1 + app/postgres/secrets/postgres/keeper/pg_repl_username.sample | 0 app/postgres/secrets/postgres/keeper/pg_su_pwd | 1 + app/postgres/secrets/postgres/keeper/pg_su_pwd.sample | 0 6 files changed, 3 insertions(+) create mode 100644 app/postgres/secrets/postgres/keeper/pg_repl_pwd delete mode 100644 app/postgres/secrets/postgres/keeper/pg_repl_pwd.sample create mode 100644 app/postgres/secrets/postgres/keeper/pg_repl_username delete mode 100644 app/postgres/secrets/postgres/keeper/pg_repl_username.sample create mode 100644 app/postgres/secrets/postgres/keeper/pg_su_pwd delete mode 100644 app/postgres/secrets/postgres/keeper/pg_su_pwd.sample (limited to 'app/postgres') diff --git a/app/postgres/secrets/postgres/keeper/pg_repl_pwd b/app/postgres/secrets/postgres/keeper/pg_repl_pwd new file mode 100644 index 0000000..ae0c229 --- /dev/null +++ b/app/postgres/secrets/postgres/keeper/pg_repl_pwd @@ -0,0 +1 @@ +SERVICE_PASSWORD replicator diff --git a/app/postgres/secrets/postgres/keeper/pg_repl_pwd.sample b/app/postgres/secrets/postgres/keeper/pg_repl_pwd.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/postgres/secrets/postgres/keeper/pg_repl_username b/app/postgres/secrets/postgres/keeper/pg_repl_username new file mode 100644 index 0000000..58e6e46 --- /dev/null +++ b/app/postgres/secrets/postgres/keeper/pg_repl_username @@ -0,0 +1 @@ +CONST replicator diff --git a/app/postgres/secrets/postgres/keeper/pg_repl_username.sample b/app/postgres/secrets/postgres/keeper/pg_repl_username.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/postgres/secrets/postgres/keeper/pg_su_pwd b/app/postgres/secrets/postgres/keeper/pg_su_pwd new file mode 100644 index 0000000..a193b9e --- /dev/null +++ b/app/postgres/secrets/postgres/keeper/pg_su_pwd @@ -0,0 +1 @@ +SERVICE_PASSWORD postgres diff --git a/app/postgres/secrets/postgres/keeper/pg_su_pwd.sample b/app/postgres/secrets/postgres/keeper/pg_su_pwd.sample deleted file mode 100644 index e69de29..0000000 -- cgit v1.2.3 From 850ccbf1c7c4ebba28b1971bafae0a6ba922b7c7 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sat, 16 Jan 2021 20:03:00 +0100 Subject: secretmgr.py does quite a few things! --- app/postgres/secrets/postgres/keeper/pg_su_pwd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/postgres') diff --git a/app/postgres/secrets/postgres/keeper/pg_su_pwd b/app/postgres/secrets/postgres/keeper/pg_su_pwd index a193b9e..907e2b8 100644 --- a/app/postgres/secrets/postgres/keeper/pg_su_pwd +++ b/app/postgres/secrets/postgres/keeper/pg_su_pwd @@ -1 +1 @@ -SERVICE_PASSWORD postgres +USER postgres superuser password -- cgit v1.2.3