diff options
24 files changed, 237 insertions, 105 deletions
diff --git a/cluster/prod/app/garage/deploy/garage.hcl b/cluster/prod/app/garage/deploy/garage.hcl index b6ea7f6..68edc94 100644 --- a/cluster/prod/app/garage/deploy/garage.hcl +++ b/cluster/prod/app/garage/deploy/garage.hcl @@ -80,9 +80,9 @@ job "garage" { #### Configuration for service ports: admin port (internal use only) service { + name = "garage-admin" port = "admin" address_mode = "host" - name = "garage-admin" # Check that Garage is alive and answering TCP connections check { type = "tcp" @@ -96,18 +96,19 @@ job "garage" { } } - #### Configuration for service ports: externally available ports (API, web) + #### Configuration for service ports: externally available ports (S3 API, K2V, web) service { + name = "garage-api" tags = [ "garage_api", "tricot garage.deuxfleurs.fr", "tricot *.garage.deuxfleurs.fr", + "tricot-on-demand-tls-ask http://garage-admin.service.prod.consul:3903/check", "tricot-site-lb", ] port = "s3" address_mode = "host" - name = "garage-api" # Check 1: Garage is alive and answering TCP connections check { name = "garage-api-live" @@ -132,6 +133,39 @@ job "garage" { } service { + name = "garage-k2v" + tags = [ + "garage_k2v", + "tricot k2v.deuxfleurs.fr", + "tricot-site-lb", + ] + port = "k2v" + address_mode = "host" + # Check 1: Garage is alive and answering TCP connections + check { + name = "garage-k2v-live" + type = "tcp" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + # Check 2: Garage is in a healthy state and requests should be routed here + check { + name = "garage-k2v-healthy" + port = "admin" + type = "http" + path = "/health" + interval = "60s" + timeout = "5s" + } + } + + service { + name = "garage-web" tags = [ "garage-web", "tricot * 1", @@ -144,7 +178,6 @@ job "garage" { ] port = "web" address_mode = "host" - name = "garage-web" # Check 1: Garage is alive and answering TCP connections check { name = "garage-web-live" @@ -183,39 +216,6 @@ job "garage" { port = "web" on_update = "ignore" } - - - service { - tags = [ - "garage_k2v", - "tricot k2v.deuxfleurs.fr", - "tricot-site-lb", - ] - port = "k2v" - address_mode = "host" - name = "garage-k2v" - # Check 1: Garage is alive and answering TCP connections - check { - name = "garage-k2v-live" - type = "tcp" - interval = "60s" - timeout = "5s" - check_restart { - limit = 3 - grace = "90s" - ignore_warnings = false - } - } - # Check 2: Garage is in a healthy state and requests should be routed here - check { - name = "garage-k2v-healthy" - port = "admin" - type = "http" - path = "/health" - interval = "60s" - timeout = "5s" - } - } } } } diff --git a/cluster/prod/cluster.nix b/cluster/prod/cluster.nix index 4b9b41a..c5008e6 100644 --- a/cluster/prod/cluster.nix +++ b/cluster/prod/cluster.nix @@ -75,6 +75,24 @@ address = "10.83.4.2"; endpoint = "82.65.41.110:33742"; }; + "onion" = { + siteName = "dathomir"; + publicKey = "gpeqalqAUaYlMuebv3glQeZyE64+OpkyIHFhfStJQA4="; + address = "10.83.5.1"; + endpoint = "82.64.238.84:33740"; + }; + "oseille" = { + siteName = "dathomir"; + publicKey = "T87GzAQt02i00iOMbEm7McA/VL9OBrG/kCrgoNh5MmY="; + address = "10.83.5.2"; + endpoint = "82.64.238.84:33741"; + }; + "io" = { + siteName = "dathomir"; + publicKey = "3+VvWJtABOAd6zUCMROhqGbNtkQRtoIkVmYn0M81jQw="; + address = "10.83.5.3"; + endpoint = "82.64.238.84:33742"; + }; }; # Pin Nomad version diff --git a/cluster/prod/known_hosts b/cluster/prod/known_hosts index 1c80749..3b6bf7f 100644 --- a/cluster/prod/known_hosts +++ b/cluster/prod/known_hosts @@ -9,3 +9,6 @@ concombre.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL3N0QOFNGkC courgette.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPCXJeo6yeQeTN7D7OZwLd8zbyU1jWywlhQ29yyk7x+G abricot.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPhHUQtc5lukPMFkiWf/sTgaUpwNFXHCJoQKu4ooRFy+ ananas.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHs0zAyBy70oyV56qaMaMAKR7VjEDnsm5LEyZJbM95BL +onion.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINjBQ67fxwuDDzRPveTko/Sgf0cev3tIvlr3CfAmhF0C +oseille.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAgQdQ5UVFFn+DXN90ut9+V7NtEopQJnES3r8soKTZW4 +io.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIvgCJ7Jew7ou1RZuaT41Sd+ucZAgxUwtdieqNqoC3+T diff --git a/cluster/prod/node/io.nix b/cluster/prod/node/io.nix new file mode 100644 index 0000000..e6d3c3a --- /dev/null +++ b/cluster/prod/node/io.nix @@ -0,0 +1,11 @@ +{ ... }: +{ + boot.loader.grub.enable = true; + boot.loader.grub.device = "/dev/sda"; + + services.openssh.ports = [ 22 33603 ]; + + deuxfleurs.hostName = "io"; + deuxfleurs.staticIPv4.address = "192.168.1.36"; + deuxfleurs.staticIPv6.address = "2a01:e0a:5e4:1d0:52e5:49ff:fe5c:5f35"; +} diff --git a/cluster/prod/node/io.site.nix b/cluster/prod/node/io.site.nix new file mode 120000 index 0000000..5e41391 --- /dev/null +++ b/cluster/prod/node/io.site.nix @@ -0,0 +1 @@ +../site/dathomir.nix
\ No newline at end of file diff --git a/cluster/prod/node/onion.nix b/cluster/prod/node/onion.nix new file mode 100644 index 0000000..8b6f8d6 --- /dev/null +++ b/cluster/prod/node/onion.nix @@ -0,0 +1,12 @@ +{ ... }: +{ + boot.loader.systemd-boot.enable = true; + boot.loader.timeout = 20; + boot.loader.efi.canTouchEfiVariables = true; + + services.openssh.ports = [ 22 33601 ]; + + deuxfleurs.hostName = "onion"; + deuxfleurs.staticIPv4.address = "192.168.1.34"; + deuxfleurs.staticIPv6.address = "2a01:e0a:5e4:1d0:223:24ff:feb0:e866"; +} diff --git a/cluster/prod/node/onion.site.nix b/cluster/prod/node/onion.site.nix new file mode 120000 index 0000000..5e41391 --- /dev/null +++ b/cluster/prod/node/onion.site.nix @@ -0,0 +1 @@ +../site/dathomir.nix
\ No newline at end of file diff --git a/cluster/prod/node/oseille.nix b/cluster/prod/node/oseille.nix new file mode 100644 index 0000000..b0f7723 --- /dev/null +++ b/cluster/prod/node/oseille.nix @@ -0,0 +1,12 @@ +{ ... }: +{ + boot.loader.systemd-boot.enable = true; + boot.loader.timeout = 20; + boot.loader.efi.canTouchEfiVariables = true; + + services.openssh.ports = [ 22 33602 ]; + + deuxfleurs.hostName = "oseille"; + deuxfleurs.staticIPv4.address = "192.168.1.35"; + deuxfleurs.staticIPv6.address = "2a01:e0a:5e4:1d0:223:24ff:feaf:f90b"; +} diff --git a/cluster/prod/node/oseille.site.nix b/cluster/prod/node/oseille.site.nix new file mode 120000 index 0000000..5e41391 --- /dev/null +++ b/cluster/prod/node/oseille.site.nix @@ -0,0 +1 @@ +../site/dathomir.nix
\ No newline at end of file diff --git a/cluster/prod/site/dathomir.nix b/cluster/prod/site/dathomir.nix new file mode 100644 index 0000000..87d9b56 --- /dev/null +++ b/cluster/prod/site/dathomir.nix @@ -0,0 +1,7 @@ +{ ... }: +{ + deuxfleurs.siteName = "dathomir"; + deuxfleurs.cnameTarget = "dathomir.site.deuxfleurs.fr"; + deuxfleurs.publicIPv4 = "82.64.238.84"; + deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; +} diff --git a/cluster/prod/ssh_config b/cluster/prod/ssh_config index 37b8953..d7aeafd 100644 --- a/cluster/prod/ssh_config +++ b/cluster/prod/ssh_config @@ -1,35 +1,43 @@ UserKnownHostsFile ./cluster/prod/known_hosts Host concombre - HostName concombre.machine.deuxfleurs.fr + HostName concombre.machine.deuxfleurs.fr Host courgette - HostName courgette.machine.deuxfleurs.fr + HostName courgette.machine.deuxfleurs.fr Host celeri - HostName celeri.machine.deuxfleurs.fr + HostName celeri.machine.deuxfleurs.fr Host dahlia - HostName dahlia.machine.deuxfleurs.fr + HostName dahlia.machine.deuxfleurs.fr Host diplotaxis - HostName diplotaxis.machine.deuxfleurs.fr + HostName diplotaxis.machine.deuxfleurs.fr Host doradille - HostName doradille.machine.deuxfleurs.fr + HostName doradille.machine.deuxfleurs.fr Host df-ykl - HostName df-ykl.machine.deuxfleurs.fr + HostName df-ykl.machine.deuxfleurs.fr Host df-ymf - HostName df-ymf.machine.deuxfleurs.fr + HostName df-ymf.machine.deuxfleurs.fr Host df-ymk - HostName df-ymk.machine.deuxfleurs.fr + HostName df-ymk.machine.deuxfleurs.fr Host abricot - HostName abricot.machine.deuxfleurs.fr + HostName abricot.machine.deuxfleurs.fr Host ananas - HostName ananas.machine.deuxfleurs.fr + HostName ananas.machine.deuxfleurs.fr +Host onion + HostName onion.machine.deuxfleurs.fr + +Host oseille + HostName oseille.machine.deuxfleurs.fr + +Host io + HostName io.machine.deuxfleurs.fr diff --git a/cluster/staging/app/core/deploy/diplonat.hcl b/cluster/staging/app/core/deploy/diplonat.hcl index 16bc27b..3e3310f 100644 --- a/cluster/staging/app/core/deploy/diplonat.hcl +++ b/cluster/staging/app/core/deploy/diplonat.hcl @@ -54,7 +54,7 @@ job "core-diplonat" { data = <<EOH DIPLONAT_REFRESH_TIME=60 DIPLONAT_EXPIRATION_TIME=300 -DIPLONAT_IPV6_ONLY=true +DIPLONAT_IPV6_ONLY={{ $site := env "meta.site" }}{{ if eq $site "corrin" }}false{{ else }}true{{ end }} DIPLONAT_CONSUL_NODE_NAME={{ env "attr.unique.hostname" }} DIPLONAT_CONSUL_URL=https://localhost:8501 DIPLONAT_CONSUL_CA_CERT=/etc/diplonat/consul-ca.crt diff --git a/cluster/staging/app/core/deploy/tricot.hcl b/cluster/staging/app/core/deploy/tricot.hcl index 62c8030..7227c6f 100644 --- a/cluster/staging/app/core/deploy/tricot.hcl +++ b/cluster/staging/app/core/deploy/tricot.hcl @@ -25,7 +25,7 @@ job "core-tricot" { config { packages = [ - "git+https://git.deuxfleurs.fr/Deuxfleurs/tricot.git?ref=redirect&rev=b76b6dcbcc47ebc61848389a6b0d5d4e8d8cde48" + "git+https://git.deuxfleurs.fr/Deuxfleurs/tricot.git?ref=main&rev=9bb505d977cb8bafd8039159241788ff25510d69" ] command = "tricot" # cap_add = [ "net_bind_service" ] # this doesn't work for whatever reason, so we need to put user = "root" instead @@ -71,6 +71,7 @@ TRICOT_CONSUL_CLIENT_KEY=/etc/tricot/consul-client.key TRICOT_HTTP_BIND_ADDR=[::]:80 TRICOT_HTTPS_BIND_ADDR=[::]:443 TRICOT_METRICS_BIND_ADDR=[::]:9334 +TRICOT_WARMUP_CERT_MEMORY_STORE=true RUST_LOG=tricot=debug RUST_BACKTRACE=1 EOH @@ -82,9 +83,6 @@ EOH name = "tricot-http" port = "http_port" tags = [ - "d53-aaaa ${attr.unique.hostname}.machine.staging.deuxfleurs.org", - "d53-aaaa ${meta.site}.site.staging.deuxfleurs.org", - "d53-aaaa staging.deuxfleurs.org", "(diplonat (tcp_port 80))" ] address_mode = "host" @@ -94,7 +92,10 @@ EOH name = "tricot-https" port = "https_port" tags = [ - "(diplonat (tcp_port 443))" + "(diplonat (tcp_port 443))", + "d53-aaaa ${attr.unique.hostname}.machine.staging.deuxfleurs.org", + "d53-aaaa ${meta.site}.site.staging.deuxfleurs.org", + "d53-aaaa staging.deuxfleurs.org" ] address_mode = "host" } diff --git a/cluster/staging/app/garage/config/garage.toml b/cluster/staging/app/garage/config/garage.toml index 6c92bf3..01eb237 100644 --- a/cluster/staging/app/garage/config/garage.toml +++ b/cluster/staging/app/garage/config/garage.toml @@ -25,6 +25,7 @@ tls_skip_verify = true [s3_api] s3_region = "garage-staging" api_bind_addr = "0.0.0.0:3990" +root_domain = ".garage.staging.deuxfleurs.org" [k2v_api] api_bind_addr = "0.0.0.0:3993" diff --git a/cluster/staging/app/garage/deploy/garage.hcl b/cluster/staging/app/garage/deploy/garage.hcl index 67655e4..b8e7227 100644 --- a/cluster/staging/app/garage/deploy/garage.hcl +++ b/cluster/staging/app/garage/deploy/garage.hcl @@ -73,22 +73,68 @@ job "garage-staging" { kill_signal = "SIGINT" kill_timeout = "20s" + restart { + interval = "5m" + attempts = 10 + delay = "1m" + mode = "delay" + } + service { name = "garage-staging-rpc" tags = ["garage-staging-rpc"] port = "rpc" } + #### Configuration for service ports: admin port (internal use only) + + service { + name = "garage-staging-admin" + tags = [ + "garage-staging-admin", + ] + port = "admin" + check { + name = "garage-tcp-liveness-check" + type = "tcp" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + } + + #### Configuration for service ports: externally available ports (S3 API, K2V, web) + service { name = "garage-staging-s3-api" tags = [ "garage-staging-api", "tricot garage.staging.deuxfleurs.org", + "tricot *.garage.staging.deuxfleurs.org", "tricot-add-header Access-Control-Allow-Origin *", + "tricot-on-demand-tls-ask http://garage-staging-admin.service.staging.consul:3909/check", "tricot-site-lb", ] port = "s3" + # Check 1: Garage is alive and answering TCP connections + check { + name = "garage-staging-api-live" + type = "tcp" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + # Check 2: Garage is in a healthy state and requests should be routed here check { + name = "garage-staging-api-healthy" port = "admin" type = "http" path = "/health" @@ -106,7 +152,21 @@ job "garage-staging" { "tricot-site-lb", ] port = "k2v" + # Check 1: Garage is alive and answering TCP connections + check { + name = "garage-staging-k2v-live" + type = "tcp" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + # Check 2: Garage is in a healthy state and requests should be routed here check { + name = "garage-staging-k2v-healthy" port = "admin" type = "http" path = "/health" @@ -119,59 +179,41 @@ job "garage-staging" { name = "garage-staging-web" tags = [ "garage-staging-web", + "tricot * 1", "tricot *.web.staging.deuxfleurs.org", "tricot staging.deuxfleurs.org", "tricot matrix.home.adnab.me/.well-known/matrix/server", + "tricot-add-header Strict-Transport-Security max-age=63072000; includeSubDomains; preload", + "tricot-add-header X-Frame-Options SAMEORIGIN", + "tricot-add-header X-XSS-Protection 1; mode=block", + "tricot-add-header X-Content-Type-Options nosniff", "tricot-add-header Access-Control-Allow-Origin *", + "tricot-on-demand-tls-ask http://garage-staging-admin.service.staging.consul:3909/check", "tricot-site-lb", ] port = "web" + # Check 1: Garage is alive and answering TCP connections check { - port = "admin" - type = "http" - path = "/health" - interval = "60s" - timeout = "5s" - } - } - - service { - name = "garage-staging-admin" - tags = [ - "garage-staging-admin", - ] - port = "admin" - check { - name = "garage-admin-health-check" - type = "http" - path = "/health" + name = "garage-staging-web-live" + type = "tcp" interval = "60s" timeout = "5s" check_restart { - limit = 10 + limit = 3 grace = "90s" - ignore_warnings = true + ignore_warnings = false } } + # Check 2: Garage is in a healthy state and requests should be routed here check { - name = "garage-tcp-liveness-check" - type = "tcp" + name = "garage-staging-web-healthy" + port = "admin" + type = "http" + path = "/health" interval = "60s" timeout = "5s" - check_restart { - limit = 3 - grace = "90s" - ignore_warnings = true - } } } - - restart { - interval = "5m" - attempts = 10 - delay = "1m" - mode = "delay" - } } } } diff --git a/cluster/staging/app/telemetry/deploy/telemetry-service.hcl b/cluster/staging/app/telemetry/deploy/telemetry-service.hcl index bf777fd..ec68aaf 100644 --- a/cluster/staging/app/telemetry/deploy/telemetry-service.hcl +++ b/cluster/staging/app/telemetry/deploy/telemetry-service.hcl @@ -79,12 +79,6 @@ job "telemetry-service" { group "grafana" { count = 1 - constraint { - attribute = "${attr.unique.hostname}" - operator = "!=" - value = "piranha" - } - network { port "grafana" { static = 3719 diff --git a/cluster/staging/node/piranha.nix b/cluster/staging/node/piranha.nix index 5783e6a..2dc0677 100644 --- a/cluster/staging/node/piranha.nix +++ b/cluster/staging/node/piranha.nix @@ -9,8 +9,8 @@ boot.loader.efi.canTouchEfiVariables = true; deuxfleurs.hostName = "piranha"; - deuxfleurs.staticIPv4.address = "192.168.1.25"; - deuxfleurs.staticIPv6.address = "2a01:cb05:911e:ec00:223:24ff:feb0:ea82"; + deuxfleurs.staticIPv4.address = "192.168.5.25"; + deuxfleurs.staticIPv6.address = "2001:912:1ac0:2200::25"; system.stateVersion = "22.11"; } diff --git a/cluster/staging/site/corrin.nix b/cluster/staging/site/corrin.nix index de1a28d..d07de32 100644 --- a/cluster/staging/site/corrin.nix +++ b/cluster/staging/site/corrin.nix @@ -2,7 +2,7 @@ { deuxfleurs.siteName = "corrin"; - deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; + deuxfleurs.staticIPv4.defaultGateway = "192.168.5.1"; deuxfleurs.cnameTarget = "corrin.site.staging.deuxfleurs.org."; - deuxfleurs.publicIPv4 = "109.222.162.50"; + deuxfleurs.publicIPv4 = "45.81.62.36"; } diff --git a/cluster/staging/ssh_config b/cluster/staging/ssh_config index afcdce5..4f38f47 100644 --- a/cluster/staging/ssh_config +++ b/cluster/staging/ssh_config @@ -9,11 +9,6 @@ Host origan HostName origan.machine.staging.deuxfleurs.org Host piranha - HostName %h.machine.staging.deuxfleurs.org - #HostName piranha.polyno.me - #OR - #ProxyJump caribou.machine.deuxfleurs.fr - #HostName 10.14.3.1 HostName piranha.machine.staging.deuxfleurs.org Host df-pw5 diff --git a/doc/architecture.md b/doc/architecture.md index 7d36643..96c6918 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -175,3 +175,12 @@ Then, other stuff can be started in any order, e.g.: - `app/cryptpad` - `app/drone-ci` + +## Operating garage + +Garage is operated using its command-line interface, which can be accessed using +any node of the cluster running garage: +``` +docker ps # to find the identifier of the container running garage +docker exec -ti <id> /garage <cli args...> +``` diff --git a/gather_facts b/gather_facts new file mode 100755 index 0000000..ac91d09 --- /dev/null +++ b/gather_facts @@ -0,0 +1,6 @@ +#!/usr/bin/env ./sshtool + +cmd lsblk -o name,size,type,mountpoint,rota,fstype,fsused,fsuse% +cmd "lscpu | grep 'Model name'" +cmd lscpu -e=cpu,minmhz,maxmhz,mhz +cmd lsmem --summary diff --git a/nix/configuration.nix b/nix/configuration.nix index d88d6ef..68751a2 100644 --- a/nix/configuration.nix +++ b/nix/configuration.nix @@ -65,6 +65,9 @@ SystemMaxUse=1G wireguard-tools ]; + # Enable support for all terminal emulators such as urxvt + environment.enableAllTerminfo = true; + programs.vim.defaultEditor = true; # Enable network time @@ -73,7 +76,7 @@ SystemMaxUse=1G # Enable the OpenSSH daemon and disable password login. services.openssh.enable = true; - services.openssh.passwordAuthentication = false; + services.openssh.settings.PasswordAuthentication = false; virtualisation.docker = { enable = true; diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 7b9065e..f9fd068 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -204,6 +204,13 @@ in # link-local addresses networkConfig.IPv6AcceptRA = mkIf noRA false; networkConfig.LinkLocalAddressing = mkIf noRA "no"; + + # By default, systemd-networkd may try to use DHCPv6 depending on RA flags. + # Disable DHCPv6 client and IPv6 Prefix Delegation in all cases. + ipv6AcceptRAConfig.DHCPv6Client = false; + dhcpV6Config.UseAddress = false; + dhcpV6Config.UseDelegatedPrefix = false; + }; # Configure Unbound as a central DNS server for everything @@ -13,7 +13,7 @@ CMDFILE=./$(basename $CMDFILE) CLUSTER="$1" if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then - echo "Usage: $CMDFILE <cluster name>" + echo "Usage: $CMDFILE <cluster name> [host1] [host2] [...]" echo "The cluster name must be the name of a subdirectory of cluster/" exit 1 fi |