diff options
author | Alex Auvolat <alex@adnab.me> | 2022-11-17 00:17:56 +0100 |
---|---|---|
committer | Alex Auvolat <alex@adnab.me> | 2022-11-17 00:17:56 +0100 |
commit | 945dd4fa9ae898562648bc33b4798adaaca08be4 (patch) | |
tree | addfe2b30272f8079dad71061448a8d222499542 | |
parent | 3c5f4b55e6e07414d204f682c449ad39fb4916c0 (diff) | |
download | nixcfg-945dd4fa9ae898562648bc33b4798adaaca08be4.tar.gz nixcfg-945dd4fa9ae898562648bc33b4798adaaca08be4.zip |
Run Garage as a Nomad Nix job on staging cluster
-rw-r--r-- | cluster/staging/app/garage/deploy/garage-nix.hcl | 218 | ||||
-rw-r--r-- | cluster/staging/cluster.nix | 20 | ||||
-rw-r--r-- | cluster/staging/node/cariacou.nix | 18 |
3 files changed, 238 insertions, 18 deletions
diff --git a/cluster/staging/app/garage/deploy/garage-nix.hcl b/cluster/staging/app/garage/deploy/garage-nix.hcl new file mode 100644 index 0000000..373fd10 --- /dev/null +++ b/cluster/staging/app/garage/deploy/garage-nix.hcl @@ -0,0 +1,218 @@ +job "garage-staging" { + type = "system" + #datacenters = [ "neptune", "pluton" ] + datacenters = [ "neptune" ] + + priority = 80 + + constraint { + attribute = "${attr.cpu.arch}" + value = "amd64" + } + + group "garage-staging" { + network { + port "s3" { static = 3990 } + port "rpc" { static = 3991 } + port "web" { static = 3992 } + port "k2v" { static = 3993 } + port "admin" { static = 3909 } + } + + update { + max_parallel = 1 + min_healthy_time = "30s" + healthy_deadline = "5m" + } + + # task "repair" { + # lifecycle { + # hook = "prestart" + # sidecar = false + # } + + # driver = "docker" + + # config { + # image = "dxflrs/amd64_garage:v0.7.99-k2v" + # command = "/garage" + # args = [ "offline-repair", "--yes", "object_counters" ] + # network_mode = "host" + # volumes = [ + # "/mnt/storage/garage-staging/data:/data", + # "/mnt/ssd/garage-staging/meta:/meta", + # "secrets/garage.toml:/etc/garage.toml", + # ] + # } + + # template { + # data = file("../config/garage.toml") + # destination = "secrets/garage.toml" + # } + + # resources { + # memory = 2000 + # cpu = 1000 + # } + # } + + task "server" { + driver = "nix" + + config { + packages = [ + "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c" + ] + command = [ "/bin/garage", "server" ] + bind = { + "/mnt/storage/garage-staging/data" = "/data", + "/mnt/ssd/garage-staging/meta" = "/meta", + } + } + + template { + data = file("../config/garage.toml") + destination = "etc/garage.toml" + } + + template { + data = "{{ key \"secrets/consul/consul-ca.crt\" }}" + destination = "etc/garage/consul-ca.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.crt\" }}" + destination = "etc/garage/consul-client.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.key\" }}" + destination = "etc/garage/consul-client.key" + } + + template { + data = <<EOH +RUST_LOG=garage=debug +EOH + destination = "secrets/env" + env = true + } + + resources { + memory = 2000 + memory_max = 3000 + cpu = 500 + } + + kill_signal = "SIGINT" + kill_timeout = "20s" + + service { + tags = [ + "garage-staging-api", + "tricot garage-staging.home.adnab.me", + "tricot garage.staging.deuxfleurs.org", + "tricot-add-header Access-Control-Allow-Origin *", + ] + port = 3990 + address_mode = "driver" + name = "garage-staging-api" + check { + type = "tcp" + port = 3990 + address_mode = "driver" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + } + + service { + tags = [ + "garage-staging-k2v-api", + "tricot k2v.staging.deuxfleurs.org", + "tricot-add-header Access-Control-Allow-Origin *", + ] + port = 3993 + address_mode = "driver" + name = "garage-staging-k2v-api" + # check { + # type = "tcp" + # port = 3993 + # address_mode = "driver" + # interval = "60s" + # timeout = "5s" + # check_restart { + # limit = 3 + # grace = "90s" + # ignore_warnings = false + # } + # } + } + + service { + tags = ["garage-staging-rpc"] + port = 3991 + address_mode = "driver" + name = "garage-staging-rpc" + check { + type = "tcp" + port = 3991 + address_mode = "driver" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + } + + service { + tags = [ + "garage-staging-web", + "tricot *.web.staging.deuxfleurs.org", + "tricot staging.deuxfleurs.org", + "tricot matrix.home.adnab.me/.well-known/matrix/server", + "tricot-add-header Access-Control-Allow-Origin *", + ] + port = 3992 + address_mode = "driver" + name = "garage-staging-web" + check { + type = "tcp" + port = 3992 + address_mode = "driver" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + } + + service { + tags = [ + "garage-staging-admin", + ] + port = 3909 + address_mode = "driver" + name = "garage-staging-admin" + } + + restart { + interval = "30m" + attempts = 10 + delay = "15s" + mode = "delay" + } + } + } +} diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix index 6e0a589..f5cfdc4 100644 --- a/cluster/staging/cluster.nix +++ b/cluster/staging/cluster.nix @@ -68,4 +68,24 @@ # For Garage ipv6 communication networking.firewall.allowedTCPPorts = [ 3991 ]; + + ## ----- + + ## EXPERIMENTAL ON STAGING: NIX NOMAD DRIVER + services.nomad.dropPrivileges = false; + services.nomad.extraPackages = [ + pkgs.nix + pkgs.git + ]; + services.nomad.extraSettingsPlugins = + let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; }; + in [ nomad_driver_nix ]; + + # the nix driver requires flakes to be enabled + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + + # use our cache as additionnal substituer (we put precompiled packages there, + # like we used to do on the docker hub) + nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ]; + nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ]; } diff --git a/cluster/staging/node/cariacou.nix b/cluster/staging/node/cariacou.nix index d653979..61d3f28 100644 --- a/cluster/staging/node/cariacou.nix +++ b/cluster/staging/node/cariacou.nix @@ -21,22 +21,4 @@ deuxfleurs.cluster_ip = "10.14.1.1"; deuxfleurs.is_raft_server = true; - - # ---- - - # experimental: config for nomad nix driver - services.nomad.dropPrivileges = false; - services.nomad.extraPackages = [ - pkgs.nix - pkgs.git - ]; - services.nomad.extraSettingsPlugins = - let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; }; - in [ nomad_driver_nix ]; - # the nix driver requires flakes to be enabled - nix.settings.experimental-features = [ "nix-command" "flakes" ]; - - # use our cache as additionnal substituer - nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ]; - nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ]; } |