diff options
Diffstat (limited to 'cluster')
| -rw-r--r-- | cluster/staging/app/garage/deploy/garage-nix-exec2.hcl | 220 | ||||
| -rw-r--r-- | cluster/staging/cluster.nix | 49 |
2 files changed, 262 insertions, 7 deletions
diff --git a/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl new file mode 100644 index 0000000..ad36b4e --- /dev/null +++ b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl @@ -0,0 +1,220 @@ +job "garage-staging" { + type = "system" + #datacenters = [ "neptune", "pluton" ] + datacenters = [ "neptune" ] + + priority = 80 + + constraint { + attribute = "${attr.cpu.arch}" + value = "amd64" + } + + group "garage-staging" { + network { + port "s3" { static = 3990 } + port "rpc" { static = 3991 } + port "web" { static = 3992 } + port "k2v" { static = 3993 } + port "admin" { static = 3909 } + } + + update { + max_parallel = 1 + min_healthy_time = "30s" + healthy_deadline = "5m" + } + + # task "repair" { + # lifecycle { + # hook = "prestart" + # sidecar = false + # } + + # driver = "docker" + + # config { + # image = "dxflrs/amd64_garage:v0.7.99-k2v" + # command = "/garage" + # args = [ "offline-repair", "--yes", "object_counters" ] + # network_mode = "host" + # volumes = [ + # "/mnt/storage/garage-staging/data:/data", + # "/mnt/ssd/garage-staging/meta:/meta", + # "secrets/garage.toml:/etc/garage.toml", + # ] + # } + + # template { + # data = file("../config/garage.toml") + # destination = "secrets/garage.toml" + # } + + # resources { + # memory = 2000 + # cpu = 1000 + # } + # } + + task "server" { + driver = "exec2" + + config { + #command = "/usr/bin/env" + command = "/run/current-system/sw/bin/nix" + args = [ + "run", + "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c", + "--", + "server" + ] + bind = { + "/mnt/storage/garage-staging/data" = "/data", + "/mnt/ssd/garage-staging/meta" = "/meta", + } + } + + template { + data = file("../config/garage.toml") + destination = "etc/garage.toml" + } + + template { + data = "{{ key \"secrets/consul/consul-ca.crt\" }}" + destination = "etc/garage/consul-ca.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.crt\" }}" + destination = "etc/garage/consul-client.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.key\" }}" + destination = "etc/garage/consul-client.key" + } + + template { + data = <<EOH +RUST_LOG=garage=debug +EOH + destination = "secrets/env" + env = true + } + + resources { + memory = 2000 + memory_max = 3000 + cpu = 500 + } + + kill_signal = "SIGINT" + kill_timeout = "20s" + + service { + name = "garage-staging-s3-api" + tags = [ + "garage-staging-api", + "tricot garage-staging.home.adnab.me", + "tricot garage.staging.deuxfleurs.org", + "tricot-add-header Access-Control-Allow-Origin *", + ] + port = "s3" + #address_mode = "host" + #check { + # type = "tcp" + # interval = "60s" + # timeout = "5s" + # check_restart { + # limit = 3 + # grace = "90s" + # ignore_warnings = false + # } + #} + } + + service { + name = "garage-staging-k2v-api" + tags = [ + "garage-staging-k2v-api", + "tricot k2v.staging.deuxfleurs.org", + "tricot-add-header Access-Control-Allow-Origin *", + ] + port = "k2v" + #address_mode = "driver" + # check { + # type = "tcp" + # port = 3993 + # address_mode = "driver" + # interval = "60s" + # timeout = "5s" + # check_restart { + # limit = 3 + # grace = "90s" + # ignore_warnings = false + # } + # } + } + + service { + name = "garage-staging-rpc" + tags = ["garage-staging-rpc"] + port = "rpc" + #address_mode = "driver" + #check { + # type = "tcp" + # port = 3991 + # address_mode = "driver" + # interval = "60s" + # timeout = "5s" + # check_restart { + # limit = 3 + # grace = "90s" + # ignore_warnings = false + # } + #} + } + + service { + name = "garage-staging-web" + tags = [ + "garage-staging-web", + "tricot *.web.staging.deuxfleurs.org", + "tricot staging.deuxfleurs.org", + "tricot matrix.home.adnab.me/.well-known/matrix/server", + "tricot-add-header Access-Control-Allow-Origin *", + ] + port = "web" + #address_mode = "driver" + #check { + # type = "tcp" + # port = 3992 + # address_mode = "driver" + # interval = "60s" + # timeout = "5s" + # check_restart { + # limit = 3 + # grace = "90s" + # ignore_warnings = false + # } + #} + } + + service { + name = "garage-staging-admin" + tags = [ + "garage-staging-admin", + ] + port = "admin" + #address_mode = "driver" + } + + restart { + interval = "1m" + attempts = 10 + delay = "15s" + mode = "delay" + } + } + } +} diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix index cbabdc9..79e3cf0 100644 --- a/cluster/staging/cluster.nix +++ b/cluster/staging/cluster.nix @@ -74,18 +74,53 @@ ## ----- - ## EXPERIMENTAL ON STAGING: NIX NOMAD DRIVER + ## EXPERIMENTAL ON STAGING: NIX NOMAD JOBS services.nomad.dropPrivileges = false; + + # ----- nomad-driver-nix & nomad-driver-exec2 ----- + services.nomad.extraSettingsPlugins = + let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; }; + nomad_driver_exec2 = import ./nomad-driver-exec2.nix { inherit pkgs; }; + in [ + (pkgs.symlinkJoin { + name = "nomad-drivers"; + paths = [ + nomad_driver_nix + nomad_driver_exec2 + ]; + }) + ]; + # the nix driver requires flakes to be enabled and some commands to be available + nix.settings.experimental-features = [ "nix-command" "flakes" ]; services.nomad.extraPackages = [ pkgs.nix pkgs.git ]; - services.nomad.extraSettingsPlugins = - let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; }; - in [ nomad_driver_nix ]; - - # the nix driver requires flakes to be enabled - nix.settings.experimental-features = [ "nix-command" "flakes" ]; + # default config for the exec2 driver + services.nomad.settings.plugin = [ + { + "exec2-driver" = [ + { + config = [ + { + bind_read_only = { + "/etc/static" = "/etc/static"; + "/etc/passwd" = "/etc/passwd"; + "/etc/resolv.conf" = "/etc/resolv.conf"; + "/etc/nix/nix.conf" = "/etc/nix/nix.conf"; + "/etc/ssl" = "/etc/ssl"; + "/etc/nsswitch.conf" = "/etc/nsswitch.conf"; + "/nix" = "/nix"; + "/bin" = "/bin"; + "/usr" = "/usr"; + "/run/current-system/sw" = "/run/current-system/sw"; + }; + } + ]; + } + ]; + } + ]; # use our cache as additionnal substituer (we put precompiled packages there, # like we used to do on the docker hub) |