aboutsummaryrefslogtreecommitdiff
path: root/cluster
diff options
context:
space:
mode:
authorAlex Auvolat <alex@adnab.me>2022-11-28 22:58:39 +0100
committerAlex Auvolat <alex@adnab.me>2022-11-28 22:58:39 +0100
commit105c08172827499504816106ffe1b61d76c2d4c8 (patch)
tree1a9f7472fa03c45089191e8e3167e3e9d5533297 /cluster
parenta327876e253b464c049faeafeb24ab2687425014 (diff)
downloadnixcfg-105c08172827499504816106ffe1b61d76c2d4c8.tar.gz
nixcfg-105c08172827499504816106ffe1b61d76c2d4c8.zip
Staging: ability to run Nix jobs using exec2 driver
Diffstat (limited to 'cluster')
-rw-r--r--cluster/staging/app/garage/deploy/garage-nix-exec2.hcl220
-rw-r--r--cluster/staging/cluster.nix49
2 files changed, 262 insertions, 7 deletions
diff --git a/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
new file mode 100644
index 0000000..ad36b4e
--- /dev/null
+++ b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
@@ -0,0 +1,220 @@
+job "garage-staging" {
+ type = "system"
+ #datacenters = [ "neptune", "pluton" ]
+ datacenters = [ "neptune" ]
+
+ priority = 80
+
+ constraint {
+ attribute = "${attr.cpu.arch}"
+ value = "amd64"
+ }
+
+ group "garage-staging" {
+ network {
+ port "s3" { static = 3990 }
+ port "rpc" { static = 3991 }
+ port "web" { static = 3992 }
+ port "k2v" { static = 3993 }
+ port "admin" { static = 3909 }
+ }
+
+ update {
+ max_parallel = 1
+ min_healthy_time = "30s"
+ healthy_deadline = "5m"
+ }
+
+ # task "repair" {
+ # lifecycle {
+ # hook = "prestart"
+ # sidecar = false
+ # }
+
+ # driver = "docker"
+
+ # config {
+ # image = "dxflrs/amd64_garage:v0.7.99-k2v"
+ # command = "/garage"
+ # args = [ "offline-repair", "--yes", "object_counters" ]
+ # network_mode = "host"
+ # volumes = [
+ # "/mnt/storage/garage-staging/data:/data",
+ # "/mnt/ssd/garage-staging/meta:/meta",
+ # "secrets/garage.toml:/etc/garage.toml",
+ # ]
+ # }
+
+ # template {
+ # data = file("../config/garage.toml")
+ # destination = "secrets/garage.toml"
+ # }
+
+ # resources {
+ # memory = 2000
+ # cpu = 1000
+ # }
+ # }
+
+ task "server" {
+ driver = "exec2"
+
+ config {
+ #command = "/usr/bin/env"
+ command = "/run/current-system/sw/bin/nix"
+ args = [
+ "run",
+ "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c",
+ "--",
+ "server"
+ ]
+ bind = {
+ "/mnt/storage/garage-staging/data" = "/data",
+ "/mnt/ssd/garage-staging/meta" = "/meta",
+ }
+ }
+
+ template {
+ data = file("../config/garage.toml")
+ destination = "etc/garage.toml"
+ }
+
+ template {
+ data = "{{ key \"secrets/consul/consul-ca.crt\" }}"
+ destination = "etc/garage/consul-ca.crt"
+ }
+
+ template {
+ data = "{{ key \"secrets/consul/consul-client.crt\" }}"
+ destination = "etc/garage/consul-client.crt"
+ }
+
+ template {
+ data = "{{ key \"secrets/consul/consul-client.key\" }}"
+ destination = "etc/garage/consul-client.key"
+ }
+
+ template {
+ data = <<EOH
+RUST_LOG=garage=debug
+EOH
+ destination = "secrets/env"
+ env = true
+ }
+
+ resources {
+ memory = 2000
+ memory_max = 3000
+ cpu = 500
+ }
+
+ kill_signal = "SIGINT"
+ kill_timeout = "20s"
+
+ service {
+ name = "garage-staging-s3-api"
+ tags = [
+ "garage-staging-api",
+ "tricot garage-staging.home.adnab.me",
+ "tricot garage.staging.deuxfleurs.org",
+ "tricot-add-header Access-Control-Allow-Origin *",
+ ]
+ port = "s3"
+ #address_mode = "host"
+ #check {
+ # type = "tcp"
+ # interval = "60s"
+ # timeout = "5s"
+ # check_restart {
+ # limit = 3
+ # grace = "90s"
+ # ignore_warnings = false
+ # }
+ #}
+ }
+
+ service {
+ name = "garage-staging-k2v-api"
+ tags = [
+ "garage-staging-k2v-api",
+ "tricot k2v.staging.deuxfleurs.org",
+ "tricot-add-header Access-Control-Allow-Origin *",
+ ]
+ port = "k2v"
+ #address_mode = "driver"
+ # check {
+ # type = "tcp"
+ # port = 3993
+ # address_mode = "driver"
+ # interval = "60s"
+ # timeout = "5s"
+ # check_restart {
+ # limit = 3
+ # grace = "90s"
+ # ignore_warnings = false
+ # }
+ # }
+ }
+
+ service {
+ name = "garage-staging-rpc"
+ tags = ["garage-staging-rpc"]
+ port = "rpc"
+ #address_mode = "driver"
+ #check {
+ # type = "tcp"
+ # port = 3991
+ # address_mode = "driver"
+ # interval = "60s"
+ # timeout = "5s"
+ # check_restart {
+ # limit = 3
+ # grace = "90s"
+ # ignore_warnings = false
+ # }
+ #}
+ }
+
+ service {
+ name = "garage-staging-web"
+ tags = [
+ "garage-staging-web",
+ "tricot *.web.staging.deuxfleurs.org",
+ "tricot staging.deuxfleurs.org",
+ "tricot matrix.home.adnab.me/.well-known/matrix/server",
+ "tricot-add-header Access-Control-Allow-Origin *",
+ ]
+ port = "web"
+ #address_mode = "driver"
+ #check {
+ # type = "tcp"
+ # port = 3992
+ # address_mode = "driver"
+ # interval = "60s"
+ # timeout = "5s"
+ # check_restart {
+ # limit = 3
+ # grace = "90s"
+ # ignore_warnings = false
+ # }
+ #}
+ }
+
+ service {
+ name = "garage-staging-admin"
+ tags = [
+ "garage-staging-admin",
+ ]
+ port = "admin"
+ #address_mode = "driver"
+ }
+
+ restart {
+ interval = "1m"
+ attempts = 10
+ delay = "15s"
+ mode = "delay"
+ }
+ }
+ }
+}
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix
index cbabdc9..79e3cf0 100644
--- a/cluster/staging/cluster.nix
+++ b/cluster/staging/cluster.nix
@@ -74,18 +74,53 @@
## -----
- ## EXPERIMENTAL ON STAGING: NIX NOMAD DRIVER
+ ## EXPERIMENTAL ON STAGING: NIX NOMAD JOBS
services.nomad.dropPrivileges = false;
+
+ # ----- nomad-driver-nix & nomad-driver-exec2 -----
+ services.nomad.extraSettingsPlugins =
+ let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
+ nomad_driver_exec2 = import ./nomad-driver-exec2.nix { inherit pkgs; };
+ in [
+ (pkgs.symlinkJoin {
+ name = "nomad-drivers";
+ paths = [
+ nomad_driver_nix
+ nomad_driver_exec2
+ ];
+ })
+ ];
+ # the nix driver requires flakes to be enabled and some commands to be available
+ nix.settings.experimental-features = [ "nix-command" "flakes" ];
services.nomad.extraPackages = [
pkgs.nix
pkgs.git
];
- services.nomad.extraSettingsPlugins =
- let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
- in [ nomad_driver_nix ];
-
- # the nix driver requires flakes to be enabled
- nix.settings.experimental-features = [ "nix-command" "flakes" ];
+ # default config for the exec2 driver
+ services.nomad.settings.plugin = [
+ {
+ "exec2-driver" = [
+ {
+ config = [
+ {
+ bind_read_only = {
+ "/etc/static" = "/etc/static";
+ "/etc/passwd" = "/etc/passwd";
+ "/etc/resolv.conf" = "/etc/resolv.conf";
+ "/etc/nix/nix.conf" = "/etc/nix/nix.conf";
+ "/etc/ssl" = "/etc/ssl";
+ "/etc/nsswitch.conf" = "/etc/nsswitch.conf";
+ "/nix" = "/nix";
+ "/bin" = "/bin";
+ "/usr" = "/usr";
+ "/run/current-system/sw" = "/run/current-system/sw";
+ };
+ }
+ ];
+ }
+ ];
+ }
+ ];
# use our cache as additionnal substituer (we put precompiled packages there,
# like we used to do on the docker hub)