aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cluster/staging/app/garage/deploy/garage-nix-exec2.hcl220
-rw-r--r--cluster/staging/cluster.nix49
-rwxr-xr-xdeploy_nixos6
-rw-r--r--nix/deuxfleurs.nix1
-rw-r--r--nix/nomad-driver-exec2.nix31
-rwxr-xr-xupgrade_nixos7
6 files changed, 305 insertions, 9 deletions
diff --git a/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
new file mode 100644
index 0000000..ad36b4e
--- /dev/null
+++ b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
@@ -0,0 +1,220 @@
+job "garage-staging" {
+ type = "system"
+ #datacenters = [ "neptune", "pluton" ]
+ datacenters = [ "neptune" ]
+
+ priority = 80
+
+ constraint {
+ attribute = "${attr.cpu.arch}"
+ value = "amd64"
+ }
+
+ group "garage-staging" {
+ network {
+ port "s3" { static = 3990 }
+ port "rpc" { static = 3991 }
+ port "web" { static = 3992 }
+ port "k2v" { static = 3993 }
+ port "admin" { static = 3909 }
+ }
+
+ update {
+ max_parallel = 1
+ min_healthy_time = "30s"
+ healthy_deadline = "5m"
+ }
+
+ # task "repair" {
+ # lifecycle {
+ # hook = "prestart"
+ # sidecar = false
+ # }
+
+ # driver = "docker"
+
+ # config {
+ # image = "dxflrs/amd64_garage:v0.7.99-k2v"
+ # command = "/garage"
+ # args = [ "offline-repair", "--yes", "object_counters" ]
+ # network_mode = "host"
+ # volumes = [
+ # "/mnt/storage/garage-staging/data:/data",
+ # "/mnt/ssd/garage-staging/meta:/meta",
+ # "secrets/garage.toml:/etc/garage.toml",
+ # ]
+ # }
+
+ # template {
+ # data = file("../config/garage.toml")
+ # destination = "secrets/garage.toml"
+ # }
+
+ # resources {
+ # memory = 2000
+ # cpu = 1000
+ # }
+ # }
+
+ task "server" {
+ driver = "exec2"
+
+ config {
+ #command = "/usr/bin/env"
+ command = "/run/current-system/sw/bin/nix"
+ args = [
+ "run",
+ "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c",
+ "--",
+ "server"
+ ]
+ bind = {
+ "/mnt/storage/garage-staging/data" = "/data",
+ "/mnt/ssd/garage-staging/meta" = "/meta",
+ }
+ }
+
+ template {
+ data = file("../config/garage.toml")
+ destination = "etc/garage.toml"
+ }
+
+ template {
+ data = "{{ key \"secrets/consul/consul-ca.crt\" }}"
+ destination = "etc/garage/consul-ca.crt"
+ }
+
+ template {
+ data = "{{ key \"secrets/consul/consul-client.crt\" }}"
+ destination = "etc/garage/consul-client.crt"
+ }
+
+ template {
+ data = "{{ key \"secrets/consul/consul-client.key\" }}"
+ destination = "etc/garage/consul-client.key"
+ }
+
+ template {
+ data = <<EOH
+RUST_LOG=garage=debug
+EOH
+ destination = "secrets/env"
+ env = true
+ }
+
+ resources {
+ memory = 2000
+ memory_max = 3000
+ cpu = 500
+ }
+
+ kill_signal = "SIGINT"
+ kill_timeout = "20s"
+
+ service {
+ name = "garage-staging-s3-api"
+ tags = [
+ "garage-staging-api",
+ "tricot garage-staging.home.adnab.me",
+ "tricot garage.staging.deuxfleurs.org",
+ "tricot-add-header Access-Control-Allow-Origin *",
+ ]
+ port = "s3"
+ #address_mode = "host"
+ #check {
+ # type = "tcp"
+ # interval = "60s"
+ # timeout = "5s"
+ # check_restart {
+ # limit = 3
+ # grace = "90s"
+ # ignore_warnings = false
+ # }
+ #}
+ }
+
+ service {
+ name = "garage-staging-k2v-api"
+ tags = [
+ "garage-staging-k2v-api",
+ "tricot k2v.staging.deuxfleurs.org",
+ "tricot-add-header Access-Control-Allow-Origin *",
+ ]
+ port = "k2v"
+ #address_mode = "driver"
+ # check {
+ # type = "tcp"
+ # port = 3993
+ # address_mode = "driver"
+ # interval = "60s"
+ # timeout = "5s"
+ # check_restart {
+ # limit = 3
+ # grace = "90s"
+ # ignore_warnings = false
+ # }
+ # }
+ }
+
+ service {
+ name = "garage-staging-rpc"
+ tags = ["garage-staging-rpc"]
+ port = "rpc"
+ #address_mode = "driver"
+ #check {
+ # type = "tcp"
+ # port = 3991
+ # address_mode = "driver"
+ # interval = "60s"
+ # timeout = "5s"
+ # check_restart {
+ # limit = 3
+ # grace = "90s"
+ # ignore_warnings = false
+ # }
+ #}
+ }
+
+ service {
+ name = "garage-staging-web"
+ tags = [
+ "garage-staging-web",
+ "tricot *.web.staging.deuxfleurs.org",
+ "tricot staging.deuxfleurs.org",
+ "tricot matrix.home.adnab.me/.well-known/matrix/server",
+ "tricot-add-header Access-Control-Allow-Origin *",
+ ]
+ port = "web"
+ #address_mode = "driver"
+ #check {
+ # type = "tcp"
+ # port = 3992
+ # address_mode = "driver"
+ # interval = "60s"
+ # timeout = "5s"
+ # check_restart {
+ # limit = 3
+ # grace = "90s"
+ # ignore_warnings = false
+ # }
+ #}
+ }
+
+ service {
+ name = "garage-staging-admin"
+ tags = [
+ "garage-staging-admin",
+ ]
+ port = "admin"
+ #address_mode = "driver"
+ }
+
+ restart {
+ interval = "1m"
+ attempts = 10
+ delay = "15s"
+ mode = "delay"
+ }
+ }
+ }
+}
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix
index cbabdc9..79e3cf0 100644
--- a/cluster/staging/cluster.nix
+++ b/cluster/staging/cluster.nix
@@ -74,18 +74,53 @@
## -----
- ## EXPERIMENTAL ON STAGING: NIX NOMAD DRIVER
+ ## EXPERIMENTAL ON STAGING: NIX NOMAD JOBS
services.nomad.dropPrivileges = false;
+
+ # ----- nomad-driver-nix & nomad-driver-exec2 -----
+ services.nomad.extraSettingsPlugins =
+ let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
+ nomad_driver_exec2 = import ./nomad-driver-exec2.nix { inherit pkgs; };
+ in [
+ (pkgs.symlinkJoin {
+ name = "nomad-drivers";
+ paths = [
+ nomad_driver_nix
+ nomad_driver_exec2
+ ];
+ })
+ ];
+ # the nix driver requires flakes to be enabled and some commands to be available
+ nix.settings.experimental-features = [ "nix-command" "flakes" ];
services.nomad.extraPackages = [
pkgs.nix
pkgs.git
];
- services.nomad.extraSettingsPlugins =
- let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
- in [ nomad_driver_nix ];
-
- # the nix driver requires flakes to be enabled
- nix.settings.experimental-features = [ "nix-command" "flakes" ];
+ # default config for the exec2 driver
+ services.nomad.settings.plugin = [
+ {
+ "exec2-driver" = [
+ {
+ config = [
+ {
+ bind_read_only = {
+ "/etc/static" = "/etc/static";
+ "/etc/passwd" = "/etc/passwd";
+ "/etc/resolv.conf" = "/etc/resolv.conf";
+ "/etc/nix/nix.conf" = "/etc/nix/nix.conf";
+ "/etc/ssl" = "/etc/ssl";
+ "/etc/nsswitch.conf" = "/etc/nsswitch.conf";
+ "/nix" = "/nix";
+ "/bin" = "/bin";
+ "/usr" = "/usr";
+ "/run/current-system/sw" = "/run/current-system/sw";
+ };
+ }
+ ];
+ }
+ ];
+ }
+ ];
# use our cache as additionnal substituer (we put precompiled packages there,
# like we used to do on the docker hub)
diff --git a/deploy_nixos b/deploy_nixos
index 6a9f4a6..583c391 100755
--- a/deploy_nixos
+++ b/deploy_nixos
@@ -3,11 +3,15 @@
copy nix/configuration.nix /etc/nixos/configuration.nix
copy nix/deuxfleurs.nix /etc/nixos/deuxfleurs.nix
copy nix/remote-unlock.nix /etc/nixos/remote-unlock.nix
-copy nix/nomad-driver-nix.nix /etc/nixos/nomad-driver-nix.nix
copy cluster/$CLUSTER/cluster.nix /etc/nixos/cluster.nix
copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix
copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix
+if [ "$CLUSTER" = "staging" ]; then
+ copy nix/nomad-driver-nix.nix /etc/nixos/nomad-driver-nix.nix
+ copy nix/nomad-driver-exec2.nix /etc/nixos/nomad-driver-exec2.nix
+fi
+
# use ./upgrade_nixos instead to upgrade NixOS
#cmd "nix-channel --add https://nixos.org/channels/nixos-22.05 nixos"
#cmd nixos-rebuild switch --upgrade --show-trace
diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix
index edf97fb..a83dda4 100644
--- a/nix/deuxfleurs.nix
+++ b/nix/deuxfleurs.nix
@@ -171,6 +171,7 @@ in
# and to pass directly to public DNS resolver all others
services.unbound = {
enable = true;
+ enableRootTrustAnchor = false; # disable DNSSEC as it causes issues
settings = {
server = {
interface = [ "127.0.0.1" "${cfg.lan_ip}" "172.17.0.1" ];
diff --git a/nix/nomad-driver-exec2.nix b/nix/nomad-driver-exec2.nix
new file mode 100644
index 0000000..16aa8c0
--- /dev/null
+++ b/nix/nomad-driver-exec2.nix
@@ -0,0 +1,31 @@
+{
+ pkgs ? import <nixpkgs> {}
+}:
+pkgs.buildGoModule rec {
+ pname = "nomad-driver-exec2";
+ version = "0.1.0";
+
+ src = pkgs.fetchFromGitHub {
+ owner = "Alexis211";
+ repo = "nomad-driver-exec2";
+ rev = "14bd8f2010c3a1f90c6d1297f47ebbf46ee87dde";
+ sha256 = "sha256-k75VEi2+9mS9v3bx/ygumy0niQP7bLWIgB0Zwhuky+k=";
+ };
+
+ vendorSha256 = "sha256-EQdTxVOBRYQjg7kAB+pFJYcNwk0zlsjjJxM+EO/cK84=";
+
+ ldflags = [
+ "-X github.com/Alexis211/nomad-driver-exec2/exec2.pluginVersion=${version}"
+ ];
+
+ postInstall = ''
+ mv $out/bin/nomad-driver-exec2 $out/bin/exec2-driver
+ '';
+
+ meta = with pkgs.lib; {
+ description = "Nomad exec driver with support for bind mouns";
+ homepage = "https://github.com/Alexis211/nomad-driver-exec2";
+ license = licenses.mpl20;
+ platforms = platforms.linux;
+ };
+}
diff --git a/upgrade_nixos b/upgrade_nixos
index 1297a2b..df3d259 100755
--- a/upgrade_nixos
+++ b/upgrade_nixos
@@ -1,6 +1,11 @@
#!/usr/bin/env ./sshtool
-cmd nix-channel --add https://nixos.org/channels/nixos-22.05 nixos
+if [ "$CLUSTER" = "staging" ]; then
+ cmd nix-channel --add https://nixos.org/channels/nixos-22.11 nixos
+else
+ cmd nix-channel --add https://nixos.org/channels/nixos-22.05 nixos
+fi
+
cmd nix-channel --update
cmd nixos-rebuild boot
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-20 02:32:50 +0000