From 105c08172827499504816106ffe1b61d76c2d4c8 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Mon, 28 Nov 2022 22:58:39 +0100
Subject: Staging: ability to run Nix jobs using exec2 driver

---
 .../staging/app/garage/deploy/garage-nix-exec2.hcl | 220 +++++++++++++++++++++
 cluster/staging/cluster.nix                        |  49 ++++-
 deploy_nixos                                       |   6 +-
 nix/deuxfleurs.nix                                 |   1 +
 nix/nomad-driver-exec2.nix                         |  31 +++
 upgrade_nixos                                      |   7 +-
 6 files changed, 305 insertions(+), 9 deletions(-)
 create mode 100644 cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
 create mode 100644 nix/nomad-driver-exec2.nix

diff --git a/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
new file mode 100644
index 0000000..ad36b4e
--- /dev/null
+++ b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
@@ -0,0 +1,220 @@
+job "garage-staging" {
+  type = "system"
+  #datacenters = [ "neptune", "pluton" ]
+  datacenters = [ "neptune" ]
+
+  priority = 80
+
+  constraint {
+    attribute = "${attr.cpu.arch}"
+    value     = "amd64"
+  }
+
+  group "garage-staging" {
+    network {
+      port "s3" { static = 3990 }
+      port "rpc" { static = 3991 }
+      port "web" { static = 3992 }
+      port "k2v" { static = 3993 }
+      port "admin" { static = 3909 }
+    }
+
+    update {
+      max_parallel = 1
+      min_healthy_time = "30s"
+      healthy_deadline = "5m"
+    }
+
+    # task "repair" {
+    #   lifecycle {
+    #     hook = "prestart"
+    #     sidecar = false
+    #   }
+
+    #   driver = "docker"
+
+    #   config {
+    #     image = "dxflrs/amd64_garage:v0.7.99-k2v"
+    #     command = "/garage"
+    #     args = [ "offline-repair", "--yes", "object_counters" ]
+    #     network_mode = "host"
+    #     volumes = [
+    #       "/mnt/storage/garage-staging/data:/data",
+    #       "/mnt/ssd/garage-staging/meta:/meta",
+    #       "secrets/garage.toml:/etc/garage.toml",
+    #     ]
+    #   }
+
+    #   template {
+    #     data = file("../config/garage.toml")
+    #     destination = "secrets/garage.toml"
+    #   }
+
+    #   resources {
+    #     memory = 2000
+    #     cpu = 1000
+    #   }
+    # }
+
+    task "server" {
+      driver = "exec2"
+
+      config {
+        #command = "/usr/bin/env"
+        command = "/run/current-system/sw/bin/nix"
+        args = [
+          "run",
+          "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c",
+          "--",
+          "server"
+        ]
+        bind = {
+          "/mnt/storage/garage-staging/data" = "/data",
+          "/mnt/ssd/garage-staging/meta" = "/meta",
+        }
+      }
+
+      template {
+        data = file("../config/garage.toml")
+        destination = "etc/garage.toml"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-ca.crt\" }}"
+        destination = "etc/garage/consul-ca.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-client.crt\" }}"
+        destination = "etc/garage/consul-client.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-client.key\" }}"
+        destination = "etc/garage/consul-client.key"
+      }
+
+      template {
+        data = <<EOH
+RUST_LOG=garage=debug
+EOH
+        destination = "secrets/env"
+        env = true
+      }
+
+      resources {
+        memory = 2000
+        memory_max = 3000
+        cpu = 500
+      }
+
+      kill_signal = "SIGINT"
+      kill_timeout = "20s"
+
+      service {
+        name = "garage-staging-s3-api"
+        tags = [
+          "garage-staging-api",
+          "tricot garage-staging.home.adnab.me",
+          "tricot garage.staging.deuxfleurs.org",
+          "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = "s3"
+        #address_mode = "host"
+        #check {
+        #  type = "tcp"
+        #  interval = "60s"
+        #  timeout = "5s"
+        #  check_restart {
+        #    limit = 3
+        #    grace = "90s"
+        #    ignore_warnings = false
+        #  }
+        #}
+      }
+
+      service {
+        name = "garage-staging-k2v-api"
+        tags = [
+          "garage-staging-k2v-api",
+          "tricot k2v.staging.deuxfleurs.org",
+          "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = "k2v"
+        #address_mode = "driver"
+        # check {
+        #   type = "tcp"
+        #   port = 3993
+        #   address_mode = "driver"
+        #   interval = "60s"
+        #   timeout = "5s"
+        #   check_restart {
+        #     limit = 3
+        #     grace = "90s"
+        #     ignore_warnings = false
+        #   }
+        # }
+      }
+
+      service {
+        name = "garage-staging-rpc"
+        tags = ["garage-staging-rpc"]
+        port = "rpc"
+        #address_mode = "driver"
+        #check {
+        #  type = "tcp"
+        #  port = 3991
+        #  address_mode = "driver"
+        #  interval = "60s"
+        #  timeout = "5s"
+        #  check_restart {
+        #    limit = 3
+        #    grace = "90s"
+        #    ignore_warnings = false
+        #  }
+        #}
+      }
+
+      service {
+        name = "garage-staging-web"
+        tags = [
+            "garage-staging-web",
+            "tricot *.web.staging.deuxfleurs.org",
+            "tricot staging.deuxfleurs.org",
+            "tricot matrix.home.adnab.me/.well-known/matrix/server",
+            "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = "web"
+        #address_mode = "driver"
+        #check {
+        #  type = "tcp"
+        #  port = 3992
+        #  address_mode = "driver"
+        #  interval = "60s"
+        #  timeout = "5s"
+        #  check_restart {
+        #    limit = 3
+        #    grace = "90s"
+        #    ignore_warnings = false
+        #  }
+        #}
+      }
+
+      service {
+        name = "garage-staging-admin"
+        tags = [
+          "garage-staging-admin",
+        ]
+        port = "admin"
+        #address_mode = "driver"
+      }
+
+      restart { 
+        interval = "1m"  
+        attempts = 10  
+        delay    = "15s"  
+        mode     = "delay"
+      }
+    }
+  }
+}
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix
index cbabdc9..79e3cf0 100644
--- a/cluster/staging/cluster.nix
+++ b/cluster/staging/cluster.nix
@@ -74,18 +74,53 @@
 
   ## -----
 
-  ## EXPERIMENTAL ON STAGING: NIX NOMAD DRIVER
+  ## EXPERIMENTAL ON STAGING: NIX NOMAD JOBS
   services.nomad.dropPrivileges = false;
+
+  # ----- nomad-driver-nix & nomad-driver-exec2 -----
+  services.nomad.extraSettingsPlugins =
+    let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
+        nomad_driver_exec2 = import ./nomad-driver-exec2.nix { inherit pkgs; };
+    in [
+      (pkgs.symlinkJoin {
+        name = "nomad-drivers";
+        paths = [
+          nomad_driver_nix
+          nomad_driver_exec2
+        ];
+      })
+    ];
+  # the nix driver requires flakes to be enabled and some commands to be available
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
   services.nomad.extraPackages = [
     pkgs.nix
     pkgs.git
   ];
-  services.nomad.extraSettingsPlugins =
-    let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
-    in [ nomad_driver_nix ];
-
-  # the nix driver requires flakes to be enabled
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  # default config for the exec2 driver
+  services.nomad.settings.plugin = [
+    {
+      "exec2-driver" = [
+        {
+          config = [
+            {
+              bind_read_only = {
+                "/etc/static" = "/etc/static";
+                "/etc/passwd" = "/etc/passwd";
+                "/etc/resolv.conf" = "/etc/resolv.conf";
+                "/etc/nix/nix.conf" = "/etc/nix/nix.conf";
+                "/etc/ssl" = "/etc/ssl";
+                "/etc/nsswitch.conf" = "/etc/nsswitch.conf";
+                "/nix" = "/nix";
+                "/bin" = "/bin";
+                "/usr" = "/usr";
+                "/run/current-system/sw" = "/run/current-system/sw";
+              };
+            }
+          ];
+        }
+      ];
+    }
+  ];
 
   # use our cache as additionnal substituer (we put precompiled packages there,
   # like we used to do on the docker hub)
diff --git a/deploy_nixos b/deploy_nixos
index 6a9f4a6..583c391 100755
--- a/deploy_nixos
+++ b/deploy_nixos
@@ -3,11 +3,15 @@
 copy nix/configuration.nix /etc/nixos/configuration.nix
 copy nix/deuxfleurs.nix /etc/nixos/deuxfleurs.nix
 copy nix/remote-unlock.nix /etc/nixos/remote-unlock.nix
-copy nix/nomad-driver-nix.nix /etc/nixos/nomad-driver-nix.nix
 copy cluster/$CLUSTER/cluster.nix /etc/nixos/cluster.nix
 copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix
 copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix
 
+if [ "$CLUSTER" = "staging" ]; then
+	copy nix/nomad-driver-nix.nix /etc/nixos/nomad-driver-nix.nix
+	copy nix/nomad-driver-exec2.nix /etc/nixos/nomad-driver-exec2.nix
+fi
+
 # use ./upgrade_nixos instead to upgrade NixOS
 #cmd "nix-channel --add https://nixos.org/channels/nixos-22.05 nixos"
 #cmd nixos-rebuild switch --upgrade --show-trace
diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix
index edf97fb..a83dda4 100644
--- a/nix/deuxfleurs.nix
+++ b/nix/deuxfleurs.nix
@@ -171,6 +171,7 @@ in
     # and to pass directly to public DNS resolver all others
     services.unbound = {
       enable = true;
+      enableRootTrustAnchor = false;  # disable DNSSEC as it causes issues
       settings = {
         server = {
           interface = [ "127.0.0.1" "${cfg.lan_ip}" "172.17.0.1" ];
diff --git a/nix/nomad-driver-exec2.nix b/nix/nomad-driver-exec2.nix
new file mode 100644
index 0000000..16aa8c0
--- /dev/null
+++ b/nix/nomad-driver-exec2.nix
@@ -0,0 +1,31 @@
+{
+  pkgs ? import <nixpkgs> {}
+}:
+pkgs.buildGoModule rec {
+  pname = "nomad-driver-exec2";
+  version = "0.1.0";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "Alexis211";
+    repo = "nomad-driver-exec2";
+    rev = "14bd8f2010c3a1f90c6d1297f47ebbf46ee87dde";
+    sha256 = "sha256-k75VEi2+9mS9v3bx/ygumy0niQP7bLWIgB0Zwhuky+k=";
+  };
+
+  vendorSha256 = "sha256-EQdTxVOBRYQjg7kAB+pFJYcNwk0zlsjjJxM+EO/cK84=";
+
+  ldflags = [
+    "-X github.com/Alexis211/nomad-driver-exec2/exec2.pluginVersion=${version}"
+  ];
+
+  postInstall = ''
+    mv $out/bin/nomad-driver-exec2 $out/bin/exec2-driver
+  '';
+
+  meta = with pkgs.lib; {
+    description = "Nomad exec driver with support for bind mouns";
+    homepage = "https://github.com/Alexis211/nomad-driver-exec2";
+    license = licenses.mpl20;
+    platforms = platforms.linux;
+  };
+}
diff --git a/upgrade_nixos b/upgrade_nixos
index 1297a2b..df3d259 100755
--- a/upgrade_nixos
+++ b/upgrade_nixos
@@ -1,6 +1,11 @@
 #!/usr/bin/env ./sshtool
 
-cmd nix-channel --add https://nixos.org/channels/nixos-22.05 nixos
+if [ "$CLUSTER" = "staging" ]; then
+	cmd nix-channel --add https://nixos.org/channels/nixos-22.11 nixos
+else
+	cmd nix-channel --add https://nixos.org/channels/nixos-22.05 nixos
+fi
+
 cmd nix-channel --update
 cmd nixos-rebuild boot
 
-- 
cgit v1.2.3