From 105c08172827499504816106ffe1b61d76c2d4c8 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Mon, 28 Nov 2022 22:58:39 +0100
Subject: Staging: ability to run Nix jobs using exec2 driver

---
 .../staging/app/garage/deploy/garage-nix-exec2.hcl | 220 +++++++++++++++++++++
 cluster/staging/cluster.nix                        |  49 ++++-
 2 files changed, 262 insertions(+), 7 deletions(-)
 create mode 100644 cluster/staging/app/garage/deploy/garage-nix-exec2.hcl

(limited to 'cluster')

diff --git a/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
new file mode 100644
index 0000000..ad36b4e
--- /dev/null
+++ b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
@@ -0,0 +1,220 @@
+job "garage-staging" {
+  type = "system"
+  #datacenters = [ "neptune", "pluton" ]
+  datacenters = [ "neptune" ]
+
+  priority = 80
+
+  constraint {
+    attribute = "${attr.cpu.arch}"
+    value     = "amd64"
+  }
+
+  group "garage-staging" {
+    network {
+      port "s3" { static = 3990 }
+      port "rpc" { static = 3991 }
+      port "web" { static = 3992 }
+      port "k2v" { static = 3993 }
+      port "admin" { static = 3909 }
+    }
+
+    update {
+      max_parallel = 1
+      min_healthy_time = "30s"
+      healthy_deadline = "5m"
+    }
+
+    # task "repair" {
+    #   lifecycle {
+    #     hook = "prestart"
+    #     sidecar = false
+    #   }
+
+    #   driver = "docker"
+
+    #   config {
+    #     image = "dxflrs/amd64_garage:v0.7.99-k2v"
+    #     command = "/garage"
+    #     args = [ "offline-repair", "--yes", "object_counters" ]
+    #     network_mode = "host"
+    #     volumes = [
+    #       "/mnt/storage/garage-staging/data:/data",
+    #       "/mnt/ssd/garage-staging/meta:/meta",
+    #       "secrets/garage.toml:/etc/garage.toml",
+    #     ]
+    #   }
+
+    #   template {
+    #     data = file("../config/garage.toml")
+    #     destination = "secrets/garage.toml"
+    #   }
+
+    #   resources {
+    #     memory = 2000
+    #     cpu = 1000
+    #   }
+    # }
+
+    task "server" {
+      driver = "exec2"
+
+      config {
+        #command = "/usr/bin/env"
+        command = "/run/current-system/sw/bin/nix"
+        args = [
+          "run",
+          "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c",
+          "--",
+          "server"
+        ]
+        bind = {
+          "/mnt/storage/garage-staging/data" = "/data",
+          "/mnt/ssd/garage-staging/meta" = "/meta",
+        }
+      }
+
+      template {
+        data = file("../config/garage.toml")
+        destination = "etc/garage.toml"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-ca.crt\" }}"
+        destination = "etc/garage/consul-ca.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-client.crt\" }}"
+        destination = "etc/garage/consul-client.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-client.key\" }}"
+        destination = "etc/garage/consul-client.key"
+      }
+
+      template {
+        data = <<EOH
+RUST_LOG=garage=debug
+EOH
+        destination = "secrets/env"
+        env = true
+      }
+
+      resources {
+        memory = 2000
+        memory_max = 3000
+        cpu = 500
+      }
+
+      kill_signal = "SIGINT"
+      kill_timeout = "20s"
+
+      service {
+        name = "garage-staging-s3-api"
+        tags = [
+          "garage-staging-api",
+          "tricot garage-staging.home.adnab.me",
+          "tricot garage.staging.deuxfleurs.org",
+          "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = "s3"
+        #address_mode = "host"
+        #check {
+        #  type = "tcp"
+        #  interval = "60s"
+        #  timeout = "5s"
+        #  check_restart {
+        #    limit = 3
+        #    grace = "90s"
+        #    ignore_warnings = false
+        #  }
+        #}
+      }
+
+      service {
+        name = "garage-staging-k2v-api"
+        tags = [
+          "garage-staging-k2v-api",
+          "tricot k2v.staging.deuxfleurs.org",
+          "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = "k2v"
+        #address_mode = "driver"
+        # check {
+        #   type = "tcp"
+        #   port = 3993
+        #   address_mode = "driver"
+        #   interval = "60s"
+        #   timeout = "5s"
+        #   check_restart {
+        #     limit = 3
+        #     grace = "90s"
+        #     ignore_warnings = false
+        #   }
+        # }
+      }
+
+      service {
+        name = "garage-staging-rpc"
+        tags = ["garage-staging-rpc"]
+        port = "rpc"
+        #address_mode = "driver"
+        #check {
+        #  type = "tcp"
+        #  port = 3991
+        #  address_mode = "driver"
+        #  interval = "60s"
+        #  timeout = "5s"
+        #  check_restart {
+        #    limit = 3
+        #    grace = "90s"
+        #    ignore_warnings = false
+        #  }
+        #}
+      }
+
+      service {
+        name = "garage-staging-web"
+        tags = [
+            "garage-staging-web",
+            "tricot *.web.staging.deuxfleurs.org",
+            "tricot staging.deuxfleurs.org",
+            "tricot matrix.home.adnab.me/.well-known/matrix/server",
+            "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = "web"
+        #address_mode = "driver"
+        #check {
+        #  type = "tcp"
+        #  port = 3992
+        #  address_mode = "driver"
+        #  interval = "60s"
+        #  timeout = "5s"
+        #  check_restart {
+        #    limit = 3
+        #    grace = "90s"
+        #    ignore_warnings = false
+        #  }
+        #}
+      }
+
+      service {
+        name = "garage-staging-admin"
+        tags = [
+          "garage-staging-admin",
+        ]
+        port = "admin"
+        #address_mode = "driver"
+      }
+
+      restart { 
+        interval = "1m"  
+        attempts = 10  
+        delay    = "15s"  
+        mode     = "delay"
+      }
+    }
+  }
+}
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix
index cbabdc9..79e3cf0 100644
--- a/cluster/staging/cluster.nix
+++ b/cluster/staging/cluster.nix
@@ -74,18 +74,53 @@
 
   ## -----
 
-  ## EXPERIMENTAL ON STAGING: NIX NOMAD DRIVER
+  ## EXPERIMENTAL ON STAGING: NIX NOMAD JOBS
   services.nomad.dropPrivileges = false;
+
+  # ----- nomad-driver-nix & nomad-driver-exec2 -----
+  services.nomad.extraSettingsPlugins =
+    let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
+        nomad_driver_exec2 = import ./nomad-driver-exec2.nix { inherit pkgs; };
+    in [
+      (pkgs.symlinkJoin {
+        name = "nomad-drivers";
+        paths = [
+          nomad_driver_nix
+          nomad_driver_exec2
+        ];
+      })
+    ];
+  # the nix driver requires flakes to be enabled and some commands to be available
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
   services.nomad.extraPackages = [
     pkgs.nix
     pkgs.git
   ];
-  services.nomad.extraSettingsPlugins =
-    let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
-    in [ nomad_driver_nix ];
-
-  # the nix driver requires flakes to be enabled
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  # default config for the exec2 driver
+  services.nomad.settings.plugin = [
+    {
+      "exec2-driver" = [
+        {
+          config = [
+            {
+              bind_read_only = {
+                "/etc/static" = "/etc/static";
+                "/etc/passwd" = "/etc/passwd";
+                "/etc/resolv.conf" = "/etc/resolv.conf";
+                "/etc/nix/nix.conf" = "/etc/nix/nix.conf";
+                "/etc/ssl" = "/etc/ssl";
+                "/etc/nsswitch.conf" = "/etc/nsswitch.conf";
+                "/nix" = "/nix";
+                "/bin" = "/bin";
+                "/usr" = "/usr";
+                "/run/current-system/sw" = "/run/current-system/sw";
+              };
+            }
+          ];
+        }
+      ];
+    }
+  ];
 
   # use our cache as additionnal substituer (we put precompiled packages there,
   # like we used to do on the docker hub)
-- 
cgit v1.2.3