From c9f9ed4c711b8db80ce03f9aff3f9cc25c568b3a Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 29 Nov 2022 14:21:12 +0100
Subject: Deploy garage on staging using nix2 driver

---
 .../staging/app/garage/deploy/garage-nix-exec2.hcl | 220 ---------------------
 cluster/staging/app/garage/deploy/garage-nix2.hcl  | 219 ++++++++++++++++++++
 cluster/staging/cluster.nix                        |  21 +-
 3 files changed, 224 insertions(+), 236 deletions(-)
 delete mode 100644 cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
 create mode 100644 cluster/staging/app/garage/deploy/garage-nix2.hcl

(limited to 'cluster')

diff --git a/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl b/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
deleted file mode 100644
index ad36b4e..0000000
--- a/cluster/staging/app/garage/deploy/garage-nix-exec2.hcl
+++ /dev/null
@@ -1,220 +0,0 @@
-job "garage-staging" {
-  type = "system"
-  #datacenters = [ "neptune", "pluton" ]
-  datacenters = [ "neptune" ]
-
-  priority = 80
-
-  constraint {
-    attribute = "${attr.cpu.arch}"
-    value     = "amd64"
-  }
-
-  group "garage-staging" {
-    network {
-      port "s3" { static = 3990 }
-      port "rpc" { static = 3991 }
-      port "web" { static = 3992 }
-      port "k2v" { static = 3993 }
-      port "admin" { static = 3909 }
-    }
-
-    update {
-      max_parallel = 1
-      min_healthy_time = "30s"
-      healthy_deadline = "5m"
-    }
-
-    # task "repair" {
-    #   lifecycle {
-    #     hook = "prestart"
-    #     sidecar = false
-    #   }
-
-    #   driver = "docker"
-
-    #   config {
-    #     image = "dxflrs/amd64_garage:v0.7.99-k2v"
-    #     command = "/garage"
-    #     args = [ "offline-repair", "--yes", "object_counters" ]
-    #     network_mode = "host"
-    #     volumes = [
-    #       "/mnt/storage/garage-staging/data:/data",
-    #       "/mnt/ssd/garage-staging/meta:/meta",
-    #       "secrets/garage.toml:/etc/garage.toml",
-    #     ]
-    #   }
-
-    #   template {
-    #     data = file("../config/garage.toml")
-    #     destination = "secrets/garage.toml"
-    #   }
-
-    #   resources {
-    #     memory = 2000
-    #     cpu = 1000
-    #   }
-    # }
-
-    task "server" {
-      driver = "exec2"
-
-      config {
-        #command = "/usr/bin/env"
-        command = "/run/current-system/sw/bin/nix"
-        args = [
-          "run",
-          "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c",
-          "--",
-          "server"
-        ]
-        bind = {
-          "/mnt/storage/garage-staging/data" = "/data",
-          "/mnt/ssd/garage-staging/meta" = "/meta",
-        }
-      }
-
-      template {
-        data = file("../config/garage.toml")
-        destination = "etc/garage.toml"
-      }
-
-      template {
-        data = "{{ key \"secrets/consul/consul-ca.crt\" }}"
-        destination = "etc/garage/consul-ca.crt"
-      }
-
-      template {
-        data = "{{ key \"secrets/consul/consul-client.crt\" }}"
-        destination = "etc/garage/consul-client.crt"
-      }
-
-      template {
-        data = "{{ key \"secrets/consul/consul-client.key\" }}"
-        destination = "etc/garage/consul-client.key"
-      }
-
-      template {
-        data = <<EOH
-RUST_LOG=garage=debug
-EOH
-        destination = "secrets/env"
-        env = true
-      }
-
-      resources {
-        memory = 2000
-        memory_max = 3000
-        cpu = 500
-      }
-
-      kill_signal = "SIGINT"
-      kill_timeout = "20s"
-
-      service {
-        name = "garage-staging-s3-api"
-        tags = [
-          "garage-staging-api",
-          "tricot garage-staging.home.adnab.me",
-          "tricot garage.staging.deuxfleurs.org",
-          "tricot-add-header Access-Control-Allow-Origin *",
-        ]
-        port = "s3"
-        #address_mode = "host"
-        #check {
-        #  type = "tcp"
-        #  interval = "60s"
-        #  timeout = "5s"
-        #  check_restart {
-        #    limit = 3
-        #    grace = "90s"
-        #    ignore_warnings = false
-        #  }
-        #}
-      }
-
-      service {
-        name = "garage-staging-k2v-api"
-        tags = [
-          "garage-staging-k2v-api",
-          "tricot k2v.staging.deuxfleurs.org",
-          "tricot-add-header Access-Control-Allow-Origin *",
-        ]
-        port = "k2v"
-        #address_mode = "driver"
-        # check {
-        #   type = "tcp"
-        #   port = 3993
-        #   address_mode = "driver"
-        #   interval = "60s"
-        #   timeout = "5s"
-        #   check_restart {
-        #     limit = 3
-        #     grace = "90s"
-        #     ignore_warnings = false
-        #   }
-        # }
-      }
-
-      service {
-        name = "garage-staging-rpc"
-        tags = ["garage-staging-rpc"]
-        port = "rpc"
-        #address_mode = "driver"
-        #check {
-        #  type = "tcp"
-        #  port = 3991
-        #  address_mode = "driver"
-        #  interval = "60s"
-        #  timeout = "5s"
-        #  check_restart {
-        #    limit = 3
-        #    grace = "90s"
-        #    ignore_warnings = false
-        #  }
-        #}
-      }
-
-      service {
-        name = "garage-staging-web"
-        tags = [
-            "garage-staging-web",
-            "tricot *.web.staging.deuxfleurs.org",
-            "tricot staging.deuxfleurs.org",
-            "tricot matrix.home.adnab.me/.well-known/matrix/server",
-            "tricot-add-header Access-Control-Allow-Origin *",
-        ]
-        port = "web"
-        #address_mode = "driver"
-        #check {
-        #  type = "tcp"
-        #  port = 3992
-        #  address_mode = "driver"
-        #  interval = "60s"
-        #  timeout = "5s"
-        #  check_restart {
-        #    limit = 3
-        #    grace = "90s"
-        #    ignore_warnings = false
-        #  }
-        #}
-      }
-
-      service {
-        name = "garage-staging-admin"
-        tags = [
-          "garage-staging-admin",
-        ]
-        port = "admin"
-        #address_mode = "driver"
-      }
-
-      restart { 
-        interval = "1m"  
-        attempts = 10  
-        delay    = "15s"  
-        mode     = "delay"
-      }
-    }
-  }
-}
diff --git a/cluster/staging/app/garage/deploy/garage-nix2.hcl b/cluster/staging/app/garage/deploy/garage-nix2.hcl
new file mode 100644
index 0000000..a60b506
--- /dev/null
+++ b/cluster/staging/app/garage/deploy/garage-nix2.hcl
@@ -0,0 +1,219 @@
+job "garage-staging" {
+  type = "system"
+  #datacenters = [ "neptune", "pluton" ]
+  datacenters = [ "neptune" ]
+
+  priority = 80
+
+  constraint {
+    attribute = "${attr.cpu.arch}"
+    value     = "amd64"
+  }
+
+  group "garage-staging" {
+    network {
+      port "s3" { static = 3990 }
+      port "rpc" { static = 3991 }
+      port "web" { static = 3992 }
+      port "k2v" { static = 3993 }
+      port "admin" { static = 3909 }
+    }
+
+    update {
+      max_parallel = 1
+      min_healthy_time = "30s"
+      healthy_deadline = "5m"
+    }
+
+    # task "repair" {
+    #   lifecycle {
+    #     hook = "prestart"
+    #     sidecar = false
+    #   }
+
+    #   driver = "docker"
+
+    #   config {
+    #     image = "dxflrs/amd64_garage:v0.7.99-k2v"
+    #     command = "/garage"
+    #     args = [ "offline-repair", "--yes", "object_counters" ]
+    #     network_mode = "host"
+    #     volumes = [
+    #       "/mnt/storage/garage-staging/data:/data",
+    #       "/mnt/ssd/garage-staging/meta:/meta",
+    #       "secrets/garage.toml:/etc/garage.toml",
+    #     ]
+    #   }
+
+    #   template {
+    #     data = file("../config/garage.toml")
+    #     destination = "secrets/garage.toml"
+    #   }
+
+    #   resources {
+    #     memory = 2000
+    #     cpu = 1000
+    #   }
+    # }
+
+    task "server" {
+      driver = "nix2"
+
+      config {
+        packages = [
+          "bash", # so that we can enter a shell inside container
+          "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c",
+        ]
+        command = "garage"
+        args = [ "server" ]
+        bind = {
+          "/mnt/storage/garage-staging/data" = "/data",
+          "/mnt/ssd/garage-staging/meta" = "/meta",
+        }
+      }
+      user = "root"
+
+      template {
+        data = file("../config/garage.toml")
+        destination = "etc/garage.toml"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-ca.crt\" }}"
+        destination = "etc/garage/consul-ca.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-client.crt\" }}"
+        destination = "etc/garage/consul-client.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-client.key\" }}"
+        destination = "etc/garage/consul-client.key"
+      }
+
+      template {
+        data = <<EOH
+RUST_LOG=garage=debug
+EOH
+        destination = "secrets/env"
+        env = true
+      }
+
+      resources {
+        memory = 2000
+        memory_max = 3000
+        cpu = 500
+      }
+
+      kill_signal = "SIGINT"
+      kill_timeout = "20s"
+
+      service {
+        name = "garage-staging-s3-api"
+        tags = [
+          "garage-staging-api",
+          "tricot garage-staging.home.adnab.me",
+          "tricot garage.staging.deuxfleurs.org",
+          "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = "s3"
+        #address_mode = "host"
+        #check {
+        #  type = "tcp"
+        #  interval = "60s"
+        #  timeout = "5s"
+        #  check_restart {
+        #    limit = 3
+        #    grace = "90s"
+        #    ignore_warnings = false
+        #  }
+        #}
+      }
+
+      service {
+        name = "garage-staging-k2v-api"
+        tags = [
+          "garage-staging-k2v-api",
+          "tricot k2v.staging.deuxfleurs.org",
+          "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = "k2v"
+        #address_mode = "driver"
+        # check {
+        #   type = "tcp"
+        #   port = 3993
+        #   address_mode = "driver"
+        #   interval = "60s"
+        #   timeout = "5s"
+        #   check_restart {
+        #     limit = 3
+        #     grace = "90s"
+        #     ignore_warnings = false
+        #   }
+        # }
+      }
+
+      service {
+        name = "garage-staging-rpc"
+        tags = ["garage-staging-rpc"]
+        port = "rpc"
+        #address_mode = "driver"
+        #check {
+        #  type = "tcp"
+        #  port = 3991
+        #  address_mode = "driver"
+        #  interval = "60s"
+        #  timeout = "5s"
+        #  check_restart {
+        #    limit = 3
+        #    grace = "90s"
+        #    ignore_warnings = false
+        #  }
+        #}
+      }
+
+      service {
+        name = "garage-staging-web"
+        tags = [
+            "garage-staging-web",
+            "tricot *.web.staging.deuxfleurs.org",
+            "tricot staging.deuxfleurs.org",
+            "tricot matrix.home.adnab.me/.well-known/matrix/server",
+            "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = "web"
+        #address_mode = "driver"
+        #check {
+        #  type = "tcp"
+        #  port = 3992
+        #  address_mode = "driver"
+        #  interval = "60s"
+        #  timeout = "5s"
+        #  check_restart {
+        #    limit = 3
+        #    grace = "90s"
+        #    ignore_warnings = false
+        #  }
+        #}
+      }
+
+      service {
+        name = "garage-staging-admin"
+        tags = [
+          "garage-staging-admin",
+        ]
+        port = "admin"
+        #address_mode = "driver"
+      }
+
+      restart { 
+        interval = "1m"  
+        attempts = 10  
+        delay    = "15s"  
+        mode     = "delay"
+      }
+    }
+  }
+}
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix
index 79e3cf0..71a8d98 100644
--- a/cluster/staging/cluster.nix
+++ b/cluster/staging/cluster.nix
@@ -77,16 +77,16 @@
   ## EXPERIMENTAL ON STAGING: NIX NOMAD JOBS
   services.nomad.dropPrivileges = false;
 
-  # ----- nomad-driver-nix & nomad-driver-exec2 -----
+  # ----- nomad-driver-nix & nomad-driver-nix2 -----
   services.nomad.extraSettingsPlugins =
     let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
-        nomad_driver_exec2 = import ./nomad-driver-exec2.nix { inherit pkgs; };
+        nomad_driver_nix2 = import ./nomad-driver-nix2.nix { inherit pkgs; };
     in [
       (pkgs.symlinkJoin {
         name = "nomad-drivers";
         paths = [
           nomad_driver_nix
-          nomad_driver_exec2
+          nomad_driver_nix2
         ];
       })
     ];
@@ -99,22 +99,11 @@
   # default config for the exec2 driver
   services.nomad.settings.plugin = [
     {
-      "exec2-driver" = [
+      "nix2-driver" = [
         {
           config = [
             {
-              bind_read_only = {
-                "/etc/static" = "/etc/static";
-                "/etc/passwd" = "/etc/passwd";
-                "/etc/resolv.conf" = "/etc/resolv.conf";
-                "/etc/nix/nix.conf" = "/etc/nix/nix.conf";
-                "/etc/ssl" = "/etc/ssl";
-                "/etc/nsswitch.conf" = "/etc/nsswitch.conf";
-                "/nix" = "/nix";
-                "/bin" = "/bin";
-                "/usr" = "/usr";
-                "/run/current-system/sw" = "/run/current-system/sw";
-              };
+              # default_nixpkgs = "github:nixos/nixpkgs/nixos-22.11";
             }
           ];
         }
-- 
cgit v1.2.3