jepsen ssh into containers seem to work ?

7 files changed, 150 insertions, 19 deletions

diff --git a/script/jepsen.garage/destroy-tap.sh b/script/jepsen.garage/destroy-tap.sh
new file mode 100755
index 00000000..544b3053
--- /dev/null
+++ b/script/jepsen.garage/destroy-tap.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+USER=$(whoami)
+
+for NODE in 1 2 3 4 5; do
+	sudo ip link delete microvm-n$NODE
+done
+
+
diff --git a/script/jepsen.garage/flake.lock b/script/jepsen.garage/flake.lock
new file mode 100644
index 00000000..63dd88f8
--- /dev/null
+++ b/script/jepsen.garage/flake.lock
@@ -0,0 +1,64 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1678901627,
+        "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "microvm": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1681747916,
+        "narHash": "sha256-tpWJMHWbTrFD2Nmj3Y3qYXoaTP4LFT0P0wt5zW8/aI8=",
+        "owner": "astro",
+        "repo": "microvm.nix",
+        "rev": "68f1b9ece0f116d5ea1d1ecaf17f7b526303df81",
+        "type": "github"
+      },
+      "original": {
+        "owner": "astro",
+        "repo": "microvm.nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1681737997,
+        "narHash": "sha256-pHhjgsIkRMu80LmVe8QoKIZB6VZGRRxFmIvsC5S89k4=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "f00994e78cd39e6fc966f0c4103f908e63284780",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "microvm": "microvm",
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/script/jepsen.garage/flake.nix b/script/jepsen.garage/flake.nix
index 76ee8ebf..2afa9ea1 100644
--- a/script/jepsen.garage/flake.nix
+++ b/script/jepsen.garage/flake.nix
@@ -4,19 +4,56 @@
   inputs.microvm.url = "github:astro/microvm.nix";
   inputs.microvm.inputs.nixpkgs.follows = "nixpkgs";
 
-  outputs = { self, nixpkgs, microvm }: {
-    # Example nixosConfigurations entry
-    nixosConfigurations.my-microvm = nixpkgs.lib.nixosSystem {
+  outputs = { self, nixpkgs, microvm }:
+  with nixpkgs.lib;
+  let
+    addressMap =
+      {
+        "n1" = { ip = "10.1.0.10"; mac = "02:00:00:00:99:01"; };
+        "n2" = { ip = "10.2.0.10"; mac = "02:00:00:00:99:02"; };
+        "n3" = { ip = "10.3.0.10"; mac = "02:00:00:00:99:03"; };
+        "n4" = { ip = "10.4.0.10"; mac = "02:00:00:00:99:04"; };
+        "n5" = { ip = "10.5.0.10"; mac = "02:00:00:00:99:05"; };
+      };
+    toHostsEntry = name: { ip, ... }: "${ip} ${name}";
+    extraHosts =
+      builtins.concatStringsSep "\n"
+        (attrsets.mapAttrsToList toHostsEntry addressMap);
+
+    nodeConfig = hostName: { ip, mac }: nixosSystem {
       system = "x86_64-linux";
       modules = [
         # Include the microvm module
         microvm.nixosModules.microvm
         # Add more modules here
         {
-          networking.hostName = "my-microvm";
+          networking = {
+            inherit hostName extraHosts;
+          };
+
           microvm.hypervisor = "cloud-hypervisor";
+          microvm.interfaces = [ {
+            inherit mac;
+            type = "tap";
+            id = "microvm-${hostName}";
+          } ];
+
+          services.openssh = {
+            enable = true;
+            permitRootLogin = "yes";
+          };
+          users.users.root.initialPassword = "root";
+
+          #services.garage = {
+          #  enable = true;
+          #  logLevel = "debug";
+          #  settings.replication_mode = "3";
+          #};
         }
       ];
     };
+  in
+  {
+    nixosConfigurations = mapAttrs nodeConfig addressMap;
   };
 }
diff --git a/script/jepsen.garage/garage-cluster.nix b/script/jepsen.garage/garage-cluster.nix
index 32fedc04..ebc73aaf 100644
--- a/script/jepsen.garage/garage-cluster.nix
+++ b/script/jepsen.garage/garage-cluster.nix
@@ -29,30 +29,36 @@ let
         services.openssh = {
           enable = true;
           permitRootLogin = "yes";
+          kexAlgorithms = [ "curve25519-sha256@libssh.org" "ecdh-sha2-nistp256" "ecdh-sha2-nistp384" "ecdh-sha2-nistp521" "diffie-hellman-group-exchange-sha256" "diffie-hellman-group14-sha1" "diffie-hellman-group-exchange-sha1" "diffie-hellman-group1-sha1" ];
         };
         users.users.root.initialPassword = "root";
+        users.users.root.openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9"
+        ];
 
         system.stateVersion = "22.11";
 
         services.garage = {
           enable = true;
           logLevel = "debug";
-          settings.replication_mode = "3";
+          settings = {
+            replication_mode = "3";
+            db_engine = "lmdb";
+            rpc_secret = "b597bb28ebdc90cdc4f15712733ca678cfb9a7e0311e0b9e93db9610fc3685e6";
+            rpc_bind_addr = "0.0.0.0:3901";
+            s3_api = {
+              region = "garage";
+              api_bind_addr = "0.0.0.0:3900";
+            };
+            k2v_api.api_bind_addr = "0.0.0.0:3902";
+            admin = {
+              api_bind_addr = "0.0.0.0:3903";
+              admin_token = "icanhazadmin";
+            };
+          };
         };
 
-        # Workaround for nixos-container issue
-        # (see https://github.com/NixOS/nixpkgs/issues/67265 and
-        # https://github.com/NixOS/nixpkgs/pull/81371#issuecomment-605526099).
-        # The etcd service is of type "notify", which means that
-        # etcd would not be considered started until etcd is fully online;
-        # however, since NixOS container networking only works sometime *after*
-        # multi-user.target, we forgo etcd's notification entirely.
-        systemd.services.etcd.serviceConfig.Type = lib.mkForce "exec";
-
-        systemd.services.etcd.serviceConfig.StandardOutput = "file:/var/log/etcd.log";
-        systemd.services.etcd.serviceConfig.StandardError = "file:/var/log/etcd.log";
-
-        networking.firewall.allowedTCPPorts = [ 2379 2380 ];
+        networking.firewall.allowedTCPPorts = [ 3901 3900 3902 3903 ];
       };
   };
 in
diff --git a/script/jepsen.garage/nodes b/script/jepsen.garage/nodes
new file mode 100644
index 00000000..b8f3eeb1
--- /dev/null
+++ b/script/jepsen.garage/nodes
@@ -0,0 +1,5 @@
+n1.containers
+n2.containers
+n3.containers
+n4.containers
+n5.containers
diff --git a/script/jepsen.garage/project.clj b/script/jepsen.garage/project.clj
index ac3df57e..2ab03e49 100644
--- a/script/jepsen.garage/project.clj
+++ b/script/jepsen.garage/project.clj
@@ -5,5 +5,5 @@
             :url "https://www.gnu.org/licenses/gpl-3.0.en.html"}
   :main jepsen.garage
   :dependencies [[org.clojure/clojure "1.11.1"]
-                 [jepsen "0.2.1-SNAPSHOT"]]
+                 [jepsen "0.3.2-SNAPSHOT"]]
   :repl-options {:init-ns jepsen.garage})
diff --git a/script/jepsen.garage/setup-tap.sh b/script/jepsen.garage/setup-tap.sh
new file mode 100755
index 00000000..7e09abc3
--- /dev/null
+++ b/script/jepsen.garage/setup-tap.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+USER=$(whoami)
+
+for NODE in 1 2 3 4 5; do
+	sudo ip tuntap add microvm-n$NODE mode tap user $USER
+	sudo ip addr add dev microvm-n$NODE 10.$NODE.0.1
+done
+
+