diff options
| -rwxr-xr-x | script/jepsen.garage/destroy-tap.sh | 9 | ||||
| -rw-r--r-- | script/jepsen.garage/flake.lock | 64 | ||||
| -rw-r--r-- | script/jepsen.garage/flake.nix | 45 | ||||
| -rw-r--r-- | script/jepsen.garage/garage-cluster.nix | 34 | ||||
| -rw-r--r-- | script/jepsen.garage/nodes | 5 | ||||
| -rw-r--r-- | script/jepsen.garage/project.clj | 2 | ||||
| -rwxr-xr-x | script/jepsen.garage/setup-tap.sh | 10 |
7 files changed, 150 insertions, 19 deletions
diff --git a/script/jepsen.garage/destroy-tap.sh b/script/jepsen.garage/destroy-tap.sh new file mode 100755 index 00000000..544b3053 --- /dev/null +++ b/script/jepsen.garage/destroy-tap.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +USER=$(whoami) + +for NODE in 1 2 3 4 5; do + sudo ip link delete microvm-n$NODE +done + + diff --git a/script/jepsen.garage/flake.lock b/script/jepsen.garage/flake.lock new file mode 100644 index 00000000..63dd88f8 --- /dev/null +++ b/script/jepsen.garage/flake.lock @@ -0,0 +1,64 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1678901627, + "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "microvm": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1681747916, + "narHash": "sha256-tpWJMHWbTrFD2Nmj3Y3qYXoaTP4LFT0P0wt5zW8/aI8=", + "owner": "astro", + "repo": "microvm.nix", + "rev": "68f1b9ece0f116d5ea1d1ecaf17f7b526303df81", + "type": "github" + }, + "original": { + "owner": "astro", + "repo": "microvm.nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1681737997, + "narHash": "sha256-pHhjgsIkRMu80LmVe8QoKIZB6VZGRRxFmIvsC5S89k4=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "f00994e78cd39e6fc966f0c4103f908e63284780", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "microvm": "microvm", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/script/jepsen.garage/flake.nix b/script/jepsen.garage/flake.nix index 76ee8ebf..2afa9ea1 100644 --- a/script/jepsen.garage/flake.nix +++ b/script/jepsen.garage/flake.nix @@ -4,19 +4,56 @@ inputs.microvm.url = "github:astro/microvm.nix"; inputs.microvm.inputs.nixpkgs.follows = "nixpkgs"; - outputs = { self, nixpkgs, microvm }: { - # Example nixosConfigurations entry - nixosConfigurations.my-microvm = nixpkgs.lib.nixosSystem { + outputs = { self, nixpkgs, microvm }: + with nixpkgs.lib; + let + addressMap = + { + "n1" = { ip = "10.1.0.10"; mac = "02:00:00:00:99:01"; }; + "n2" = { ip = "10.2.0.10"; mac = "02:00:00:00:99:02"; }; + "n3" = { ip = "10.3.0.10"; mac = "02:00:00:00:99:03"; }; + "n4" = { ip = "10.4.0.10"; mac = "02:00:00:00:99:04"; }; + "n5" = { ip = "10.5.0.10"; mac = "02:00:00:00:99:05"; }; + }; + toHostsEntry = name: { ip, ... }: "${ip} ${name}"; + extraHosts = + builtins.concatStringsSep "\n" + (attrsets.mapAttrsToList toHostsEntry addressMap); + + nodeConfig = hostName: { ip, mac }: nixosSystem { system = "x86_64-linux"; modules = [ # Include the microvm module microvm.nixosModules.microvm # Add more modules here { - networking.hostName = "my-microvm"; + networking = { + inherit hostName extraHosts; + }; + microvm.hypervisor = "cloud-hypervisor"; + microvm.interfaces = [ { + inherit mac; + type = "tap"; + id = "microvm-${hostName}"; + } ]; + + services.openssh = { + enable = true; + permitRootLogin = "yes"; + }; + users.users.root.initialPassword = "root"; + + #services.garage = { + # enable = true; + # logLevel = "debug"; + # settings.replication_mode = "3"; + #}; } ]; }; + in + { + nixosConfigurations = mapAttrs nodeConfig addressMap; }; } diff --git a/script/jepsen.garage/garage-cluster.nix b/script/jepsen.garage/garage-cluster.nix index 32fedc04..ebc73aaf 100644 --- a/script/jepsen.garage/garage-cluster.nix +++ b/script/jepsen.garage/garage-cluster.nix @@ -29,30 +29,36 @@ let services.openssh = { enable = true; permitRootLogin = "yes"; + kexAlgorithms = [ "curve25519-sha256@libssh.org" "ecdh-sha2-nistp256" "ecdh-sha2-nistp384" "ecdh-sha2-nistp521" "diffie-hellman-group-exchange-sha256" "diffie-hellman-group14-sha1" "diffie-hellman-group-exchange-sha1" "diffie-hellman-group1-sha1" ]; }; users.users.root.initialPassword = "root"; + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9" + ]; system.stateVersion = "22.11"; services.garage = { enable = true; logLevel = "debug"; - settings.replication_mode = "3"; + settings = { + replication_mode = "3"; + db_engine = "lmdb"; + rpc_secret = "b597bb28ebdc90cdc4f15712733ca678cfb9a7e0311e0b9e93db9610fc3685e6"; + rpc_bind_addr = "0.0.0.0:3901"; + s3_api = { + region = "garage"; + api_bind_addr = "0.0.0.0:3900"; + }; + k2v_api.api_bind_addr = "0.0.0.0:3902"; + admin = { + api_bind_addr = "0.0.0.0:3903"; + admin_token = "icanhazadmin"; + }; + }; }; - # Workaround for nixos-container issue - # (see https://github.com/NixOS/nixpkgs/issues/67265 and - # https://github.com/NixOS/nixpkgs/pull/81371#issuecomment-605526099). - # The etcd service is of type "notify", which means that - # etcd would not be considered started until etcd is fully online; - # however, since NixOS container networking only works sometime *after* - # multi-user.target, we forgo etcd's notification entirely. - systemd.services.etcd.serviceConfig.Type = lib.mkForce "exec"; - - systemd.services.etcd.serviceConfig.StandardOutput = "file:/var/log/etcd.log"; - systemd.services.etcd.serviceConfig.StandardError = "file:/var/log/etcd.log"; - - networking.firewall.allowedTCPPorts = [ 2379 2380 ]; + networking.firewall.allowedTCPPorts = [ 3901 3900 3902 3903 ]; }; }; in diff --git a/script/jepsen.garage/nodes b/script/jepsen.garage/nodes new file mode 100644 index 00000000..b8f3eeb1 --- /dev/null +++ b/script/jepsen.garage/nodes @@ -0,0 +1,5 @@ +n1.containers +n2.containers +n3.containers +n4.containers +n5.containers diff --git a/script/jepsen.garage/project.clj b/script/jepsen.garage/project.clj index ac3df57e..2ab03e49 100644 --- a/script/jepsen.garage/project.clj +++ b/script/jepsen.garage/project.clj @@ -5,5 +5,5 @@ :url "https://www.gnu.org/licenses/gpl-3.0.en.html"} :main jepsen.garage :dependencies [[org.clojure/clojure "1.11.1"] - [jepsen "0.2.1-SNAPSHOT"]] + [jepsen "0.3.2-SNAPSHOT"]] :repl-options {:init-ns jepsen.garage}) diff --git a/script/jepsen.garage/setup-tap.sh b/script/jepsen.garage/setup-tap.sh new file mode 100755 index 00000000..7e09abc3 --- /dev/null +++ b/script/jepsen.garage/setup-tap.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +USER=$(whoami) + +for NODE in 1 2 3 4 5; do + sudo ip tuntap add microvm-n$NODE mode tap user $USER + sudo ip addr add dev microvm-n$NODE 10.$NODE.0.1 +done + + |