Remove wesher, reconfigure staging without it

1 files changed, 73 insertions, 24 deletions

diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix
index 9546f5d..8be16af 100644
--- a/nix/deuxfleurs.nix
+++ b/nix/deuxfleurs.nix
@@ -7,6 +7,37 @@ in
   with pkgs.lib;
 {
   options.deuxfleurs =
+    let wg_node = with types; submodule {
+      options = {
+        hostname = mkOption {
+          type = str;
+          description = "Host name";
+        };
+        site_name = mkOption {
+          type = nullOr str;
+          description = "Site where the node is located";
+          default = null;
+        };
+        IP = mkOption {
+          type = str;
+          description = "IP Address in the Wireguard network";
+        };
+        publicKey = mkOption {
+          type = str;
+          description = "Public key";
+        };
+        endpoint = mkOption {
+          type = nullOr str;
+          description = "Wireguard endpoint on the public Internet";
+        };
+        lan_endpoint = mkOption {
+          type = nullOr str;
+          description = "Wireguard endpoint for nodes in the same site";
+          default = null;
+        };
+      };
+    };
+  in
   {
     # Parameters for individual nodes
     network_interface = mkOption {
@@ -30,27 +61,22 @@ in
       type = types.int;
     };
 
-    wesher_cluster_prefix = mkOption {
-      description = "IP address prefix for the Wesher overlay network";
-      type = types.str;
-    };
-    wesher_cluster_prefix_length = mkOption {
-      description = "IP address prefix length for the Wesher overlay network";
-      type = types.int;
-      default = 16;
-    };
-
     cluster_ip = mkOption {
       description = "IP address of this node on the Wesher mesh network";
       type = types.str;
     };
+    wireguard_port = mkOption {
+      description = "Port for incoming Wireguard VPN connections";
+      type = types.port;
+      default = 33799;
+    };
+
     is_raft_server = mkOption {
       description = "Make this node a RAFT server for the Nomad and Consul deployments";
       type = types.bool;
       default = false;
     };
 
-
     # Parameters that generally vary between sites
     lan_default_gateway = mkOption {
       description = "IP address of the default route on the locak network interface";
@@ -70,6 +96,19 @@ in
       description = "Name of this Deuxfleurs deployment";
       type = types.str;
     };
+    cluster_prefix = mkOption {
+      description = "IP address prefix for the Wireguard overlay network";
+      type = types.str;
+    };
+    cluster_prefix_length = mkOption {
+      description = "IP address prefix length for the Wireguard overlay network";
+      type = types.int;
+      default = 16;
+    };
+    cluster_nodes = mkOption {
+      description = "Nodes that are part of the cluster";
+      type = types.listOf wg_node;
+    };
     admin_accounts = mkOption {
       description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
       type = types.attrsOf (types.listOf types.str);
@@ -116,19 +155,24 @@ in
       cfg.lan_ip
     ] ++ cfg.nameservers;
 
-    # wesher overlay network
-    services.wesher = {
-      enable = true;
-      bindIface = cfg.network_interface;
-      overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
-      interface = "wg0";
-      logLevel = "debug";
+    # Configure Wireguard VPN between all nodes
+    networking.wireguard.interfaces.wg0 = {
+      ips = [ "${cfg.cluster_ip}/16" ];
+      listenPort = cfg.wireguard_port;
+      privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
+      peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
+        publicKey = publicKey;
+        allowedIPs = [ "${IP}/32" ];
+        endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
+                   then lan_endpoint else endpoint;
+        persistentKeepalive = 25;
+      }) cfg.cluster_nodes;
     };
 
     # Configure /etc/hosts to link all hostnames to their Wireguard IP
-    #networking.extraHosts = builtins.concatStringsSep "\n" (map
-    #  ({ hostname, IP, ...}: "${IP} ${hostname}")
-    #  (cfg.cluster_nodes ++ cfg.admin_nodes));
+    networking.extraHosts = builtins.concatStringsSep "\n" (map
+      ({ hostname, IP, ...}: "${IP} ${hostname}")
+      cfg.cluster_nodes);
 
     # Enable Hashicorp Consul & Nomad
     services.consul.enable = true;
@@ -245,11 +289,16 @@ in
     networking.firewall = {
       enable = true;
 
-      # Allow anyone to connect on SSH port
       allowedTCPPorts = [
+        # Allow anyone to connect on SSH port
         (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
       ];
 
+      allowedUDPPorts = [
+        # Allow peers to connect to Wireguard
+        cfg.wireguard_port
+      ];
+
       # Allow specific hosts access to specific things in the cluster
       extraCommands = ''
         # Allow everything from router (usefull for UPnP/IGD)
@@ -259,14 +308,14 @@ in
         iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
 
         # Allow other nodes on VPN to access all ports
-        iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+        iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
       '';
 
       # When stopping firewall, delete all rules that were configured manually above
       extraStopCommands = ''
         iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
         iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
-        iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+        iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
       '';
     };
   };