Replace ad-hoc wireguard by wesher on staging cluster

2 files changed, 71 insertions, 104 deletions

diff --git a/nix/configuration.nix b/nix/configuration.nix
index 3f3aa49..984307c 100644
--- a/nix/configuration.nix
+++ b/nix/configuration.nix
@@ -69,6 +69,7 @@ SystemMaxUse=1G
     rclone
     docker
     docker-compose
+    wireguard
     wesher
   ];
 
@@ -81,43 +82,6 @@ SystemMaxUse=1G
   services.openssh.enable = true;
   services.openssh.passwordAuthentication = false;
 
-  services.wesher = {
-    enable = true;
-    join = [ "192.168.1.22" "192.168.1.23" ];
-    bindAddr = config.deuxfleurs.lan_ip;  # for now
-    overlayNet = "10.14.0.0/16";
-  };
-
-  # ---- CONFIG FOR DEUXFLEURS CLUSTER ----
-
-  # Open ports in the firewall.
-  networking.firewall = {
-    enable = true;
-
-    # Allow anyone to connect on SSH port
-    allowedTCPPorts = [
-      (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
-    ];
-
-    # Allow specific hosts access to specific things in the cluster
-    extraCommands = ''
-      # Allow everything from router (usefull for UPnP/IGD)
-      iptables -A INPUT -s 192.168.1.254 -j ACCEPT
-
-      # Allow docker containers to access all ports
-      iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
-
-      # Allow other nodes on VPN to access all ports
-      iptables -A INPUT -s 10.42.0.0/16 -j ACCEPT
-    '';
-
-    # When stopping firewall, delete all rules that were configured manually above
-    extraStopCommands = ''
-      iptables -D INPUT -s 192.168.1.254 -j ACCEPT
-      iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
-      iptables -D INPUT -s 10.42.0.0/16 -j ACCEPT
-    '';
-  };
 
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions
diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix
index 7772f0e..2050776 100644
--- a/nix/deuxfleurs.nix
+++ b/nix/deuxfleurs.nix
@@ -7,37 +7,6 @@ in
   with pkgs.lib;
 {
   options.deuxfleurs = 
-    let wg_node = with types; submodule {
-      options = {
-        hostname = mkOption {
-          type = str;
-          description = "Host name";
-        };
-        site_name = mkOption {
-          type = nullOr str;
-          description = "Site where the node is located";
-          default = null;
-        };
-        IP = mkOption {
-          type = str;
-          description = "IP Address";
-        };
-        publicKey = mkOption {
-          type = str;
-          description = "Public key";
-        };
-        endpoint = mkOption {
-          type = nullOr str;
-          description = "Wireguard endpoint on the public Internet";
-        };
-        lan_endpoint = mkOption {
-          type = nullOr str;
-          description = "Wireguard endpoint for nodes in the same site";
-          default = null;
-        };
-      };
-    };
-  in
   {
     # Parameters for individual nodes
     network_interface = mkOption {
@@ -52,14 +21,28 @@ in
       description = "Prefix length associated with lan_ip";
       type = types.int;
     };
+    ipv6 = mkOption {
+      description = "Public IPv6 address of this node";
+      type = types.str;
+    };
+    ipv6_prefix_length = mkOption {
+      description = "Prefix length associated with ipv6 ip";
+      type = types.int;
+    };
 
-    vpn_ip = mkOption {
-      description = "IP address of this node on the Wireguard VPN";
+    wesher_cluster_prefix = mkOption {
+      description = "IP address prefix for the Wesher overlay network";
       type = types.str;
     };
-    vpn_listen_port = mkOption {
-      description = "Port for incoming Wireguard VPN connections";
-      type = types.port;
+    wesher_cluster_prefix_length = mkOption {
+      description = "IP address prefix length for the Wesher overlay network";
+      type = types.int;
+      default = 16;
+    };
+
+    cluster_ip = mkOption {
+      description = "IP address of this node on the Wesher mesh network";
+      type = types.str;
     };
     is_raft_server = mkOption {
       description = "Make this node a RAFT server for the Nomad and Consul deployments";
@@ -83,14 +66,6 @@ in
       description = "Name of this Deuxfleurs deployment";
       type = types.str;
     };
-    cluster_nodes = mkOption {
-      description = "Nodes that are part of the cluster";
-      type = types.listOf wg_node;
-    };
-    admin_nodes = mkOption {
-      description = "Machines that are part of the Wireguard VPN for administration purposes";
-      type = types.listOf wg_node;
-    };
     admin_accounts = mkOption {
       description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
       type = types.attrsOf (types.listOf types.str);
@@ -114,32 +89,31 @@ in
           prefixLength = cfg.lan_ip_prefix_length;
         }
       ];
+      ipv6.addresses = [
+        {
+          address = cfg.ipv6;
+          prefixLength = cfg.ipv6_prefix_length;
+        }
+      ];
     };
     networking.defaultGateway = {
       address = cfg.lan_default_gateway;
       interface = cfg.network_interface;
     };
 
-    # Configure Wireguard VPN between all nodes
-    networking.wireguard.interfaces.wg0 = {
-      ips = [ "${cfg.vpn_ip}/16" ];
-      listenPort = cfg.vpn_listen_port;
-      privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
-      peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
-        publicKey = publicKey;
-        allowedIPs = [ "${IP}/32" ];
-        endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
-                   then lan_endpoint else endpoint;
-        persistentKeepalive = 25;
-      }) (cfg.cluster_nodes ++ cfg.admin_nodes);
+    # wesher overlay network
+    services.wesher = {
+      enable = true;
+      bindAddr = cfg.ipv6;
+      overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
+      interface = "wg0";
+      logLevel = "debug";
     };
 
-    networking.firewall.allowedUDPPorts = [ cfg.vpn_listen_port ];
-
     # Configure /etc/hosts to link all hostnames to their Wireguard IP
-    networking.extraHosts = builtins.concatStringsSep "\n" (map
-      ({ hostname, IP, ...}: "${IP} ${hostname}")
-      (cfg.cluster_nodes ++ cfg.admin_nodes));
+    #networking.extraHosts = builtins.concatStringsSep "\n" (map
+    #  ({ hostname, IP, ...}: "${IP} ${hostname}")
+    #  (cfg.cluster_nodes ++ cfg.admin_nodes));
 
     # Enable Hashicorp Consul & Nomad
     services.consul.enable = true;
@@ -156,14 +130,12 @@ in
         "site" = cfg.site_name;
       };
       ui = true;
-      bind_addr = cfg.vpn_ip;
+      bind_addr = "${cfg.cluster_ip}";
 
       ports.http = -1;
       addresses.https = "0.0.0.0";
       ports.https = 8501;
 
-      retry_join = map (node_info: node_info.IP) cfg.cluster_nodes;
-
       ca_file = "/var/lib/consul/pki/consul-ca.crt";
       cert_file = "/var/lib/consul/pki/consul2022.crt";
       key_file = "/var/lib/consul/pki/consul2022.key";
@@ -185,9 +157,9 @@ in
       region = cfg.cluster_name;
       datacenter = cfg.site_name;
       advertise = {
-        rpc = cfg.vpn_ip;
-        http = cfg.vpn_ip;
-        serf = cfg.vpn_ip;
+        rpc = "${cfg.cluster_ip}";
+        http = "${cfg.cluster_ip}";
+        serf = "${cfg.cluster_ip}";
       };
       consul = {
         address = "localhost:8501";
@@ -227,5 +199,36 @@ in
         }
       ];
     };
+
+    # ---- Firewall config ----
+
+    # Open ports in the firewall.
+    networking.firewall = {
+      enable = true;
+
+      # Allow anyone to connect on SSH port
+      allowedTCPPorts = [
+        (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
+      ];
+
+      # Allow specific hosts access to specific things in the cluster
+      extraCommands = ''
+        # Allow everything from router (usefull for UPnP/IGD)
+        iptables -A INPUT -s 192.168.1.254 -j ACCEPT
+
+        # Allow docker containers to access all ports
+        iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
+
+        # Allow other nodes on VPN to access all ports
+        iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+      '';
+
+      # When stopping firewall, delete all rules that were configured manually above
+      extraStopCommands = ''
+        iptables -D INPUT -s 192.168.1.254 -j ACCEPT
+        iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
+        iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+      '';
+    };
   };
 }