Remove wesher, reconfigure staging without it

4 files changed, 73 insertions, 193 deletions

diff --git a/nix/configuration.nix b/nix/configuration.nix
index 53246e3..26619d3 100644
--- a/nix/configuration.nix
+++ b/nix/configuration.nix
@@ -11,8 +11,6 @@
       ./hardware-configuration.nix
       # Include generic Deuxfleurs module
       ./deuxfleurs.nix
-      # Wesher module
-      ./wesher_service.nix
       # Configuration for this deployment (a cluster)
       ./cluster.nix
       # Configuration local for this Deuxfleurs site (set of nodes)
@@ -21,10 +19,6 @@
       ./node.nix
     ];
 
-  nixpkgs.overlays = [
-    (import ./wesher.nix)
-  ];
-
   # The global useDHCP flag is deprecated, therefore explicitly set to false here.
   # Per-interface useDHCP will be mandatory in the future, so this generated config
   # replicates the default behaviour.
@@ -70,7 +64,6 @@ SystemMaxUse=1G
     docker
     docker-compose
     wireguard-tools
-    wesher
     #qemu
     #qemu_kvm
   ];
diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix
index 9546f5d..8be16af 100644
--- a/nix/deuxfleurs.nix
+++ b/nix/deuxfleurs.nix
@@ -7,6 +7,37 @@ in
   with pkgs.lib;
 {
   options.deuxfleurs =
+    let wg_node = with types; submodule {
+      options = {
+        hostname = mkOption {
+          type = str;
+          description = "Host name";
+        };
+        site_name = mkOption {
+          type = nullOr str;
+          description = "Site where the node is located";
+          default = null;
+        };
+        IP = mkOption {
+          type = str;
+          description = "IP Address in the Wireguard network";
+        };
+        publicKey = mkOption {
+          type = str;
+          description = "Public key";
+        };
+        endpoint = mkOption {
+          type = nullOr str;
+          description = "Wireguard endpoint on the public Internet";
+        };
+        lan_endpoint = mkOption {
+          type = nullOr str;
+          description = "Wireguard endpoint for nodes in the same site";
+          default = null;
+        };
+      };
+    };
+  in
   {
     # Parameters for individual nodes
     network_interface = mkOption {
@@ -30,27 +61,22 @@ in
       type = types.int;
     };
 
-    wesher_cluster_prefix = mkOption {
-      description = "IP address prefix for the Wesher overlay network";
-      type = types.str;
-    };
-    wesher_cluster_prefix_length = mkOption {
-      description = "IP address prefix length for the Wesher overlay network";
-      type = types.int;
-      default = 16;
-    };
-
     cluster_ip = mkOption {
       description = "IP address of this node on the Wesher mesh network";
       type = types.str;
     };
+    wireguard_port = mkOption {
+      description = "Port for incoming Wireguard VPN connections";
+      type = types.port;
+      default = 33799;
+    };
+
     is_raft_server = mkOption {
       description = "Make this node a RAFT server for the Nomad and Consul deployments";
       type = types.bool;
       default = false;
     };
 
-
     # Parameters that generally vary between sites
     lan_default_gateway = mkOption {
       description = "IP address of the default route on the locak network interface";
@@ -70,6 +96,19 @@ in
       description = "Name of this Deuxfleurs deployment";
       type = types.str;
     };
+    cluster_prefix = mkOption {
+      description = "IP address prefix for the Wireguard overlay network";
+      type = types.str;
+    };
+    cluster_prefix_length = mkOption {
+      description = "IP address prefix length for the Wireguard overlay network";
+      type = types.int;
+      default = 16;
+    };
+    cluster_nodes = mkOption {
+      description = "Nodes that are part of the cluster";
+      type = types.listOf wg_node;
+    };
     admin_accounts = mkOption {
       description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
       type = types.attrsOf (types.listOf types.str);
@@ -116,19 +155,24 @@ in
       cfg.lan_ip
     ] ++ cfg.nameservers;
 
-    # wesher overlay network
-    services.wesher = {
-      enable = true;
-      bindIface = cfg.network_interface;
-      overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
-      interface = "wg0";
-      logLevel = "debug";
+    # Configure Wireguard VPN between all nodes
+    networking.wireguard.interfaces.wg0 = {
+      ips = [ "${cfg.cluster_ip}/16" ];
+      listenPort = cfg.wireguard_port;
+      privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
+      peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
+        publicKey = publicKey;
+        allowedIPs = [ "${IP}/32" ];
+        endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
+                   then lan_endpoint else endpoint;
+        persistentKeepalive = 25;
+      }) cfg.cluster_nodes;
     };
 
     # Configure /etc/hosts to link all hostnames to their Wireguard IP
-    #networking.extraHosts = builtins.concatStringsSep "\n" (map
-    #  ({ hostname, IP, ...}: "${IP} ${hostname}")
-    #  (cfg.cluster_nodes ++ cfg.admin_nodes));
+    networking.extraHosts = builtins.concatStringsSep "\n" (map
+      ({ hostname, IP, ...}: "${IP} ${hostname}")
+      cfg.cluster_nodes);
 
     # Enable Hashicorp Consul & Nomad
     services.consul.enable = true;
@@ -245,11 +289,16 @@ in
     networking.firewall = {
       enable = true;
 
-      # Allow anyone to connect on SSH port
       allowedTCPPorts = [
+        # Allow anyone to connect on SSH port
         (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
       ];
 
+      allowedUDPPorts = [
+        # Allow peers to connect to Wireguard
+        cfg.wireguard_port
+      ];
+
       # Allow specific hosts access to specific things in the cluster
       extraCommands = ''
         # Allow everything from router (usefull for UPnP/IGD)
@@ -259,14 +308,14 @@ in
         iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
 
         # Allow other nodes on VPN to access all ports
-        iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+        iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
       '';
 
       # When stopping firewall, delete all rules that were configured manually above
       extraStopCommands = ''
         iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
         iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
-        iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+        iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
       '';
     };
   };
diff --git a/nix/wesher.nix b/nix/wesher.nix
deleted file mode 100644
index c39a3e6..0000000
--- a/nix/wesher.nix
+++ /dev/null
@@ -1,25 +0,0 @@
-self: super:
-{
-  wesher = super.buildGoModule rec {
-    pname = "wesher";
-    version = "0.2.6";
-
-    src = super.fetchFromGitHub {
-      owner = "costela";
-      repo = "wesher";
-      rev = "v${version}";
-      sha256 = "1iagmnw2yf15r0fpikk610w0lm0gcxw83lcwfjyr2jv1q2ys71hh";
-    };
-
-    vendorSha256 = "0nyg0wzn8d4rzjs8yrxxj3gha94043ll80s1ql0fml025q2f3705";
-
-    checkPhase = "true";
-
-    meta = with super.lib; {
-      description = "wireguard overlay mesh network manager";
-      homepage = "https://github.com/costela/wesher";
-      license = licenses.gpl3Plus;
-      platforms = platforms.linux;
-    };
-  };
-}
diff --git a/nix/wesher_service.nix b/nix/wesher_service.nix
deleted file mode 100644
index d269a2f..0000000
--- a/nix/wesher_service.nix
+++ /dev/null
@@ -1,137 +0,0 @@
-{ config, lib, pkgs, ... }:
-with lib;
-let
-  keysPath = "/var/lib/wesher/secrets";
-  cfg = config.services.wesher;
-in {
-  options = with types; {
-    services.wesher = {
-      enable = mkEnableOption "wesher wireguard overlay mesh network manager";
-
-      package = mkOption {
-        type = package;
-        default = pkgs.wesher;
-        defaultText = literalExpression "pkgs.wesher";
-        description = "Wesher package to use.";
-      };
-
-      clusterKey = mkOption {
-        type = nullOr str;
-        default = null;
-        description = "shared key for cluster membership to use on first initialization, if no key was previously used by Wesher. Must be 32 bytes base64 encoded; will be generated if not provided. Setting this parameter value will not overwrite an existing cluster key; to do so please delete ${keysPath}";
-      };
-
-      bindAddr = mkOption {
-        type = nullOr str;
-        default = null;
-        description = "IP address to bind to for cluster membership (cannot be used with --bind-iface)";
-      };
-
-      bindIface = mkOption {
-        type = nullOr str;
-        default = null;
-        description = "Interface to bind to for cluster membership (cannot be used with --bind-addr)";
-      };
-
-      join = mkOption {
-        type = listOf str;
-        default = [];
-        description = "list of hostnames or IP addresses to existing cluster members; if not provided, will attempt resuming any known state or otherwise wait for further members";
-      };
-
-      clusterPort = mkOption {
-        type = port;
-        default = 7946;
-        description = "port used for membership gossip traffic (both TCP and UDP); must be the same accross cluster";
-      };
-
-      wireguardPort = mkOption {
-        type = port;
-        default = 51820;
-        description = "port used for wireguard traffic (UDP); must be the same accross cluster";
-      };
-
-      overlayNet = mkOption {
-        type = str;
-        default = "10.0.0.0/8";
-        description = "the network in which to allocate addresses for the overlay mesh network (CIDR format); smaller networks increase the chance of IP collision";
-      };
-
-      interface = mkOption {
-        type = str;
-        default = "wgoverlay";
-        description = "name of the wireguard interface to create and manage";
-      };
-
-      logLevel = mkOption {
-        type = str;
-        default = "warn";
-        description = "set the verbosity (one of debug/info/warn/error)";
-      };
-      
-    };
-  };
-
-  config = mkIf cfg.enable (let binWesher = cfg.package + "/bin/wesher";
-  in {
-    system.activationScripts.wesher = if (cfg.clusterKey != null) then ''
-      if [ ! -e ${keysPath} ]
-      then
-        mkdir --mode=700 -p ${builtins.dirOf keysPath}
-        echo "WESHER_CLUSTER_KEY=${cfg.clusterKey}" > ${keysPath}
-      fi
-    '' else ''
-      if [ ! -e ${keysPath} ]
-      then
-        mkdir --mode=700 -p ${builtins.dirOf keysPath}
-        echo "WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64)" > ${keysPath}
-      fi
-    '';
-
-    systemd.services.wesher = {
-      description = "wesher wireguard overlay mesh network manager";
-      bindsTo = [ "network-online.target" ];
-      after = [ "network-online.target" ];
-      wantedBy = [ "multi-user.target" ];
-
-      environment = {
-        WESHER_JOIN = builtins.concatStringsSep "," cfg.join;
-        WESHER_CLUSTER_PORT = builtins.toString cfg.clusterPort;
-        WESHER_WIREGUARD_PORT = builtins.toString cfg.wireguardPort;
-        WESHER_OVERLAY_NET = cfg.overlayNet;
-        WESHER_INTERFACE = cfg.interface;
-        WESHER_LOG_LEVEL = cfg.logLevel;
-        WESHER_NO_ETC_HOSTS = "true";
-      } 
-      // (if (cfg.bindAddr != null) then { WESHER_BIND_ADDR = cfg.bindAddr; } else {})
-      // (if (cfg.bindIface != null) then { WESHER_BIND_IFACE = cfg.bindIface; } else {})
-      ;
-
-      serviceConfig = {
-        ExecStart = "${binWesher}";
-        Restart = "always";
-
-        EnvironmentFile = keysPath;
-
-        User = "wesher";
-        DynamicUser = true;
-        StateDirectory = "wesher";
-
-        AmbientCapabilities = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE";
-        CapabilityBoundingSet = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE";
-        MemoryDenyWriteExecute = true;
-        ProtectControlGroups = true;
-        ProtectKernelModules = true;
-        ProtectKernelTunables = true;
-        RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6 AF_NETLINK";
-        RestrictNamespaces = true;
-        RestrictRealtime = true;
-        SystemCallArchitectures = "native";
-        SystemCallFilter = "~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @resources";
-      };
-    };
-
-    networking.firewall.allowedUDPPorts = mkIf cfg.enable [ cfg.clusterPort cfg.wireguardPort ];
-    networking.firewall.allowedTCPPorts = mkIf cfg.enable [ cfg.clusterPort ];
-  });
-}