diff options
| -rw-r--r-- | cluster/staging/cluster.nix | 45 | ||||
| -rwxr-xr-x | deploy_nixos | 6 | ||||
| -rwxr-xr-x | deploy_wesher_key | 4 | ||||
| -rwxr-xr-x | gen_wesher_key | 17 | ||||
| -rw-r--r-- | nix/configuration.nix | 7 | ||||
| -rw-r--r-- | nix/deuxfleurs.nix | 97 | ||||
| -rw-r--r-- | nix/wesher.nix | 25 | ||||
| -rw-r--r-- | nix/wesher_service.nix | 137 |
8 files changed, 106 insertions, 232 deletions
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix index 0836dcb..d71b274 100644 --- a/cluster/staging/cluster.nix +++ b/cluster/staging/cluster.nix @@ -3,16 +3,36 @@ { deuxfleurs.cluster_name = "staging"; - # Bootstrap nodes for Wesher overlay network - services.wesher.join = [ - "192.168.1.21" # cariacou - "192.168.1.22" # carcajou - "192.168.1.23" # caribou - ]; + # The IP range to use for the Wireguard overlay of this cluster + deuxfleurs.cluster_prefix = "10.14.0.0"; + deuxfleurs.cluster_prefix_length = 16; - # The IP range to use for the Wesher overlay of this cluster - deuxfleurs.wesher_cluster_prefix = "10.14.0.0"; - deuxfleurs.wesher_cluster_prefix_length = 16; + deuxfleurs.cluster_nodes = [ + { + hostname = "cariacou"; + site_name = "neptune"; + publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA="; + IP = "10.14.179.56"; + lan_endpoint = "192.168.1.21:33799"; + endpoint = "192.168.1.21:33799"; ## TODO nat + } + { + hostname = "carcajou"; + site_name = "neptune"; + publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk="; + IP = "10.14.252.121"; + lan_endpoint = "192.168.1.22:33799"; + endpoint = "192.168.1.22:33799"; ## TODO nat + } + { + hostname = "caribou"; + site_name = "neptune"; + publicKey = "X1OlfgjZDdJw3yTg0ytGFlJZXzqT0yHJmYP15R9IpD8="; + IP = "10.14.181.82"; + lan_endpoint = "192.168.1.23:33799"; + endpoint = "192.168.1.23:33799"; ## TODO nat + } + ]; # Bootstrap IPs for Consul cluster, # these are IPs on the Wesher overlay @@ -22,13 +42,6 @@ "10.14.252.121" # carcajou ]; - # Add wesher hosts to /etc/hosts, needed for elasticsearch - networking.extraHosts = '' -10.14.181.82 caribou -10.14.179.56 cariacou -10.14.242.121 carcajou - ''; - deuxfleurs.admin_accounts = { lx = [ # Keys for accessing nodes from outside diff --git a/deploy_nixos b/deploy_nixos index 30691e0..f62843d 100755 --- a/deploy_nixos +++ b/deploy_nixos @@ -3,10 +3,12 @@ copy nix/configuration.nix /etc/nixos/configuration.nix copy nix/deuxfleurs.nix /etc/nixos/deuxfleurs.nix copy nix/remote-unlock.nix /etc/nixos/remote-unlock.nix -copy nix/wesher.nix /etc/nixos/wesher.nix -copy nix/wesher_service.nix /etc/nixos/wesher_service.nix copy cluster/$CLUSTER/cluster.nix /etc/nixos/cluster.nix copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix +cmd 'mkdir -p /var/lib/deuxfleurs/wireguard-keys' +cmd 'test -f /var/lib/deuxfleurs/wireguard-keys/private || (wg genkey > /var/lib/deuxfleurs/wireguard-keys/private; chmod 600 /var/lib/deuxfleurs/wireguard-keys/private)' +cmd 'echo "Public key: $(wg pubkey < /var/lib/deuxfleurs/wireguard-keys/private)"' + cmd nixos-rebuild switch --show-trace diff --git a/deploy_wesher_key b/deploy_wesher_key deleted file mode 100755 index 8f7ed77..0000000 --- a/deploy_wesher_key +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env ./sshtool - -write_pass deuxfleurs/cluster/$CLUSTER/wesher_key /var/lib/wesher/secrets -cmd systemctl restart wesher diff --git a/gen_wesher_key b/gen_wesher_key deleted file mode 100755 index c66fade..0000000 --- a/gen_wesher_key +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env sh - -cd $(dirname $0) - -CLUSTER="$1" -if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then - echo "Usage: $0 <cluster name>" - echo "The cluster name must be the name of a subdirectory of cluster/" - exit 1 -fi - -K=deuxfleurs/cluster/$CLUSTER/wesher_key -if ! pass $K >/dev/null; then - pass insert -m $K <<EOF -WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64) -EOF -fi diff --git a/nix/configuration.nix b/nix/configuration.nix index 53246e3..26619d3 100644 --- a/nix/configuration.nix +++ b/nix/configuration.nix @@ -11,8 +11,6 @@ ./hardware-configuration.nix # Include generic Deuxfleurs module ./deuxfleurs.nix - # Wesher module - ./wesher_service.nix # Configuration for this deployment (a cluster) ./cluster.nix # Configuration local for this Deuxfleurs site (set of nodes) @@ -21,10 +19,6 @@ ./node.nix ]; - nixpkgs.overlays = [ - (import ./wesher.nix) - ]; - # The global useDHCP flag is deprecated, therefore explicitly set to false here. # Per-interface useDHCP will be mandatory in the future, so this generated config # replicates the default behaviour. @@ -70,7 +64,6 @@ SystemMaxUse=1G docker docker-compose wireguard-tools - wesher #qemu #qemu_kvm ]; diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 9546f5d..8be16af 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -7,6 +7,37 @@ in with pkgs.lib; { options.deuxfleurs = + let wg_node = with types; submodule { + options = { + hostname = mkOption { + type = str; + description = "Host name"; + }; + site_name = mkOption { + type = nullOr str; + description = "Site where the node is located"; + default = null; + }; + IP = mkOption { + type = str; + description = "IP Address in the Wireguard network"; + }; + publicKey = mkOption { + type = str; + description = "Public key"; + }; + endpoint = mkOption { + type = nullOr str; + description = "Wireguard endpoint on the public Internet"; + }; + lan_endpoint = mkOption { + type = nullOr str; + description = "Wireguard endpoint for nodes in the same site"; + default = null; + }; + }; + }; + in { # Parameters for individual nodes network_interface = mkOption { @@ -30,27 +61,22 @@ in type = types.int; }; - wesher_cluster_prefix = mkOption { - description = "IP address prefix for the Wesher overlay network"; - type = types.str; - }; - wesher_cluster_prefix_length = mkOption { - description = "IP address prefix length for the Wesher overlay network"; - type = types.int; - default = 16; - }; - cluster_ip = mkOption { description = "IP address of this node on the Wesher mesh network"; type = types.str; }; + wireguard_port = mkOption { + description = "Port for incoming Wireguard VPN connections"; + type = types.port; + default = 33799; + }; + is_raft_server = mkOption { description = "Make this node a RAFT server for the Nomad and Consul deployments"; type = types.bool; default = false; }; - # Parameters that generally vary between sites lan_default_gateway = mkOption { description = "IP address of the default route on the locak network interface"; @@ -70,6 +96,19 @@ in description = "Name of this Deuxfleurs deployment"; type = types.str; }; + cluster_prefix = mkOption { + description = "IP address prefix for the Wireguard overlay network"; + type = types.str; + }; + cluster_prefix_length = mkOption { + description = "IP address prefix length for the Wireguard overlay network"; + type = types.int; + default = 16; + }; + cluster_nodes = mkOption { + description = "Nodes that are part of the cluster"; + type = types.listOf wg_node; + }; admin_accounts = mkOption { description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys"; type = types.attrsOf (types.listOf types.str); @@ -116,19 +155,24 @@ in cfg.lan_ip ] ++ cfg.nameservers; - # wesher overlay network - services.wesher = { - enable = true; - bindIface = cfg.network_interface; - overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}"; - interface = "wg0"; - logLevel = "debug"; + # Configure Wireguard VPN between all nodes + networking.wireguard.interfaces.wg0 = { + ips = [ "${cfg.cluster_ip}/16" ]; + listenPort = cfg.wireguard_port; + privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; + peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: { + publicKey = publicKey; + allowedIPs = [ "${IP}/32" ]; + endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null + then lan_endpoint else endpoint; + persistentKeepalive = 25; + }) cfg.cluster_nodes; }; # Configure /etc/hosts to link all hostnames to their Wireguard IP - #networking.extraHosts = builtins.concatStringsSep "\n" (map - # ({ hostname, IP, ...}: "${IP} ${hostname}") - # (cfg.cluster_nodes ++ cfg.admin_nodes)); + networking.extraHosts = builtins.concatStringsSep "\n" (map + ({ hostname, IP, ...}: "${IP} ${hostname}") + cfg.cluster_nodes); # Enable Hashicorp Consul & Nomad services.consul.enable = true; @@ -245,11 +289,16 @@ in networking.firewall = { enable = true; - # Allow anyone to connect on SSH port allowedTCPPorts = [ + # Allow anyone to connect on SSH port (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports) ]; + allowedUDPPorts = [ + # Allow peers to connect to Wireguard + cfg.wireguard_port + ]; + # Allow specific hosts access to specific things in the cluster extraCommands = '' # Allow everything from router (usefull for UPnP/IGD) @@ -259,14 +308,14 @@ in iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # Allow other nodes on VPN to access all ports - iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT + iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT ''; # When stopping firewall, delete all rules that were configured manually above extraStopCommands = '' iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT - iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT + iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT ''; }; }; diff --git a/nix/wesher.nix b/nix/wesher.nix deleted file mode 100644 index c39a3e6..0000000 --- a/nix/wesher.nix +++ /dev/null @@ -1,25 +0,0 @@ -self: super: -{ - wesher = super.buildGoModule rec { - pname = "wesher"; - version = "0.2.6"; - - src = super.fetchFromGitHub { - owner = "costela"; - repo = "wesher"; - rev = "v${version}"; - sha256 = "1iagmnw2yf15r0fpikk610w0lm0gcxw83lcwfjyr2jv1q2ys71hh"; - }; - - vendorSha256 = "0nyg0wzn8d4rzjs8yrxxj3gha94043ll80s1ql0fml025q2f3705"; - - checkPhase = "true"; - - meta = with super.lib; { - description = "wireguard overlay mesh network manager"; - homepage = "https://github.com/costela/wesher"; - license = licenses.gpl3Plus; - platforms = platforms.linux; - }; - }; -} diff --git a/nix/wesher_service.nix b/nix/wesher_service.nix deleted file mode 100644 index d269a2f..0000000 --- a/nix/wesher_service.nix +++ /dev/null @@ -1,137 +0,0 @@ -{ config, lib, pkgs, ... }: -with lib; -let - keysPath = "/var/lib/wesher/secrets"; - cfg = config.services.wesher; -in { - options = with types; { - services.wesher = { - enable = mkEnableOption "wesher wireguard overlay mesh network manager"; - - package = mkOption { - type = package; - default = pkgs.wesher; - defaultText = literalExpression "pkgs.wesher"; - description = "Wesher package to use."; - }; - - clusterKey = mkOption { - type = nullOr str; - default = null; - description = "shared key for cluster membership to use on first initialization, if no key was previously used by Wesher. Must be 32 bytes base64 encoded; will be generated if not provided. Setting this parameter value will not overwrite an existing cluster key; to do so please delete ${keysPath}"; - }; - - bindAddr = mkOption { - type = nullOr str; - default = null; - description = "IP address to bind to for cluster membership (cannot be used with --bind-iface)"; - }; - - bindIface = mkOption { - type = nullOr str; - default = null; - description = "Interface to bind to for cluster membership (cannot be used with --bind-addr)"; - }; - - join = mkOption { - type = listOf str; - default = []; - description = "list of hostnames or IP addresses to existing cluster members; if not provided, will attempt resuming any known state or otherwise wait for further members"; - }; - - clusterPort = mkOption { - type = port; - default = 7946; - description = "port used for membership gossip traffic (both TCP and UDP); must be the same accross cluster"; - }; - - wireguardPort = mkOption { - type = port; - default = 51820; - description = "port used for wireguard traffic (UDP); must be the same accross cluster"; - }; - - overlayNet = mkOption { - type = str; - default = "10.0.0.0/8"; - description = "the network in which to allocate addresses for the overlay mesh network (CIDR format); smaller networks increase the chance of IP collision"; - }; - - interface = mkOption { - type = str; - default = "wgoverlay"; - description = "name of the wireguard interface to create and manage"; - }; - - logLevel = mkOption { - type = str; - default = "warn"; - description = "set the verbosity (one of debug/info/warn/error)"; - }; - - }; - }; - - config = mkIf cfg.enable (let binWesher = cfg.package + "/bin/wesher"; - in { - system.activationScripts.wesher = if (cfg.clusterKey != null) then '' - if [ ! -e ${keysPath} ] - then - mkdir --mode=700 -p ${builtins.dirOf keysPath} - echo "WESHER_CLUSTER_KEY=${cfg.clusterKey}" > ${keysPath} - fi - '' else '' - if [ ! -e ${keysPath} ] - then - mkdir --mode=700 -p ${builtins.dirOf keysPath} - echo "WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64)" > ${keysPath} - fi - ''; - - systemd.services.wesher = { - description = "wesher wireguard overlay mesh network manager"; - bindsTo = [ "network-online.target" ]; - after = [ "network-online.target" ]; - wantedBy = [ "multi-user.target" ]; - - environment = { - WESHER_JOIN = builtins.concatStringsSep "," cfg.join; - WESHER_CLUSTER_PORT = builtins.toString cfg.clusterPort; - WESHER_WIREGUARD_PORT = builtins.toString cfg.wireguardPort; - WESHER_OVERLAY_NET = cfg.overlayNet; - WESHER_INTERFACE = cfg.interface; - WESHER_LOG_LEVEL = cfg.logLevel; - WESHER_NO_ETC_HOSTS = "true"; - } - // (if (cfg.bindAddr != null) then { WESHER_BIND_ADDR = cfg.bindAddr; } else {}) - // (if (cfg.bindIface != null) then { WESHER_BIND_IFACE = cfg.bindIface; } else {}) - ; - - serviceConfig = { - ExecStart = "${binWesher}"; - Restart = "always"; - - EnvironmentFile = keysPath; - - User = "wesher"; - DynamicUser = true; - StateDirectory = "wesher"; - - AmbientCapabilities = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE"; - CapabilityBoundingSet = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE"; - MemoryDenyWriteExecute = true; - ProtectControlGroups = true; - ProtectKernelModules = true; - ProtectKernelTunables = true; - RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6 AF_NETLINK"; - RestrictNamespaces = true; - RestrictRealtime = true; - SystemCallArchitectures = "native"; - SystemCallFilter = "~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @resources"; - }; - }; - - networking.firewall.allowedUDPPorts = mkIf cfg.enable [ cfg.clusterPort cfg.wireguardPort ]; - networking.firewall.allowedTCPPorts = mkIf cfg.enable [ cfg.clusterPort ]; - }); -} |