aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlex Auvolat <alex@adnab.me>2022-08-23 23:55:15 +0200
committerAlex Auvolat <alex@adnab.me>2022-08-23 23:55:15 +0200
commitab901fc81d483f24334680a53cacd6b981bb37a7 (patch)
treef104754d49f5c4f09eea34aef3945c9564c1f4df
parenta7ac31cdf52b7042cef86f4e04cde96cbc36711f (diff)
downloadnixcfg-ab901fc81d483f24334680a53cacd6b981bb37a7.tar.gz
nixcfg-ab901fc81d483f24334680a53cacd6b981bb37a7.zip
Remove wesher, reconfigure staging without it
-rw-r--r--cluster/staging/cluster.nix45
-rwxr-xr-xdeploy_nixos6
-rwxr-xr-xdeploy_wesher_key4
-rwxr-xr-xgen_wesher_key17
-rw-r--r--nix/configuration.nix7
-rw-r--r--nix/deuxfleurs.nix97
-rw-r--r--nix/wesher.nix25
-rw-r--r--nix/wesher_service.nix137
8 files changed, 106 insertions, 232 deletions
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix
index 0836dcb..d71b274 100644
--- a/cluster/staging/cluster.nix
+++ b/cluster/staging/cluster.nix
@@ -3,16 +3,36 @@
{
deuxfleurs.cluster_name = "staging";
- # Bootstrap nodes for Wesher overlay network
- services.wesher.join = [
- "192.168.1.21" # cariacou
- "192.168.1.22" # carcajou
- "192.168.1.23" # caribou
- ];
+ # The IP range to use for the Wireguard overlay of this cluster
+ deuxfleurs.cluster_prefix = "10.14.0.0";
+ deuxfleurs.cluster_prefix_length = 16;
- # The IP range to use for the Wesher overlay of this cluster
- deuxfleurs.wesher_cluster_prefix = "10.14.0.0";
- deuxfleurs.wesher_cluster_prefix_length = 16;
+ deuxfleurs.cluster_nodes = [
+ {
+ hostname = "cariacou";
+ site_name = "neptune";
+ publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
+ IP = "10.14.179.56";
+ lan_endpoint = "192.168.1.21:33799";
+ endpoint = "192.168.1.21:33799"; ## TODO nat
+ }
+ {
+ hostname = "carcajou";
+ site_name = "neptune";
+ publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
+ IP = "10.14.252.121";
+ lan_endpoint = "192.168.1.22:33799";
+ endpoint = "192.168.1.22:33799"; ## TODO nat
+ }
+ {
+ hostname = "caribou";
+ site_name = "neptune";
+ publicKey = "X1OlfgjZDdJw3yTg0ytGFlJZXzqT0yHJmYP15R9IpD8=";
+ IP = "10.14.181.82";
+ lan_endpoint = "192.168.1.23:33799";
+ endpoint = "192.168.1.23:33799"; ## TODO nat
+ }
+ ];
# Bootstrap IPs for Consul cluster,
# these are IPs on the Wesher overlay
@@ -22,13 +42,6 @@
"10.14.252.121" # carcajou
];
- # Add wesher hosts to /etc/hosts, needed for elasticsearch
- networking.extraHosts = ''
-10.14.181.82 caribou
-10.14.179.56 cariacou
-10.14.242.121 carcajou
- '';
-
deuxfleurs.admin_accounts = {
lx = [
# Keys for accessing nodes from outside
diff --git a/deploy_nixos b/deploy_nixos
index 30691e0..f62843d 100755
--- a/deploy_nixos
+++ b/deploy_nixos
@@ -3,10 +3,12 @@
copy nix/configuration.nix /etc/nixos/configuration.nix
copy nix/deuxfleurs.nix /etc/nixos/deuxfleurs.nix
copy nix/remote-unlock.nix /etc/nixos/remote-unlock.nix
-copy nix/wesher.nix /etc/nixos/wesher.nix
-copy nix/wesher_service.nix /etc/nixos/wesher_service.nix
copy cluster/$CLUSTER/cluster.nix /etc/nixos/cluster.nix
copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix
copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix
+cmd 'mkdir -p /var/lib/deuxfleurs/wireguard-keys'
+cmd 'test -f /var/lib/deuxfleurs/wireguard-keys/private || (wg genkey > /var/lib/deuxfleurs/wireguard-keys/private; chmod 600 /var/lib/deuxfleurs/wireguard-keys/private)'
+cmd 'echo "Public key: $(wg pubkey < /var/lib/deuxfleurs/wireguard-keys/private)"'
+
cmd nixos-rebuild switch --show-trace
diff --git a/deploy_wesher_key b/deploy_wesher_key
deleted file mode 100755
index 8f7ed77..0000000
--- a/deploy_wesher_key
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env ./sshtool
-
-write_pass deuxfleurs/cluster/$CLUSTER/wesher_key /var/lib/wesher/secrets
-cmd systemctl restart wesher
diff --git a/gen_wesher_key b/gen_wesher_key
deleted file mode 100755
index c66fade..0000000
--- a/gen_wesher_key
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env sh
-
-cd $(dirname $0)
-
-CLUSTER="$1"
-if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
- echo "Usage: $0 <cluster name>"
- echo "The cluster name must be the name of a subdirectory of cluster/"
- exit 1
-fi
-
-K=deuxfleurs/cluster/$CLUSTER/wesher_key
-if ! pass $K >/dev/null; then
- pass insert -m $K <<EOF
-WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64)
-EOF
-fi
diff --git a/nix/configuration.nix b/nix/configuration.nix
index 53246e3..26619d3 100644
--- a/nix/configuration.nix
+++ b/nix/configuration.nix
@@ -11,8 +11,6 @@
./hardware-configuration.nix
# Include generic Deuxfleurs module
./deuxfleurs.nix
- # Wesher module
- ./wesher_service.nix
# Configuration for this deployment (a cluster)
./cluster.nix
# Configuration local for this Deuxfleurs site (set of nodes)
@@ -21,10 +19,6 @@
./node.nix
];
- nixpkgs.overlays = [
- (import ./wesher.nix)
- ];
-
# The global useDHCP flag is deprecated, therefore explicitly set to false here.
# Per-interface useDHCP will be mandatory in the future, so this generated config
# replicates the default behaviour.
@@ -70,7 +64,6 @@ SystemMaxUse=1G
docker
docker-compose
wireguard-tools
- wesher
#qemu
#qemu_kvm
];
diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix
index 9546f5d..8be16af 100644
--- a/nix/deuxfleurs.nix
+++ b/nix/deuxfleurs.nix
@@ -7,6 +7,37 @@ in
with pkgs.lib;
{
options.deuxfleurs =
+ let wg_node = with types; submodule {
+ options = {
+ hostname = mkOption {
+ type = str;
+ description = "Host name";
+ };
+ site_name = mkOption {
+ type = nullOr str;
+ description = "Site where the node is located";
+ default = null;
+ };
+ IP = mkOption {
+ type = str;
+ description = "IP Address in the Wireguard network";
+ };
+ publicKey = mkOption {
+ type = str;
+ description = "Public key";
+ };
+ endpoint = mkOption {
+ type = nullOr str;
+ description = "Wireguard endpoint on the public Internet";
+ };
+ lan_endpoint = mkOption {
+ type = nullOr str;
+ description = "Wireguard endpoint for nodes in the same site";
+ default = null;
+ };
+ };
+ };
+ in
{
# Parameters for individual nodes
network_interface = mkOption {
@@ -30,27 +61,22 @@ in
type = types.int;
};
- wesher_cluster_prefix = mkOption {
- description = "IP address prefix for the Wesher overlay network";
- type = types.str;
- };
- wesher_cluster_prefix_length = mkOption {
- description = "IP address prefix length for the Wesher overlay network";
- type = types.int;
- default = 16;
- };
-
cluster_ip = mkOption {
description = "IP address of this node on the Wesher mesh network";
type = types.str;
};
+ wireguard_port = mkOption {
+ description = "Port for incoming Wireguard VPN connections";
+ type = types.port;
+ default = 33799;
+ };
+
is_raft_server = mkOption {
description = "Make this node a RAFT server for the Nomad and Consul deployments";
type = types.bool;
default = false;
};
-
# Parameters that generally vary between sites
lan_default_gateway = mkOption {
description = "IP address of the default route on the locak network interface";
@@ -70,6 +96,19 @@ in
description = "Name of this Deuxfleurs deployment";
type = types.str;
};
+ cluster_prefix = mkOption {
+ description = "IP address prefix for the Wireguard overlay network";
+ type = types.str;
+ };
+ cluster_prefix_length = mkOption {
+ description = "IP address prefix length for the Wireguard overlay network";
+ type = types.int;
+ default = 16;
+ };
+ cluster_nodes = mkOption {
+ description = "Nodes that are part of the cluster";
+ type = types.listOf wg_node;
+ };
admin_accounts = mkOption {
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
type = types.attrsOf (types.listOf types.str);
@@ -116,19 +155,24 @@ in
cfg.lan_ip
] ++ cfg.nameservers;
- # wesher overlay network
- services.wesher = {
- enable = true;
- bindIface = cfg.network_interface;
- overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
- interface = "wg0";
- logLevel = "debug";
+ # Configure Wireguard VPN between all nodes
+ networking.wireguard.interfaces.wg0 = {
+ ips = [ "${cfg.cluster_ip}/16" ];
+ listenPort = cfg.wireguard_port;
+ privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
+ peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
+ publicKey = publicKey;
+ allowedIPs = [ "${IP}/32" ];
+ endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
+ then lan_endpoint else endpoint;
+ persistentKeepalive = 25;
+ }) cfg.cluster_nodes;
};
# Configure /etc/hosts to link all hostnames to their Wireguard IP
- #networking.extraHosts = builtins.concatStringsSep "\n" (map
- # ({ hostname, IP, ...}: "${IP} ${hostname}")
- # (cfg.cluster_nodes ++ cfg.admin_nodes));
+ networking.extraHosts = builtins.concatStringsSep "\n" (map
+ ({ hostname, IP, ...}: "${IP} ${hostname}")
+ cfg.cluster_nodes);
# Enable Hashicorp Consul & Nomad
services.consul.enable = true;
@@ -245,11 +289,16 @@ in
networking.firewall = {
enable = true;
- # Allow anyone to connect on SSH port
allowedTCPPorts = [
+ # Allow anyone to connect on SSH port
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
];
+ allowedUDPPorts = [
+ # Allow peers to connect to Wireguard
+ cfg.wireguard_port
+ ];
+
# Allow specific hosts access to specific things in the cluster
extraCommands = ''
# Allow everything from router (usefull for UPnP/IGD)
@@ -259,14 +308,14 @@ in
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
# Allow other nodes on VPN to access all ports
- iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+ iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
'';
# When stopping firewall, delete all rules that were configured manually above
extraStopCommands = ''
iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
- iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+ iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
'';
};
};
diff --git a/nix/wesher.nix b/nix/wesher.nix
deleted file mode 100644
index c39a3e6..0000000
--- a/nix/wesher.nix
+++ /dev/null
@@ -1,25 +0,0 @@
-self: super:
-{
- wesher = super.buildGoModule rec {
- pname = "wesher";
- version = "0.2.6";
-
- src = super.fetchFromGitHub {
- owner = "costela";
- repo = "wesher";
- rev = "v${version}";
- sha256 = "1iagmnw2yf15r0fpikk610w0lm0gcxw83lcwfjyr2jv1q2ys71hh";
- };
-
- vendorSha256 = "0nyg0wzn8d4rzjs8yrxxj3gha94043ll80s1ql0fml025q2f3705";
-
- checkPhase = "true";
-
- meta = with super.lib; {
- description = "wireguard overlay mesh network manager";
- homepage = "https://github.com/costela/wesher";
- license = licenses.gpl3Plus;
- platforms = platforms.linux;
- };
- };
-}
diff --git a/nix/wesher_service.nix b/nix/wesher_service.nix
deleted file mode 100644
index d269a2f..0000000
--- a/nix/wesher_service.nix
+++ /dev/null
@@ -1,137 +0,0 @@
-{ config, lib, pkgs, ... }:
-with lib;
-let
- keysPath = "/var/lib/wesher/secrets";
- cfg = config.services.wesher;
-in {
- options = with types; {
- services.wesher = {
- enable = mkEnableOption "wesher wireguard overlay mesh network manager";
-
- package = mkOption {
- type = package;
- default = pkgs.wesher;
- defaultText = literalExpression "pkgs.wesher";
- description = "Wesher package to use.";
- };
-
- clusterKey = mkOption {
- type = nullOr str;
- default = null;
- description = "shared key for cluster membership to use on first initialization, if no key was previously used by Wesher. Must be 32 bytes base64 encoded; will be generated if not provided. Setting this parameter value will not overwrite an existing cluster key; to do so please delete ${keysPath}";
- };
-
- bindAddr = mkOption {
- type = nullOr str;
- default = null;
- description = "IP address to bind to for cluster membership (cannot be used with --bind-iface)";
- };
-
- bindIface = mkOption {
- type = nullOr str;
- default = null;
- description = "Interface to bind to for cluster membership (cannot be used with --bind-addr)";
- };
-
- join = mkOption {
- type = listOf str;
- default = [];
- description = "list of hostnames or IP addresses to existing cluster members; if not provided, will attempt resuming any known state or otherwise wait for further members";
- };
-
- clusterPort = mkOption {
- type = port;
- default = 7946;
- description = "port used for membership gossip traffic (both TCP and UDP); must be the same accross cluster";
- };
-
- wireguardPort = mkOption {
- type = port;
- default = 51820;
- description = "port used for wireguard traffic (UDP); must be the same accross cluster";
- };
-
- overlayNet = mkOption {
- type = str;
- default = "10.0.0.0/8";
- description = "the network in which to allocate addresses for the overlay mesh network (CIDR format); smaller networks increase the chance of IP collision";
- };
-
- interface = mkOption {
- type = str;
- default = "wgoverlay";
- description = "name of the wireguard interface to create and manage";
- };
-
- logLevel = mkOption {
- type = str;
- default = "warn";
- description = "set the verbosity (one of debug/info/warn/error)";
- };
-
- };
- };
-
- config = mkIf cfg.enable (let binWesher = cfg.package + "/bin/wesher";
- in {
- system.activationScripts.wesher = if (cfg.clusterKey != null) then ''
- if [ ! -e ${keysPath} ]
- then
- mkdir --mode=700 -p ${builtins.dirOf keysPath}
- echo "WESHER_CLUSTER_KEY=${cfg.clusterKey}" > ${keysPath}
- fi
- '' else ''
- if [ ! -e ${keysPath} ]
- then
- mkdir --mode=700 -p ${builtins.dirOf keysPath}
- echo "WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64)" > ${keysPath}
- fi
- '';
-
- systemd.services.wesher = {
- description = "wesher wireguard overlay mesh network manager";
- bindsTo = [ "network-online.target" ];
- after = [ "network-online.target" ];
- wantedBy = [ "multi-user.target" ];
-
- environment = {
- WESHER_JOIN = builtins.concatStringsSep "," cfg.join;
- WESHER_CLUSTER_PORT = builtins.toString cfg.clusterPort;
- WESHER_WIREGUARD_PORT = builtins.toString cfg.wireguardPort;
- WESHER_OVERLAY_NET = cfg.overlayNet;
- WESHER_INTERFACE = cfg.interface;
- WESHER_LOG_LEVEL = cfg.logLevel;
- WESHER_NO_ETC_HOSTS = "true";
- }
- // (if (cfg.bindAddr != null) then { WESHER_BIND_ADDR = cfg.bindAddr; } else {})
- // (if (cfg.bindIface != null) then { WESHER_BIND_IFACE = cfg.bindIface; } else {})
- ;
-
- serviceConfig = {
- ExecStart = "${binWesher}";
- Restart = "always";
-
- EnvironmentFile = keysPath;
-
- User = "wesher";
- DynamicUser = true;
- StateDirectory = "wesher";
-
- AmbientCapabilities = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE";
- CapabilityBoundingSet = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE";
- MemoryDenyWriteExecute = true;
- ProtectControlGroups = true;
- ProtectKernelModules = true;
- ProtectKernelTunables = true;
- RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6 AF_NETLINK";
- RestrictNamespaces = true;
- RestrictRealtime = true;
- SystemCallArchitectures = "native";
- SystemCallFilter = "~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @resources";
- };
- };
-
- networking.firewall.allowedUDPPorts = mkIf cfg.enable [ cfg.clusterPort cfg.wireguardPort ];
- networking.firewall.allowedTCPPorts = mkIf cfg.enable [ cfg.clusterPort ];
- });
-}