aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cluster/staging/cluster.nix75
-rw-r--r--cluster/staging/node/carcajou.nix11
-rw-r--r--cluster/staging/node/cariacou.nix11
-rw-r--r--cluster/staging/node/caribou.nix11
-rw-r--r--cluster/staging/site/neptune.nix1
-rw-r--r--nix/configuration.nix38
-rw-r--r--nix/deuxfleurs.nix137
7 files changed, 96 insertions, 188 deletions
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix
index 79fb3fb..5007815 100644
--- a/cluster/staging/cluster.nix
+++ b/cluster/staging/cluster.nix
@@ -2,65 +2,26 @@
{
deuxfleurs.cluster_name = "staging";
- deuxfleurs.cluster_nodes = [
- {
- hostname = "spoutnik";
- site_name = "pluton";
- publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg=";
- IP = "10.42.2.2";
- endpoint = "77.141.67.109:42136";
- }
- {
- hostname = "cariacou";
- site_name = "neptune";
- publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
- IP = "10.42.2.21";
- endpoint = "82.66.112.151:33721";
- lan_endpoint = "192.168.1.21:33721";
- }
- {
- hostname = "carcajou";
- site_name = "neptune";
- publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
- IP = "10.42.2.22";
- endpoint = "82.66.112.151:33722";
- lan_endpoint = "192.168.1.22:33722";
- }
- {
- hostname = "caribou";
- site_name = "neptune";
- publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY=";
- IP = "10.42.2.23";
- endpoint = "82.66.112.151:33723";
- lan_endpoint = "192.168.1.23:33723";
- }
+
+ # Bootstrap nodes for Wesher overlay network
+ services.wesher.join = [
+ "2a01:e0a:c:a720::21" # cariacou
+ "2a01:e0a:c:a720::22" # carcajou
+ "2a01:e0a:c:a720::23" # caribou
];
- deuxfleurs.admin_nodes = [
- {
- hostname = "hammerhead";
- publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic=";
- IP = "10.42.0.1";
- endpoint = "5.135.179.11:51349";
- }
- {
- hostname = "robinson";
- publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg=";
- IP = "10.42.0.42";
- endpoint = "77.141.67.109:33742";
- }
- {
- hostname = "shiki";
- publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg=";
- IP = "10.42.0.206";
- endpoint = "37.187.118.206:51820";
- }
- {
- hostname = "lindy";
- publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks=";
- IP = "10.42.0.66";
- endpoint = "82.66.112.151:33766";
- }
+
+ # The IP range to use for the Wesher overlay of this cluster
+ deuxfleurs.wesher_cluster_prefix = "10.14.0.0";
+ deuxfleurs.wesher_cluster_prefix_length = 16;
+
+ # Bootstrap IPs for Consul cluster,
+ # these are IPs on the Wesher overlay
+ services.consul.extraConfig.retry_join = [
+ "10.14.181.82" # caribou
+ "10.14.179.56" # cariacou
+ "10.14.252.121" # carcajou
];
+
deuxfleurs.admin_accounts = {
lx = [
# Keys for accessing nodes from outside
diff --git a/cluster/staging/node/carcajou.nix b/cluster/staging/node/carcajou.nix
index 304f358..dbcc5ec 100644
--- a/cluster/staging/node/carcajou.nix
+++ b/cluster/staging/node/carcajou.nix
@@ -17,16 +17,9 @@
deuxfleurs.network_interface = "eno1";
deuxfleurs.lan_ip = "192.168.1.22";
+ deuxfleurs.ipv6 = "2a01:e0a:c:a720::22";
- networking.interfaces.eno1.ipv6.addresses = [
- {
- address = "2a01:e0a:c:a720::22";
- prefixLength = 64;
- }
- ];
-
- deuxfleurs.vpn_ip = "10.42.2.22";
- deuxfleurs.vpn_listen_port = 33722;
+ deuxfleurs.cluster_ip = "10.14.252.121";
deuxfleurs.is_raft_server = true;
# Enable netdata monitoring
diff --git a/cluster/staging/node/cariacou.nix b/cluster/staging/node/cariacou.nix
index 5f57fbd..14d1842 100644
--- a/cluster/staging/node/cariacou.nix
+++ b/cluster/staging/node/cariacou.nix
@@ -17,16 +17,9 @@
deuxfleurs.network_interface = "eno1";
deuxfleurs.lan_ip = "192.168.1.21";
+ deuxfleurs.ipv6 = "2a01:e0a:c:a720::21";
- networking.interfaces.eno1.ipv6.addresses = [
- {
- address = "2a01:e0a:c:a720::21";
- prefixLength = 64;
- }
- ];
-
- deuxfleurs.vpn_ip = "10.42.2.21";
- deuxfleurs.vpn_listen_port = 33721;
+ deuxfleurs.cluster_ip = "10.14.179.56";
deuxfleurs.is_raft_server = true;
# Enable netdata monitoring
diff --git a/cluster/staging/node/caribou.nix b/cluster/staging/node/caribou.nix
index 7b785ef..3b41972 100644
--- a/cluster/staging/node/caribou.nix
+++ b/cluster/staging/node/caribou.nix
@@ -17,16 +17,9 @@
deuxfleurs.network_interface = "eno1";
deuxfleurs.lan_ip = "192.168.1.23";
+ deuxfleurs.ipv6 = "2a01:e0a:c:a720::23";
- networking.interfaces.eno1.ipv6.addresses = [
- {
- address = "2a01:e0a:c:a720::23";
- prefixLength = 64;
- }
- ];
-
- deuxfleurs.vpn_ip = "10.42.2.23";
- deuxfleurs.vpn_listen_port = 33723;
+ deuxfleurs.cluster_ip = "10.14.181.82";
deuxfleurs.is_raft_server = true;
# Enable netdata monitoring
diff --git a/cluster/staging/site/neptune.nix b/cluster/staging/site/neptune.nix
index 1b80144..38a4bab 100644
--- a/cluster/staging/site/neptune.nix
+++ b/cluster/staging/site/neptune.nix
@@ -4,6 +4,7 @@
deuxfleurs.site_name = "neptune";
deuxfleurs.lan_default_gateway = "192.168.1.254";
deuxfleurs.lan_ip_prefix_length = 24;
+ deuxfleurs.ipv6_prefix_length = 64;
networking.nameservers = [ "192.168.1.254" ];
diff --git a/nix/configuration.nix b/nix/configuration.nix
index 3f3aa49..984307c 100644
--- a/nix/configuration.nix
+++ b/nix/configuration.nix
@@ -69,6 +69,7 @@ SystemMaxUse=1G
rclone
docker
docker-compose
+ wireguard
wesher
];
@@ -81,43 +82,6 @@ SystemMaxUse=1G
services.openssh.enable = true;
services.openssh.passwordAuthentication = false;
- services.wesher = {
- enable = true;
- join = [ "192.168.1.22" "192.168.1.23" ];
- bindAddr = config.deuxfleurs.lan_ip; # for now
- overlayNet = "10.14.0.0/16";
- };
-
- # ---- CONFIG FOR DEUXFLEURS CLUSTER ----
-
- # Open ports in the firewall.
- networking.firewall = {
- enable = true;
-
- # Allow anyone to connect on SSH port
- allowedTCPPorts = [
- (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
- ];
-
- # Allow specific hosts access to specific things in the cluster
- extraCommands = ''
- # Allow everything from router (usefull for UPnP/IGD)
- iptables -A INPUT -s 192.168.1.254 -j ACCEPT
-
- # Allow docker containers to access all ports
- iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
-
- # Allow other nodes on VPN to access all ports
- iptables -A INPUT -s 10.42.0.0/16 -j ACCEPT
- '';
-
- # When stopping firewall, delete all rules that were configured manually above
- extraStopCommands = ''
- iptables -D INPUT -s 192.168.1.254 -j ACCEPT
- iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
- iptables -D INPUT -s 10.42.0.0/16 -j ACCEPT
- '';
- };
# This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions
diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix
index 7772f0e..2050776 100644
--- a/nix/deuxfleurs.nix
+++ b/nix/deuxfleurs.nix
@@ -7,37 +7,6 @@ in
with pkgs.lib;
{
options.deuxfleurs =
- let wg_node = with types; submodule {
- options = {
- hostname = mkOption {
- type = str;
- description = "Host name";
- };
- site_name = mkOption {
- type = nullOr str;
- description = "Site where the node is located";
- default = null;
- };
- IP = mkOption {
- type = str;
- description = "IP Address";
- };
- publicKey = mkOption {
- type = str;
- description = "Public key";
- };
- endpoint = mkOption {
- type = nullOr str;
- description = "Wireguard endpoint on the public Internet";
- };
- lan_endpoint = mkOption {
- type = nullOr str;
- description = "Wireguard endpoint for nodes in the same site";
- default = null;
- };
- };
- };
- in
{
# Parameters for individual nodes
network_interface = mkOption {
@@ -52,14 +21,28 @@ in
description = "Prefix length associated with lan_ip";
type = types.int;
};
+ ipv6 = mkOption {
+ description = "Public IPv6 address of this node";
+ type = types.str;
+ };
+ ipv6_prefix_length = mkOption {
+ description = "Prefix length associated with ipv6 ip";
+ type = types.int;
+ };
- vpn_ip = mkOption {
- description = "IP address of this node on the Wireguard VPN";
+ wesher_cluster_prefix = mkOption {
+ description = "IP address prefix for the Wesher overlay network";
type = types.str;
};
- vpn_listen_port = mkOption {
- description = "Port for incoming Wireguard VPN connections";
- type = types.port;
+ wesher_cluster_prefix_length = mkOption {
+ description = "IP address prefix length for the Wesher overlay network";
+ type = types.int;
+ default = 16;
+ };
+
+ cluster_ip = mkOption {
+ description = "IP address of this node on the Wesher mesh network";
+ type = types.str;
};
is_raft_server = mkOption {
description = "Make this node a RAFT server for the Nomad and Consul deployments";
@@ -83,14 +66,6 @@ in
description = "Name of this Deuxfleurs deployment";
type = types.str;
};
- cluster_nodes = mkOption {
- description = "Nodes that are part of the cluster";
- type = types.listOf wg_node;
- };
- admin_nodes = mkOption {
- description = "Machines that are part of the Wireguard VPN for administration purposes";
- type = types.listOf wg_node;
- };
admin_accounts = mkOption {
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
type = types.attrsOf (types.listOf types.str);
@@ -114,32 +89,31 @@ in
prefixLength = cfg.lan_ip_prefix_length;
}
];
+ ipv6.addresses = [
+ {
+ address = cfg.ipv6;
+ prefixLength = cfg.ipv6_prefix_length;
+ }
+ ];
};
networking.defaultGateway = {
address = cfg.lan_default_gateway;
interface = cfg.network_interface;
};
- # Configure Wireguard VPN between all nodes
- networking.wireguard.interfaces.wg0 = {
- ips = [ "${cfg.vpn_ip}/16" ];
- listenPort = cfg.vpn_listen_port;
- privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
- peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
- publicKey = publicKey;
- allowedIPs = [ "${IP}/32" ];
- endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
- then lan_endpoint else endpoint;
- persistentKeepalive = 25;
- }) (cfg.cluster_nodes ++ cfg.admin_nodes);
+ # wesher overlay network
+ services.wesher = {
+ enable = true;
+ bindAddr = cfg.ipv6;
+ overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
+ interface = "wg0";
+ logLevel = "debug";
};
- networking.firewall.allowedUDPPorts = [ cfg.vpn_listen_port ];
-
# Configure /etc/hosts to link all hostnames to their Wireguard IP
- networking.extraHosts = builtins.concatStringsSep "\n" (map
- ({ hostname, IP, ...}: "${IP} ${hostname}")
- (cfg.cluster_nodes ++ cfg.admin_nodes));
+ #networking.extraHosts = builtins.concatStringsSep "\n" (map
+ # ({ hostname, IP, ...}: "${IP} ${hostname}")
+ # (cfg.cluster_nodes ++ cfg.admin_nodes));
# Enable Hashicorp Consul & Nomad
services.consul.enable = true;
@@ -156,14 +130,12 @@ in
"site" = cfg.site_name;
};
ui = true;
- bind_addr = cfg.vpn_ip;
+ bind_addr = "${cfg.cluster_ip}";
ports.http = -1;
addresses.https = "0.0.0.0";
ports.https = 8501;
- retry_join = map (node_info: node_info.IP) cfg.cluster_nodes;
-
ca_file = "/var/lib/consul/pki/consul-ca.crt";
cert_file = "/var/lib/consul/pki/consul2022.crt";
key_file = "/var/lib/consul/pki/consul2022.key";
@@ -185,9 +157,9 @@ in
region = cfg.cluster_name;
datacenter = cfg.site_name;
advertise = {
- rpc = cfg.vpn_ip;
- http = cfg.vpn_ip;
- serf = cfg.vpn_ip;
+ rpc = "${cfg.cluster_ip}";
+ http = "${cfg.cluster_ip}";
+ serf = "${cfg.cluster_ip}";
};
consul = {
address = "localhost:8501";
@@ -227,5 +199,36 @@ in
}
];
};
+
+ # ---- Firewall config ----
+
+ # Open ports in the firewall.
+ networking.firewall = {
+ enable = true;
+
+ # Allow anyone to connect on SSH port
+ allowedTCPPorts = [
+ (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
+ ];
+
+ # Allow specific hosts access to specific things in the cluster
+ extraCommands = ''
+ # Allow everything from router (usefull for UPnP/IGD)
+ iptables -A INPUT -s 192.168.1.254 -j ACCEPT
+
+ # Allow docker containers to access all ports
+ iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
+
+ # Allow other nodes on VPN to access all ports
+ iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+ '';
+
+ # When stopping firewall, delete all rules that were configured manually above
+ extraStopCommands = ''
+ iptables -D INPUT -s 192.168.1.254 -j ACCEPT
+ iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
+ iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+ '';
+ };
};
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-10 00:05:26 +0000