diff options
Diffstat (limited to 'os/modules')
-rw-r--r-- | os/modules/deuxfleurs.nix | 234 | ||||
-rw-r--r-- | os/modules/remote-unlock.nix | 26 | ||||
-rw-r--r-- | os/modules/wesher_service.nix | 137 |
3 files changed, 397 insertions, 0 deletions
diff --git a/os/modules/deuxfleurs.nix b/os/modules/deuxfleurs.nix new file mode 100644 index 0000000..2050776 --- /dev/null +++ b/os/modules/deuxfleurs.nix @@ -0,0 +1,234 @@ +{ config, pkgs, ... }: + +let + cfg = config.deuxfleurs; +in + with builtins; + with pkgs.lib; +{ + options.deuxfleurs = + { + # Parameters for individual nodes + network_interface = mkOption { + description = "Network interface name to configure"; + type = types.str; + }; + lan_ip = mkOption { + description = "IP address of this node on the local network interface"; + type = types.str; + }; + lan_ip_prefix_length = mkOption { + description = "Prefix length associated with lan_ip"; + type = types.int; + }; + ipv6 = mkOption { + description = "Public IPv6 address of this node"; + type = types.str; + }; + ipv6_prefix_length = mkOption { + description = "Prefix length associated with ipv6 ip"; + type = types.int; + }; + + wesher_cluster_prefix = mkOption { + description = "IP address prefix for the Wesher overlay network"; + type = types.str; + }; + wesher_cluster_prefix_length = mkOption { + description = "IP address prefix length for the Wesher overlay network"; + type = types.int; + default = 16; + }; + + cluster_ip = mkOption { + description = "IP address of this node on the Wesher mesh network"; + type = types.str; + }; + is_raft_server = mkOption { + description = "Make this node a RAFT server for the Nomad and Consul deployments"; + type = types.bool; + default = false; + }; + + + # Parameters that generally vary between sites + lan_default_gateway = mkOption { + description = "IP address of the default route on the locak network interface"; + type = types.str; + }; + site_name = mkOption { + description = "Site (availability zone) on which this node is deployed"; + type = types.str; + }; + + # Parameters common to all nodes + cluster_name = mkOption { + description = "Name of this Deuxfleurs deployment"; + type = types.str; + }; + admin_accounts = mkOption { + description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys"; + type = types.attrsOf (types.listOf types.str); + }; + }; + + config = { + # Configure admin accounts on all nodes + users.users = builtins.mapAttrs (name: publicKeys: { + isNormalUser = true; + extraGroups = [ "wheel" ]; + openssh.authorizedKeys.keys = publicKeys; + }) cfg.admin_accounts; + + # Configure network interfaces + networking.interfaces = attrsets.setAttrByPath [ cfg.network_interface ] { + useDHCP = false; + ipv4.addresses = [ + { + address = cfg.lan_ip; + prefixLength = cfg.lan_ip_prefix_length; + } + ]; + ipv6.addresses = [ + { + address = cfg.ipv6; + prefixLength = cfg.ipv6_prefix_length; + } + ]; + }; + networking.defaultGateway = { + address = cfg.lan_default_gateway; + interface = cfg.network_interface; + }; + + # wesher overlay network + services.wesher = { + enable = true; + bindAddr = cfg.ipv6; + overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}"; + interface = "wg0"; + logLevel = "debug"; + }; + + # Configure /etc/hosts to link all hostnames to their Wireguard IP + #networking.extraHosts = builtins.concatStringsSep "\n" (map + # ({ hostname, IP, ...}: "${IP} ${hostname}") + # (cfg.cluster_nodes ++ cfg.admin_nodes)); + + # Enable Hashicorp Consul & Nomad + services.consul.enable = true; + services.consul.extraConfig = + (if cfg.is_raft_server + then { + server = true; + bootstrap_expect = 3; + } + else {}) // + { + datacenter = cfg.cluster_name; + node_meta = { + "site" = cfg.site_name; + }; + ui = true; + bind_addr = "${cfg.cluster_ip}"; + + ports.http = -1; + addresses.https = "0.0.0.0"; + ports.https = 8501; + + ca_file = "/var/lib/consul/pki/consul-ca.crt"; + cert_file = "/var/lib/consul/pki/consul2022.crt"; + key_file = "/var/lib/consul/pki/consul2022.key"; + verify_incoming = true; + verify_outgoing = true; + verify_server_hostname = true; + }; + + services.nomad.enable = true; + services.nomad.package = pkgs.nomad_1_1; + services.nomad.settings = + (if cfg.is_raft_server + then { server = { + enabled = true; + bootstrap_expect = 3; + }; } + else {}) // + { + region = cfg.cluster_name; + datacenter = cfg.site_name; + advertise = { + rpc = "${cfg.cluster_ip}"; + http = "${cfg.cluster_ip}"; + serf = "${cfg.cluster_ip}"; + }; + consul = { + address = "localhost:8501"; + ca_file = "/var/lib/nomad/pki/consul2022.crt"; + cert_file = "/var/lib/nomad/pki/consul2022-client.crt"; + key_file = "/var/lib/nomad/pki/consul2022-client.key"; + ssl = true; + }; + client = { + enabled = true; + network_interface = "wg0"; + meta = { + "site" = cfg.site_name; + }; + }; + tls = { + http = true; + rpc = true; + ca_file = "/var/lib/nomad/pki/nomad-ca.crt"; + cert_file = "/var/lib/nomad/pki/nomad2022.crt"; + key_file = "/var/lib/nomad/pki/nomad2022.key"; + verify_server_hostname = true; + verify_https_client = true; + }; + plugin = [ + { + docker = [ + { + config = [ + { + volumes.enabled = true; + allow_privileged = true; + } + ]; + } + ]; + } + ]; + }; + + # ---- Firewall config ---- + + # Open ports in the firewall. + networking.firewall = { + enable = true; + + # Allow anyone to connect on SSH port + allowedTCPPorts = [ + (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports) + ]; + + # Allow specific hosts access to specific things in the cluster + extraCommands = '' + # Allow everything from router (usefull for UPnP/IGD) + iptables -A INPUT -s 192.168.1.254 -j ACCEPT + + # Allow docker containers to access all ports + iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT + + # Allow other nodes on VPN to access all ports + iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT + ''; + + # When stopping firewall, delete all rules that were configured manually above + extraStopCommands = '' + iptables -D INPUT -s 192.168.1.254 -j ACCEPT + iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT + iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT + ''; + }; + }; +} diff --git a/os/modules/remote-unlock.nix b/os/modules/remote-unlock.nix new file mode 100644 index 0000000..2975a94 --- /dev/null +++ b/os/modules/remote-unlock.nix @@ -0,0 +1,26 @@ +{ config, pkgs, ... }: + + with builtins; + with pkgs.lib; +{ + config = { + boot.initrd.availableKernelModules = [ "pps_core" "ptp" "e1000e" ]; + boot.initrd.network.enable = true; + boot.initrd.network.ssh = { + enable = true; + port = 222; + authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.admin_accounts); + hostKeys = [ "/var/lib/deuxfleurs/remote-unlock/ssh_host_ed25519_key" ]; + }; + boot.initrd.network.postCommands = '' + ip addr add ${config.deuxfleurs.lan_ip}/${toString config.deuxfleurs.lan_ip_prefix_length} dev ${config.deuxfleurs.network_interface} + ip link set dev ${config.deuxfleurs.network_interface} up + ip route add default via ${config.deuxfleurs.lan_default_gateway} dev ${config.deuxfleurs.network_interface} + ip a + ip route + ping -c 4 ${config.deuxfleurs.lan_default_gateway} + echo 'echo run cryptsetup-askpass to unlock drives' >> /root/.profile + ''; + }; +} + diff --git a/os/modules/wesher_service.nix b/os/modules/wesher_service.nix new file mode 100644 index 0000000..d269a2f --- /dev/null +++ b/os/modules/wesher_service.nix @@ -0,0 +1,137 @@ +{ config, lib, pkgs, ... }: +with lib; +let + keysPath = "/var/lib/wesher/secrets"; + cfg = config.services.wesher; +in { + options = with types; { + services.wesher = { + enable = mkEnableOption "wesher wireguard overlay mesh network manager"; + + package = mkOption { + type = package; + default = pkgs.wesher; + defaultText = literalExpression "pkgs.wesher"; + description = "Wesher package to use."; + }; + + clusterKey = mkOption { + type = nullOr str; + default = null; + description = "shared key for cluster membership to use on first initialization, if no key was previously used by Wesher. Must be 32 bytes base64 encoded; will be generated if not provided. Setting this parameter value will not overwrite an existing cluster key; to do so please delete ${keysPath}"; + }; + + bindAddr = mkOption { + type = nullOr str; + default = null; + description = "IP address to bind to for cluster membership (cannot be used with --bind-iface)"; + }; + + bindIface = mkOption { + type = nullOr str; + default = null; + description = "Interface to bind to for cluster membership (cannot be used with --bind-addr)"; + }; + + join = mkOption { + type = listOf str; + default = []; + description = "list of hostnames or IP addresses to existing cluster members; if not provided, will attempt resuming any known state or otherwise wait for further members"; + }; + + clusterPort = mkOption { + type = port; + default = 7946; + description = "port used for membership gossip traffic (both TCP and UDP); must be the same accross cluster"; + }; + + wireguardPort = mkOption { + type = port; + default = 51820; + description = "port used for wireguard traffic (UDP); must be the same accross cluster"; + }; + + overlayNet = mkOption { + type = str; + default = "10.0.0.0/8"; + description = "the network in which to allocate addresses for the overlay mesh network (CIDR format); smaller networks increase the chance of IP collision"; + }; + + interface = mkOption { + type = str; + default = "wgoverlay"; + description = "name of the wireguard interface to create and manage"; + }; + + logLevel = mkOption { + type = str; + default = "warn"; + description = "set the verbosity (one of debug/info/warn/error)"; + }; + + }; + }; + + config = mkIf cfg.enable (let binWesher = cfg.package + "/bin/wesher"; + in { + system.activationScripts.wesher = if (cfg.clusterKey != null) then '' + if [ ! -e ${keysPath} ] + then + mkdir --mode=700 -p ${builtins.dirOf keysPath} + echo "WESHER_CLUSTER_KEY=${cfg.clusterKey}" > ${keysPath} + fi + '' else '' + if [ ! -e ${keysPath} ] + then + mkdir --mode=700 -p ${builtins.dirOf keysPath} + echo "WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64)" > ${keysPath} + fi + ''; + + systemd.services.wesher = { + description = "wesher wireguard overlay mesh network manager"; + bindsTo = [ "network-online.target" ]; + after = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = { + WESHER_JOIN = builtins.concatStringsSep "," cfg.join; + WESHER_CLUSTER_PORT = builtins.toString cfg.clusterPort; + WESHER_WIREGUARD_PORT = builtins.toString cfg.wireguardPort; + WESHER_OVERLAY_NET = cfg.overlayNet; + WESHER_INTERFACE = cfg.interface; + WESHER_LOG_LEVEL = cfg.logLevel; + WESHER_NO_ETC_HOSTS = "true"; + } + // (if (cfg.bindAddr != null) then { WESHER_BIND_ADDR = cfg.bindAddr; } else {}) + // (if (cfg.bindIface != null) then { WESHER_BIND_IFACE = cfg.bindIface; } else {}) + ; + + serviceConfig = { + ExecStart = "${binWesher}"; + Restart = "always"; + + EnvironmentFile = keysPath; + + User = "wesher"; + DynamicUser = true; + StateDirectory = "wesher"; + + AmbientCapabilities = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE"; + CapabilityBoundingSet = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE"; + MemoryDenyWriteExecute = true; + ProtectControlGroups = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6 AF_NETLINK"; + RestrictNamespaces = true; + RestrictRealtime = true; + SystemCallArchitectures = "native"; + SystemCallFilter = "~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @resources"; + }; + }; + + networking.firewall.allowedUDPPorts = mkIf cfg.enable [ cfg.clusterPort cfg.wireguardPort ]; + networking.firewall.allowedTCPPorts = mkIf cfg.enable [ cfg.clusterPort ]; + }); +} |