{ config, pkgs, ... }: let cfg = config.deuxfleurs; in with builtins; with pkgs.lib; { options.deuxfleurs = { # Parameters for individual nodes network_interface = mkOption { description = "Network interface name to configure"; type = types.str; }; lan_ip = mkOption { description = "IP address of this node on the local network interface"; type = types.str; }; lan_ip_prefix_length = mkOption { description = "Prefix length associated with lan_ip"; type = types.int; }; ipv6 = mkOption { description = "Public IPv6 address of this node"; type = types.str; }; ipv6_prefix_length = mkOption { description = "Prefix length associated with ipv6 ip"; type = types.int; }; wesher_cluster_prefix = mkOption { description = "IP address prefix for the Wesher overlay network"; type = types.str; }; wesher_cluster_prefix_length = mkOption { description = "IP address prefix length for the Wesher overlay network"; type = types.int; default = 16; }; cluster_ip = mkOption { description = "IP address of this node on the Wesher mesh network"; type = types.str; }; is_raft_server = mkOption { description = "Make this node a RAFT server for the Nomad and Consul deployments"; type = types.bool; default = false; }; # Parameters that generally vary between sites lan_default_gateway = mkOption { description = "IP address of the default route on the locak network interface"; type = types.str; }; site_name = mkOption { description = "Site (availability zone) on which this node is deployed"; type = types.str; }; nameservers = mkOption { description = "External DNS servers to use"; type = types.listOf types.str; }; # Parameters common to all nodes cluster_name = mkOption { description = "Name of this Deuxfleurs deployment"; type = types.str; }; admin_accounts = mkOption { description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys"; type = types.attrsOf (types.listOf types.str); }; }; config = { # Configure admin accounts on all nodes users.users = builtins.mapAttrs (name: publicKeys: { isNormalUser = true; extraGroups = [ "wheel" ]; openssh.authorizedKeys.keys = publicKeys; }) cfg.admin_accounts; # Configure network interfaces networking.interfaces = let ip4config = { useDHCP = false; ipv4.addresses = [ { address = cfg.lan_ip; prefixLength = cfg.lan_ip_prefix_length; } ]; }; ip6config = { ipv6.addresses = [ { address = cfg.ipv6; prefixLength = cfg.ipv6_prefix_length; } ]; }; in (attrsets.setAttrByPath [ cfg.network_interface ] (ip4config // ip6config)); networking.defaultGateway = { address = cfg.lan_default_gateway; interface = cfg.network_interface; }; networking.nameservers = [ cfg.lan_ip ] ++ cfg.nameservers; # wesher overlay network services.wesher = { enable = true; bindIface = cfg.network_interface; overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}"; interface = "wg0"; logLevel = "debug"; }; # Configure /etc/hosts to link all hostnames to their Wireguard IP #networking.extraHosts = builtins.concatStringsSep "\n" (map # ({ hostname, IP, ...}: "${IP} ${hostname}") # (cfg.cluster_nodes ++ cfg.admin_nodes)); # Enable Hashicorp Consul & Nomad services.consul.enable = true; services.consul.extraConfig = (if cfg.is_raft_server then { server = true; bootstrap_expect = 3; } else {}) // { datacenter = cfg.cluster_name; node_meta = { "site" = cfg.site_name; }; ui_config = { enabled = true; }; bind_addr = "${cfg.cluster_ip}"; addresses = { https = "0.0.0.0"; dns = "0.0.0.0"; }; ports = { http = -1; https = 8501; dns = 53; }; recursors = [ cfg.nameservers ]; ca_file = "/var/lib/consul/pki/consul-ca.crt"; cert_file = "/var/lib/consul/pki/consul2022.crt"; key_file = "/var/lib/consul/pki/consul2022.key"; verify_incoming = true; verify_outgoing = true; verify_server_hostname = true; }; systemd.services.consul.serviceConfig = { AmbientCapabilities = "CAP_NET_BIND_SERVICE"; }; services.nomad.enable = true; services.nomad.package = pkgs.nomad_1_3; services.nomad.extraPackages = [ pkgs.glibc pkgs.zstd #pkgs.qemu #pkgs.qemu_kvm ]; services.nomad.settings = (if cfg.is_raft_server then { server = { enabled = true; bootstrap_expect = 3; }; } else {}) // { region = cfg.cluster_name; datacenter = cfg.site_name; advertise = { rpc = "${cfg.cluster_ip}"; http = "${cfg.cluster_ip}"; serf = "${cfg.cluster_ip}"; }; consul = { address = "localhost:8501"; ca_file = "/var/lib/nomad/pki/consul2022.crt"; cert_file = "/var/lib/nomad/pki/consul2022-client.crt"; key_file = "/var/lib/nomad/pki/consul2022-client.key"; ssl = true; }; client = { enabled = true; network_interface = "wg0"; meta = { "site" = cfg.site_name; }; }; tls = { http = true; rpc = true; ca_file = "/var/lib/nomad/pki/nomad-ca.crt"; cert_file = "/var/lib/nomad/pki/nomad2022.crt"; key_file = "/var/lib/nomad/pki/nomad2022.key"; verify_server_hostname = true; verify_https_client = true; }; plugin = [ { docker = [ { config = [ { volumes.enabled = true; allow_privileged = true; } ]; } ]; #qemu = [ # { # enabled = true; # } #]; } ]; }; # ---- Firewall config ---- # Open ports in the firewall. networking.firewall = { enable = true; # Allow anyone to connect on SSH port allowedTCPPorts = [ (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports) ]; # Allow specific hosts access to specific things in the cluster extraCommands = '' # Allow everything from router (usefull for UPnP/IGD) iptables -A INPUT -s ${cfg.lan_default_gateway} -j ACCEPT # Allow docker containers to access all ports iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # Allow other nodes on VPN to access all ports iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT ''; # When stopping firewall, delete all rules that were configured manually above extraStopCommands = '' iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT ''; }; }; }