diff options
| -rw-r--r-- | cluster/staging/cluster.nix | 84 |
1 files changed, 61 insertions, 23 deletions
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix index 7582992..c387a22 100644 --- a/cluster/staging/cluster.nix +++ b/cluster/staging/cluster.nix @@ -72,36 +72,74 @@ # For Garage ipv6 communication networking.firewall.allowedTCPPorts = [ 3991 ]; - ## ----- + ## ===== EXPERIMENTAL SECTION FOR STAGING CLUSTER ===== - ## EXPERIMENTAL ON STAGING: NIX NOMAD JOBS - services.nomad.dropPrivileges = false; + # We're doing lots of experiments so GC periodically is usefull. + nix.gc.automatic = true; - # ----- nomad-driver-nix & nomad-driver-nix2 ----- - services.nomad.extraSettingsPlugins = [ - (import ./nomad-driver-nix2.nix { inherit pkgs; }) - ]; - services.nomad.extraPackages = [ - pkgs.nix - pkgs.git - ]; - # default config for the nix2 driver - services.nomad.settings.plugin = [ - { - "nix2-driver" = [ + imports = [ + ## ---- Nix Nomad jobs using nomad-driver-nix2 ---- + ({ pkgs, ... }: { + services.nomad.dropPrivileges = false; + services.nomad.extraSettingsPlugins = [ + (import ./nomad-driver-nix2.nix { inherit pkgs; }) + ]; + services.nomad.extraPackages = [ + pkgs.nix + pkgs.git + ]; + services.nomad.settings.plugin = [ { - config = [ + "nix2-driver" = [ { - # default_nixpkgs = "github:nixos/nixpkgs/nixos-22.11"; + config = [ + { + default_nixpkgs = "github:nixos/nixpkgs/nixos-22.11"; + } + ]; } ]; } ]; - } + }) + ## ---- Nix cache: use our cache on Garage (prod cluster) ---- + # Use our cache as additionnal substituer (this acts the same way for + # our Nix packages than the Docker hub acts for our Docker images) + ({ pkgs, ... }: { + nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ]; + nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ]; + }) + ## ---- Nix mutual cache ---- + # Let nodes in a same site/zone copy from each other's Nix stores + # Note that nodes will only copy from one another packages that are + # signed by one of the trusted public keys, i.e. packages comming + # from cache.nixos.org and nix.web.deuxfleurs.fr. + # This is good as it kind of mitigates supply-chain attacks where + # one node's cache would become poisonned, although arguably when + # an attacker has gained root access on one node, it can easily + # become root on all the others through Nomad. Downsides include + # missed opportunities for not rebuilding stuff between machines + # (e.g. derivations that are built in the process of doing + # nixos-rebuild), and warnings appearing in the logs whenever such + # an opportunity was not taken due to missing signatures. + ({ pkgs, config, ... }: + let substituter_port = 1728; + in + { + services.nix-serve = { + enable = true; + port = substituter_port; + openFirewall = false; + bindAddress = config.deuxfleurs.cluster_ip; + package = pkgs.haskellPackages.nix-serve-ng; + }; + nix.settings.substituters = map + ({ IP, ... }: "http://${IP}:${builtins.toString substituter_port}") + (builtins.filter + ({ site_name, IP, ...}: + (IP != config.deuxfleurs.cluster_ip + && site_name == config.deuxfleurs.site_name)) + config.deuxfleurs.cluster_nodes); + }) ]; - - # use our cache as additionnal substituer (we put precompiled packages there, - # like we used to do on the docker hub) - nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ]; - nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ]; } |