1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
{ config, pkgs, ... } @ args:
{
deuxfleurs.cluster_name = "staging";
# The IP range to use for the Wireguard overlay of this cluster
deuxfleurs.cluster_prefix = "10.14.0.0";
deuxfleurs.cluster_prefix_length = 16;
deuxfleurs.cluster_nodes = [
{
hostname = "carcajou";
site_name = "neptune";
publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
IP = "10.14.1.2";
lan_endpoint = "192.168.1.22:33799";
endpoint = "77.207.15.215:33722";
}
{
hostname = "caribou";
site_name = "neptune";
publicKey = "lABn/axzD1jkFulX8c+K3B3CbKXORlIMDDoe8sQVxhs=";
IP = "10.14.1.3";
lan_endpoint = "192.168.1.23:33799";
endpoint = "77.207.15.215:33723";
}
{
hostname = "origan";
site_name = "jupiter";
publicKey = "smBQYUS60JDkNoqkTT7TgbpqFiM43005fcrT6472llI=";
IP = "10.14.2.33";
lan_endpoint = "192.168.1.33:33799";
endpoint = "82.64.238.84:33733";
}
{
hostname = "piranha";
site_name = "corrin";
publicKey = "m9rLf+233X1VColmeVrM/xfDGro5W6Gk5N0zqcf32WY=";
IP = "10.14.3.1";
lan_endpoint = "192.168.1.25:33799";
endpoint = "90.49.20.119:33721";
}
{
hostname = "df-pw5";
site_name = "bespin";
publicKey = "XLOYoMXF+PO4jcgfSVAk+thh4VmWx0wzWnb0xs08G1s=";
IP = "10.14.4.1";
lan_endpoint = "192.168.5.130:33799";
endpoint = "bitfrost.fiber.shirokumo.net:33734";
}
];
# Bootstrap IPs for Consul cluster,
# these are IPs on the Wireguard overlay
services.consul.extraConfig.retry_join = [
"10.14.1.1" # cariacou
"10.14.1.2" # carcajou
"10.14.1.3" # caribou
];
deuxfleurs.admin_accounts = {
lx = [
# Keys for accessing nodes from outside
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIw+IIX8+lZX9RrHAbwi/bncLYStXpI4EmK3AUcqPY2O lx@kusanagi "
];
quentin = [
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDT1+H08FdUSvdPpPKdcafq4+JRHvFVjfvG5Id97LAoROmFRUb/ZOMTLdNuD7FqvW0Da5CPxIMr8ZxfrFLtpGyuG7qdI030iIRZPlKpBh37epZHaV+l9F4ZwJQMIBO9cuyLPXgsyvM/s7tDtrdK1k7JTf2EVvoirrjSzBaMhAnhi7//to8zvujDtgDZzy6aby75bAaDetlYPBq2brWehtrf9yDDG9WAMYJqp//scje/WmhbRR6eSdim1HaUcWk5+4ZPt8sQJcy8iWxQ4jtgjqTvMOe5v8ZPkxJNBine/ZKoJsv7FzKem00xEH7opzktaGukyEqH0VwOwKhmBiqsX2yN quentin@dufour.io"
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBu+KUebaWwlugMC5fGbNhHc6IaQDAC6+1vMc4Ww7nVU1rs2nwI7L5qcWxOwNdhFaorZQZy/fJuCWdFbF61RCKGayBWPLZHGPsfqDuggYNEi1Qil1kpeCECfDQNjyMTK058ZBBhOWNMHBjlLWXUlRJDkRBBECY0vo4jRv22SvSaPUCAnkdJ9rbAp/kqb497PTIb2r1l1/ew8YdhINAlpYQFQezZVfkZdTKxt22n0QCjhupqjfh3gfNnbBX0z/iO+RvAOWRIZsjPFLC+jXl+n7cnu2cq1nvST5eHiYfXXeIgIwmeENLKqp+2Twr7PIdv22PnJkh6iR5kx7eTRxkNZdN quentin@deuxfleurs.fr"
];
adrien = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINvUFN6HmZS5oxxOtmF6ug393m5NYbSbDI4G8pX6H9GZ adrien@pratchett"
];
maximilien = [
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHMMR6zNzz8NQU80wFquhUCeiXJuGphjP+zNouKbn228GyESu8sfNBwnuZq86vblR11Lz8l2rtCM73GfAKg29qmUWUHRKWvRIYWv2vaUJcCdy0bAxIzcvCvjZX0SpnIKxe9y3Rp0LGO5WLYfw0ZFaavwFZP0Z8w1Kj9/zBmL2X2avbhkaYHi/C1yXhbvESYQysmqLa48EX/TS616MBrgR9zbI9AoTQ9NOHnR14Tve/AP/khcZoBJdm4hTttMbNkEc0wonzdylTDew263SPRs/uoqnQIpUtErdPHqU10Yup8HjXjEyFJsSwcZcM5sZOw5JKckKJwmcd0yjO/x/4/Mk5"
];
trinity = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDWGWTRoF5MjQ5bmFdQENQlNdoYtA7Wd61GM0TMHZDki"
];
kokakiwi = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPTsEgcOtb2bij+Ih8eg8ZqO7d3IMiWykv6deMzlSSS kokakiwi@kira"
];
baptiste = [
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCnGkJZZrHIUp9q0DXmVLLuhCIe7Vu1J3j6dJ1z1BglqX7yOLdFQ6LhHXx65aND/KCOM1815tJSnaAyKWEj9qJ31RVUoRl42yBn54DvQumamJUaXAHqJrXhjwxfUkF9B73ZSUzHGADlQnxcBkmrjC5FkrpC/s4xr0o7/GIBkBdtZhX9YpxBfpH6wEcCruTOlm92E3HvvjpBb/wHsoxL1f2czvWe69021gqWEYRFjqtBwP36NYZnGOJZ0RrlP3wUrGCSHxOKW+2Su+tM6g07KPJn5l1wNJiOcyBQ0/Sv7ptCJ9+rTQNeVBMoXshaucYP/bKJbqH7dONrYDgz59C4+Kax"
];
};
# For Garage ipv6 communication
networking.firewall.allowedTCPPorts = [ 3991 ];
## ===== EXPERIMENTAL SECTION FOR STAGING CLUSTER =====
# We're doing lots of experiments so GC periodically is usefull.
nix.gc.automatic = true;
imports = [
## ---- Nix Nomad jobs using nomad-driver-nix2 ----
({ pkgs, ... }: {
services.nomad.dropPrivileges = false;
services.nomad.extraSettingsPlugins = [
(import ./nomad-driver-nix2.nix { inherit pkgs; })
];
services.nomad.extraPackages = [
pkgs.nix
pkgs.git
];
services.nomad.settings.plugin = [
{
"nix2-driver" = [
{
config = [
{
default_nixpkgs = "github:nixos/nixpkgs/nixos-22.11";
}
];
}
];
}
];
})
## ---- Nix cache: use our cache on Garage (prod cluster) ----
# Use our cache as additionnal substituer (this acts the same way for
# our Nix packages than the Docker hub acts for our Docker images)
({ pkgs, ... }: {
nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ];
nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ];
})
## ---- Nix mutual cache ----
# Let nodes in a same site/zone copy from each other's Nix stores
# Note that nodes will only copy from one another packages that are
# signed by one of the trusted public keys, i.e. packages comming
# from cache.nixos.org and nix.web.deuxfleurs.fr.
# This is good as it kind of mitigates supply-chain attacks where
# one node's cache would become poisonned, although arguably when
# an attacker has gained root access on one node, it can easily
# become root on all the others through Nomad. Downsides include
# missed opportunities for not rebuilding stuff between machines
# (e.g. derivations that are built in the process of doing
# nixos-rebuild), and warnings appearing in the logs whenever such
# an opportunity was not taken due to missing signatures.
({ pkgs, config, ... }:
let substituter_port = 1728;
in
{
services.nix-serve = {
enable = true;
port = substituter_port;
openFirewall = false;
bindAddress = config.deuxfleurs.cluster_ip;
package = pkgs.haskellPackages.nix-serve-ng;
};
nix.settings.substituters = map
({ IP, ... }: "http://${IP}:${builtins.toString substituter_port}")
(builtins.filter
({ site_name, IP, ...}:
(IP != config.deuxfleurs.cluster_ip
&& site_name == config.deuxfleurs.site_name))
config.deuxfleurs.cluster_nodes);
})
];
}
|
