aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlex Auvolat <alex@adnab.me>2022-10-16 18:12:57 +0200
committerAlex Auvolat <alex@adnab.me>2022-10-16 18:12:57 +0200
commit5613ed99086618b91441d84620bdbe27a94ad1d2 (patch)
tree304f247c4099a430a29f6d44f5f41172a2c50a7d
parent42409de1b1638f5d935831dc66ddff9a1d9ba29d (diff)
downloadnixcfg-5613ed99086618b91441d84620bdbe27a94ad1d2.tar.gz
nixcfg-5613ed99086618b91441d84620bdbe27a94ad1d2.zip
Complete telemetry configuration
-rw-r--r--cluster/prod/app/telemetry/config/grafana-ldap.toml49
-rw-r--r--cluster/prod/app/telemetry/config/prometheus.yml24
-rw-r--r--cluster/prod/app/telemetry/deploy/telemetry-system.hcl4
-rw-r--r--cluster/prod/app/telemetry/deploy/telemetry.hcl46
-rwxr-xr-xdeploy_pki6
-rw-r--r--nix/deuxfleurs.nix5
6 files changed, 126 insertions, 8 deletions
diff --git a/cluster/prod/app/telemetry/config/grafana-ldap.toml b/cluster/prod/app/telemetry/config/grafana-ldap.toml
new file mode 100644
index 0000000..31cf18a
--- /dev/null
+++ b/cluster/prod/app/telemetry/config/grafana-ldap.toml
@@ -0,0 +1,49 @@
+[[servers]]
+# Ldap server host (specify multiple hosts space separated)
+host = "bottin.service.prod.consul"
+# Default port is 389 or 636 if use_ssl = true
+port = 389
+# Set to true if LDAP server should use an encrypted TLS connection (either with STARTTLS or LDAPS)
+use_ssl = false
+# If set to true, use LDAP with STARTTLS instead of LDAPS
+start_tls = false
+# set to true if you want to skip SSL cert validation
+ssl_skip_verify = false
+# set to the path to your root CA certificate or leave unset to use system defaults
+# root_ca_cert = "/path/to/certificate.crt"
+# Authentication against LDAP servers requiring client certificates
+# client_cert = "/path/to/client.crt"
+# client_key = "/path/to/client.key"
+
+# Search user bind dn
+bind_dn = "cn=grafana,ou=services,ou=users,dc=deuxfleurs,dc=fr"
+# Search user bind password
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+bind_password = "{{ key "secrets/telemetry/grafana/grafana_ldap_password" | trimSpace }}"
+
+# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)"
+# Allow login from email or username, example "(|(sAMAccountName=%s)(userPrincipalName=%s))"
+search_filter = "(cn=%s)"
+
+# An array of base dns to search through
+search_base_dns = ["ou=users,dc=deuxfleurs,dc=fr"]
+
+# group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))"
+# group_search_filter_user_attribute = "distinguishedName"
+# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"]
+
+[[servers.group_mappings]]
+group_dn = "cn=admin,ou=groups,dc=deuxfleurs,dc=fr"
+org_role = "Admin"
+grafana_admin = true
+
+[[servers.group_mappings]]
+group_dn = "*"
+org_role = "Viewer"
+
+# Specify names of the LDAP attributes your LDAP uses
+[servers.attributes]
+member_of = "memberof"
+email = "mail"
+username = "cn"
+uid = "cn"
diff --git a/cluster/prod/app/telemetry/config/prometheus.yml b/cluster/prod/app/telemetry/config/prometheus.yml
index 42d438c..a52b64d 100644
--- a/cluster/prod/app/telemetry/config/prometheus.yml
+++ b/cluster/prod/app/telemetry/config/prometheus.yml
@@ -41,3 +41,27 @@ scrape_configs:
ca_file: /etc/prometheus/consul.crt
cert_file: /etc/prometheus/consul-client.crt
key_file: /etc/prometheus/consul-client.key
+
+ # see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config
+ # and https://www.nomadproject.io/api-docs/metrics
+ # and https://learn.hashicorp.com/tutorials/nomad/prometheus-metrics
+ # dashboard at https://grafana.com/grafana/dashboards/3800
+ - job_name: 'nomad'
+ scrape_interval: 10s
+ metrics_path: "/v1/metrics"
+ params:
+ format: ['prometheus']
+ scheme: 'https'
+ tls_config:
+ ca_file: /etc/prometheus/nomad.crt
+ cert_file: /etc/prometheus/nomad-client.crt
+ key_file: /etc/prometheus/nomad-client.key
+ insecure_skip_verify: true
+ consul_sd_configs:
+ - server: 'https://localhost:8501'
+ services:
+ - 'nomad-client'
+ tls_config:
+ ca_file: /etc/prometheus/consul.crt
+ cert_file: /etc/prometheus/consul-client.crt
+ key_file: /etc/prometheus/consul-client.key
diff --git a/cluster/prod/app/telemetry/deploy/telemetry-system.hcl b/cluster/prod/app/telemetry/deploy/telemetry-system.hcl
index e4bde1a..ae9ff72 100644
--- a/cluster/prod/app/telemetry/deploy/telemetry-system.hcl
+++ b/cluster/prod/app/telemetry/deploy/telemetry-system.hcl
@@ -1,5 +1,5 @@
job "telemetry-system" {
- datacenters = ["neptune", "orion"]
+ datacenters = ["neptune", "orion", "bespin"]
type = "system"
priority = "100"
@@ -12,7 +12,7 @@ job "telemetry-system" {
driver = "docker"
config {
- image = "quay.io/prometheus/node-exporter:v1.1.2"
+ image = "quay.io/prometheus/node-exporter:v1.4.0"
network_mode = "host"
volumes = [
"/:/host:ro,rslave"
diff --git a/cluster/prod/app/telemetry/deploy/telemetry.hcl b/cluster/prod/app/telemetry/deploy/telemetry.hcl
index d35bd7e..af3088c 100644
--- a/cluster/prod/app/telemetry/deploy/telemetry.hcl
+++ b/cluster/prod/app/telemetry/deploy/telemetry.hcl
@@ -1,9 +1,9 @@
job "telemetry" {
- datacenters = ["neptune"]
+ datacenters = ["neptune", "bespin"]
type = "service"
group "prometheus" {
- count = 1
+ count = 2
network {
port "prometheus" {
@@ -11,14 +11,26 @@ job "telemetry" {
}
}
+ constraint {
+ attribute = "${attr.unique.hostname}"
+ operator = "set_contains_any"
+ value = "concombre,df-ymk"
+ }
+
task "prometheus" {
driver = "docker"
config {
- image = "prom/prometheus:v2.38.0"
+ image = "prom/prometheus:v2.39.0"
network_mode = "host"
ports = [ "prometheus" ]
+ args = [
+ "--config.file=/etc/prometheus/prometheus.yml",
+ "--storage.tsdb.path=/data",
+ "--storage.tsdb.retention.size=20GB",
+ ]
volumes = [
- "secrets:/etc/prometheus"
+ "secrets:/etc/prometheus",
+ "/mnt/ssd/prometheus:/data"
]
}
@@ -42,6 +54,21 @@ job "telemetry" {
destination = "secrets/consul-client.key"
}
+ template {
+ data = "{{ key \"secrets/nomad/nomad.crt\" }}"
+ destination = "secrets/nomad.crt"
+ }
+
+ template {
+ data = "{{ key \"secrets/nomad/nomad-client.crt\" }}"
+ destination = "secrets/nomad-client.crt"
+ }
+
+ template {
+ data = "{{ key \"secrets/nomad/nomad-client.key\" }}"
+ destination = "secrets/nomad-client.key"
+ }
+
resources {
memory = 501
cpu = 500
@@ -110,12 +137,13 @@ job "telemetry" {
task "grafana" {
driver = "docker"
config {
- image = "grafana/grafana:8.4.3"
+ image = "grafana/grafana:9.2.0"
network_mode = "host"
ports = [ "grafana" ]
volumes = [
"../alloc/data:/var/lib/grafana",
- "secrets/prometheus.yaml:/etc/grafana/provisioning/datasources/prometheus.yaml"
+ "secrets/prometheus.yaml:/etc/grafana/provisioning/datasources/prometheus.yaml",
+ "secrets/ldap.toml:/etc/grafana/ldap.toml"
]
}
@@ -125,9 +153,15 @@ job "telemetry" {
}
template {
+ data = file("../config/grafana-ldap.toml")
+ destination = "secrets/ldap.toml"
+ }
+
+ template {
data = <<EOH
GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel,grafana-worldmap-panel,grafana-polystat-panel
GF_SERVER_HTTP_PORT=3719
+GF_AUTH_LDAP_ENABLED=true
EOH
destination = "secrets/env"
env = true
diff --git a/deploy_pki b/deploy_pki
index cb39bec..d6b0d3a 100755
--- a/deploy_pki
+++ b/deploy_pki
@@ -18,6 +18,7 @@ cmd systemctl restart consul
cmd sleep 10
for file in nomad-ca.crt nomad$YEAR.crt nomad$YEAR.key \
+ nomad$YEAR-client.crt nomad$YEAR-client.key \
consul$YEAR.crt consul$YEAR-client.crt consul$YEAR-client.key
do
if pass $PKI/$file >/dev/null; then
@@ -37,3 +38,8 @@ cmd "consul kv put secrets/consul/consul-ca.crt - < /var/lib/consul/pki/consul-c
cmd "consul kv put secrets/consul/consul.crt - < /var/lib/consul/pki/consul$YEAR.crt"
cmd "consul kv put secrets/consul/consul-client.crt - < /var/lib/consul/pki/consul$YEAR-client.crt"
cmd "consul kv put secrets/consul/consul-client.key - < /var/lib/consul/pki/consul$YEAR-client.key"
+
+cmd "consul kv put secrets/nomad/nomad-ca.crt - < /var/lib/private/nomad/pki/nomad-ca.crt"
+cmd "consul kv put secrets/nomad/nomad.crt - < /var/lib/private/nomad/pki/nomad$YEAR.crt"
+cmd "consul kv put secrets/nomad/nomad-client.crt - < /var/lib/private/nomad/pki/nomad$YEAR-client.crt"
+cmd "consul kv put secrets/nomad/nomad-client.key - < /var/lib/private/nomad/pki/nomad$YEAR-client.key"
diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix
index 3e880cc..66c153c 100644
--- a/nix/deuxfleurs.nix
+++ b/nix/deuxfleurs.nix
@@ -305,6 +305,11 @@ in
"public_ipv6" = cfg.ipv6;
};
};
+ telemetry = {
+ publish_allocation_metrics = true;
+ publish_node_metrics = true;
+ prometheus_metrics = true;
+ };
tls = {
http = true;
rpc = true;