diff options
| -rw-r--r-- | cluster/prod/app/telemetry/config/grafana-ldap.toml | 49 | ||||
| -rw-r--r-- | cluster/prod/app/telemetry/config/prometheus.yml | 24 | ||||
| -rw-r--r-- | cluster/prod/app/telemetry/deploy/telemetry-system.hcl | 4 | ||||
| -rw-r--r-- | cluster/prod/app/telemetry/deploy/telemetry.hcl | 46 | ||||
| -rwxr-xr-x | deploy_pki | 6 | ||||
| -rw-r--r-- | nix/deuxfleurs.nix | 5 |
6 files changed, 126 insertions, 8 deletions
diff --git a/cluster/prod/app/telemetry/config/grafana-ldap.toml b/cluster/prod/app/telemetry/config/grafana-ldap.toml new file mode 100644 index 0000000..31cf18a --- /dev/null +++ b/cluster/prod/app/telemetry/config/grafana-ldap.toml @@ -0,0 +1,49 @@ +[[servers]] +# Ldap server host (specify multiple hosts space separated) +host = "bottin.service.prod.consul" +# Default port is 389 or 636 if use_ssl = true +port = 389 +# Set to true if LDAP server should use an encrypted TLS connection (either with STARTTLS or LDAPS) +use_ssl = false +# If set to true, use LDAP with STARTTLS instead of LDAPS +start_tls = false +# set to true if you want to skip SSL cert validation +ssl_skip_verify = false +# set to the path to your root CA certificate or leave unset to use system defaults +# root_ca_cert = "/path/to/certificate.crt" +# Authentication against LDAP servers requiring client certificates +# client_cert = "/path/to/client.crt" +# client_key = "/path/to/client.key" + +# Search user bind dn +bind_dn = "cn=grafana,ou=services,ou=users,dc=deuxfleurs,dc=fr" +# Search user bind password +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +bind_password = "{{ key "secrets/telemetry/grafana/grafana_ldap_password" | trimSpace }}" + +# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)" +# Allow login from email or username, example "(|(sAMAccountName=%s)(userPrincipalName=%s))" +search_filter = "(cn=%s)" + +# An array of base dns to search through +search_base_dns = ["ou=users,dc=deuxfleurs,dc=fr"] + +# group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" +# group_search_filter_user_attribute = "distinguishedName" +# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] + +[[servers.group_mappings]] +group_dn = "cn=admin,ou=groups,dc=deuxfleurs,dc=fr" +org_role = "Admin" +grafana_admin = true + +[[servers.group_mappings]] +group_dn = "*" +org_role = "Viewer" + +# Specify names of the LDAP attributes your LDAP uses +[servers.attributes] +member_of = "memberof" +email = "mail" +username = "cn" +uid = "cn" diff --git a/cluster/prod/app/telemetry/config/prometheus.yml b/cluster/prod/app/telemetry/config/prometheus.yml index 42d438c..a52b64d 100644 --- a/cluster/prod/app/telemetry/config/prometheus.yml +++ b/cluster/prod/app/telemetry/config/prometheus.yml @@ -41,3 +41,27 @@ scrape_configs: ca_file: /etc/prometheus/consul.crt cert_file: /etc/prometheus/consul-client.crt key_file: /etc/prometheus/consul-client.key + + # see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config + # and https://www.nomadproject.io/api-docs/metrics + # and https://learn.hashicorp.com/tutorials/nomad/prometheus-metrics + # dashboard at https://grafana.com/grafana/dashboards/3800 + - job_name: 'nomad' + scrape_interval: 10s + metrics_path: "/v1/metrics" + params: + format: ['prometheus'] + scheme: 'https' + tls_config: + ca_file: /etc/prometheus/nomad.crt + cert_file: /etc/prometheus/nomad-client.crt + key_file: /etc/prometheus/nomad-client.key + insecure_skip_verify: true + consul_sd_configs: + - server: 'https://localhost:8501' + services: + - 'nomad-client' + tls_config: + ca_file: /etc/prometheus/consul.crt + cert_file: /etc/prometheus/consul-client.crt + key_file: /etc/prometheus/consul-client.key diff --git a/cluster/prod/app/telemetry/deploy/telemetry-system.hcl b/cluster/prod/app/telemetry/deploy/telemetry-system.hcl index e4bde1a..ae9ff72 100644 --- a/cluster/prod/app/telemetry/deploy/telemetry-system.hcl +++ b/cluster/prod/app/telemetry/deploy/telemetry-system.hcl @@ -1,5 +1,5 @@ job "telemetry-system" { - datacenters = ["neptune", "orion"] + datacenters = ["neptune", "orion", "bespin"] type = "system" priority = "100" @@ -12,7 +12,7 @@ job "telemetry-system" { driver = "docker" config { - image = "quay.io/prometheus/node-exporter:v1.1.2" + image = "quay.io/prometheus/node-exporter:v1.4.0" network_mode = "host" volumes = [ "/:/host:ro,rslave" diff --git a/cluster/prod/app/telemetry/deploy/telemetry.hcl b/cluster/prod/app/telemetry/deploy/telemetry.hcl index d35bd7e..af3088c 100644 --- a/cluster/prod/app/telemetry/deploy/telemetry.hcl +++ b/cluster/prod/app/telemetry/deploy/telemetry.hcl @@ -1,9 +1,9 @@ job "telemetry" { - datacenters = ["neptune"] + datacenters = ["neptune", "bespin"] type = "service" group "prometheus" { - count = 1 + count = 2 network { port "prometheus" { @@ -11,14 +11,26 @@ job "telemetry" { } } + constraint { + attribute = "${attr.unique.hostname}" + operator = "set_contains_any" + value = "concombre,df-ymk" + } + task "prometheus" { driver = "docker" config { - image = "prom/prometheus:v2.38.0" + image = "prom/prometheus:v2.39.0" network_mode = "host" ports = [ "prometheus" ] + args = [ + "--config.file=/etc/prometheus/prometheus.yml", + "--storage.tsdb.path=/data", + "--storage.tsdb.retention.size=20GB", + ] volumes = [ - "secrets:/etc/prometheus" + "secrets:/etc/prometheus", + "/mnt/ssd/prometheus:/data" ] } @@ -42,6 +54,21 @@ job "telemetry" { destination = "secrets/consul-client.key" } + template { + data = "{{ key \"secrets/nomad/nomad.crt\" }}" + destination = "secrets/nomad.crt" + } + + template { + data = "{{ key \"secrets/nomad/nomad-client.crt\" }}" + destination = "secrets/nomad-client.crt" + } + + template { + data = "{{ key \"secrets/nomad/nomad-client.key\" }}" + destination = "secrets/nomad-client.key" + } + resources { memory = 501 cpu = 500 @@ -110,12 +137,13 @@ job "telemetry" { task "grafana" { driver = "docker" config { - image = "grafana/grafana:8.4.3" + image = "grafana/grafana:9.2.0" network_mode = "host" ports = [ "grafana" ] volumes = [ "../alloc/data:/var/lib/grafana", - "secrets/prometheus.yaml:/etc/grafana/provisioning/datasources/prometheus.yaml" + "secrets/prometheus.yaml:/etc/grafana/provisioning/datasources/prometheus.yaml", + "secrets/ldap.toml:/etc/grafana/ldap.toml" ] } @@ -125,9 +153,15 @@ job "telemetry" { } template { + data = file("../config/grafana-ldap.toml") + destination = "secrets/ldap.toml" + } + + template { data = <<EOH GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel,grafana-worldmap-panel,grafana-polystat-panel GF_SERVER_HTTP_PORT=3719 +GF_AUTH_LDAP_ENABLED=true EOH destination = "secrets/env" env = true @@ -18,6 +18,7 @@ cmd systemctl restart consul cmd sleep 10 for file in nomad-ca.crt nomad$YEAR.crt nomad$YEAR.key \ + nomad$YEAR-client.crt nomad$YEAR-client.key \ consul$YEAR.crt consul$YEAR-client.crt consul$YEAR-client.key do if pass $PKI/$file >/dev/null; then @@ -37,3 +38,8 @@ cmd "consul kv put secrets/consul/consul-ca.crt - < /var/lib/consul/pki/consul-c cmd "consul kv put secrets/consul/consul.crt - < /var/lib/consul/pki/consul$YEAR.crt" cmd "consul kv put secrets/consul/consul-client.crt - < /var/lib/consul/pki/consul$YEAR-client.crt" cmd "consul kv put secrets/consul/consul-client.key - < /var/lib/consul/pki/consul$YEAR-client.key" + +cmd "consul kv put secrets/nomad/nomad-ca.crt - < /var/lib/private/nomad/pki/nomad-ca.crt" +cmd "consul kv put secrets/nomad/nomad.crt - < /var/lib/private/nomad/pki/nomad$YEAR.crt" +cmd "consul kv put secrets/nomad/nomad-client.crt - < /var/lib/private/nomad/pki/nomad$YEAR-client.crt" +cmd "consul kv put secrets/nomad/nomad-client.key - < /var/lib/private/nomad/pki/nomad$YEAR-client.key" diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 3e880cc..66c153c 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -305,6 +305,11 @@ in "public_ipv6" = cfg.ipv6; }; }; + telemetry = { + publish_allocation_metrics = true; + publish_node_metrics = true; + prometheus_metrics = true; + }; tls = { http = true; rpc = true; |