From fe805b6bab6336554b62b997c3cf6fa7344ac08d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 3 Jan 2023 21:00:10 +0100 Subject: Fix prometheus ssl certs --- cluster/prod/app/telemetry/config/prometheus.yml | 12 +- .../app/telemetry/deploy/telemetry-service.hcl | 223 +++++++++++++++++++++ cluster/prod/app/telemetry/deploy/telemetry.hcl | 223 --------------------- 3 files changed, 229 insertions(+), 229 deletions(-) create mode 100644 cluster/prod/app/telemetry/deploy/telemetry-service.hcl delete mode 100644 cluster/prod/app/telemetry/deploy/telemetry.hcl diff --git a/cluster/prod/app/telemetry/config/prometheus.yml b/cluster/prod/app/telemetry/config/prometheus.yml index 6525513..fd426cc 100644 --- a/cluster/prod/app/telemetry/config/prometheus.yml +++ b/cluster/prod/app/telemetry/config/prometheus.yml @@ -12,7 +12,7 @@ scrape_configs: services: - 'node-exporter' tls_config: - ca_file: /etc/prometheus/consul.crt + ca_file: /etc/prometheus/consul-ca.crt cert_file: /etc/prometheus/consul-client.crt key_file: /etc/prometheus/consul-client.key @@ -22,7 +22,7 @@ scrape_configs: services: - 'tricot-metrics' tls_config: - ca_file: /etc/prometheus/consul.crt + ca_file: /etc/prometheus/consul-ca.crt cert_file: /etc/prometheus/consul-client.crt key_file: /etc/prometheus/consul-client.key @@ -35,7 +35,7 @@ scrape_configs: services: - 'garage-admin' tls_config: - ca_file: /etc/prometheus/consul.crt + ca_file: /etc/prometheus/consul-ca.crt cert_file: /etc/prometheus/consul-client.crt key_file: /etc/prometheus/consul-client.key @@ -48,7 +48,7 @@ scrape_configs: services: - 'drone' tls_config: - ca_file: /etc/prometheus/consul.crt + ca_file: /etc/prometheus/consul-ca.crt cert_file: /etc/prometheus/consul-client.crt key_file: /etc/prometheus/consul-client.key @@ -63,7 +63,7 @@ scrape_configs: format: ['prometheus'] scheme: 'https' tls_config: - ca_file: /etc/prometheus/nomad.crt + ca_file: /etc/prometheus/nomad-ca.crt cert_file: /etc/prometheus/nomad-client.crt key_file: /etc/prometheus/nomad-client.key insecure_skip_verify: true @@ -72,6 +72,6 @@ scrape_configs: services: - 'nomad-client' tls_config: - ca_file: /etc/prometheus/consul.crt + ca_file: /etc/prometheus/consul-ca.crt cert_file: /etc/prometheus/consul-client.crt key_file: /etc/prometheus/consul-client.key diff --git a/cluster/prod/app/telemetry/deploy/telemetry-service.hcl b/cluster/prod/app/telemetry/deploy/telemetry-service.hcl new file mode 100644 index 0000000..4790cbd --- /dev/null +++ b/cluster/prod/app/telemetry/deploy/telemetry-service.hcl @@ -0,0 +1,223 @@ +job "telemetry-service" { + datacenters = ["neptune", "bespin"] + type = "service" + + group "prometheus" { + count = 2 + + network { + port "prometheus" { + static = 9090 + } + } + + constraint { + attribute = "${attr.unique.hostname}" + operator = "set_contains_any" + value = "concombre,df-ymk" + } + + task "prometheus" { + driver = "docker" + config { + image = "prom/prometheus:v2.39.0" + network_mode = "host" + ports = [ "prometheus" ] + args = [ + "--config.file=/etc/prometheus/prometheus.yml", + "--storage.tsdb.path=/data", + "--storage.tsdb.retention.size=20GB", + ] + volumes = [ + "secrets:/etc/prometheus", + "/mnt/ssd/prometheus:/data" + ] + } + + template { + data = file("../config/prometheus.yml") + destination = "secrets/prometheus.yml" + } + + template { + data = "{{ key \"secrets/consul/consul-ca.crt\" }}" + destination = "secrets/consul-ca.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.crt\" }}" + destination = "secrets/consul-client.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.key\" }}" + destination = "secrets/consul-client.key" + } + + template { + data = "{{ key \"secrets/nomad/nomad-ca.crt\" }}" + destination = "secrets/nomad-ca.crt" + } + + template { + data = "{{ key \"secrets/nomad/nomad-client.crt\" }}" + destination = "secrets/nomad-client.crt" + } + + template { + data = "{{ key \"secrets/nomad/nomad-client.key\" }}" + destination = "secrets/nomad-client.key" + } + + resources { + memory = 1000 + cpu = 1000 + } + + service { + port = 9090 + address_mode = "driver" + name = "prometheus" + check { + type = "http" + path = "/" + port = 9090 + address_mode = "driver" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + } + } + } + + group "grafana" { + count = 1 + + network { + port "grafana" { + static = 3719 + } + } + + task "restore-db" { + lifecycle { + hook = "prestart" + sidecar = false + } + + driver = "docker" + config { + image = "litestream/litestream:0.3.7" + args = [ + "restore", "-config", "/etc/litestream.yml", "/ephemeral/grafana.db" + ] + volumes = [ + "../alloc/data:/ephemeral", + "secrets/litestream.yml:/etc/litestream.yml" + ] + } + user = "472" + + template { + data = file("../config/grafana-litestream.yml") + destination = "secrets/litestream.yml" + } + + resources { + memory = 200 + cpu = 100 + } + } + + task "grafana" { + driver = "docker" + config { + image = "grafana/grafana:9.2.0" + network_mode = "host" + ports = [ "grafana" ] + volumes = [ + "../alloc/data:/var/lib/grafana", + "secrets/prometheus.yaml:/etc/grafana/provisioning/datasources/prometheus.yaml", + "secrets/ldap.toml:/etc/grafana/ldap.toml" + ] + } + + template { + data = file("../config/grafana-datasource-prometheus.yaml") + destination = "secrets/prometheus.yaml" + } + + template { + data = file("../config/grafana-ldap.toml") + destination = "secrets/ldap.toml" + } + + template { + data = <