Prod-like telemetry into staging

1 files changed, 182 insertions, 0 deletions

diff --git a/experimental/bad.telemetry-elastic/deploy/telemetry-system.hcl b/experimental/bad.telemetry-elastic/deploy/telemetry-system.hcl
new file mode 100644
index 0000000..3e26c2e
--- /dev/null
+++ b/experimental/bad.telemetry-elastic/deploy/telemetry-system.hcl
@@ -0,0 +1,182 @@
+job "telemetry-system" {
+  datacenters = ["neptune"]
+  type = "system"
+
+  group "elasticsearch" {
+    network {
+      port "elastic" {
+        static = 9200
+      }
+      port "elastic_internal" {
+        static = 9300
+      }
+    }
+
+    task "elastic" {
+      driver = "docker"
+      config {
+        image = "docker.elastic.co/elasticsearch/elasticsearch:8.2.0"
+        network_mode = "host"
+        volumes = [
+          "/mnt/ssd/telemetry/es_data:/usr/share/elasticsearch/data",
+          "secrets/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12",
+        ]
+        ports = [ "elastic", "elastic_internal" ]
+        sysctl = {
+          #"vm.max_map_count" = "262144",
+        }
+        ulimit = {
+          memlock = "9223372036854775807:9223372036854775807", 
+        }
+      }
+
+      user = "1000"
+
+      resources {
+        memory = 1500
+        cpu = 500
+      }
+
+      template {
+        data = "{{ key \"secrets/telemetry/elasticsearch/elastic-certificates.p12\" }}"
+        destination = "secrets/elastic-certificates.p12"
+      }
+
+      template {
+        data = <<EOH
+node.name={{ env "attr.unique.hostname" }}
+http.port=9200
+transport.port=9300
+cluster.name=es-deuxfleurs
+cluster.initial_master_nodes=carcajou,caribou,cariacou
+discovery.seed_hosts=carcajou,caribou,cariacou
+bootstrap.memory_lock=true
+xpack.security.enabled=true
+xpack.security.authc.api_key.enabled=true
+xpack.security.transport.ssl.enabled=true
+xpack.security.transport.ssl.verification_mode=certificate
+xpack.security.transport.ssl.client_authentication=required
+xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
+xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
+cluster.routing.allocation.disk.watermark.high=75%
+cluster.routing.allocation.disk.watermark.low=65%
+ES_JAVA_OPTS=-Xms512M -Xmx512M
+EOH
+        destination = "secrets/env"
+        env = true
+      }
+    }
+  }
+
+  group "collector" {
+    network {
+      port "otel_grpc" {
+        static = 4317
+      }
+      port "apm" {
+        static = 8200
+      }
+      port "node_exporter" {
+        static = 9100
+      }
+    }
+
+    task "otel" {
+      driver = "docker"
+      config {
+        image = "otel/opentelemetry-collector-contrib:0.46.0"
+        args = [
+          "--config=/etc/otel-config.yaml",
+        ]
+        network_mode = "host"
+        ports= [ "otel_grpc" ]
+        volumes = [
+          "secrets/otel-config.yaml:/etc/otel-config.yaml"
+        ]
+      }
+
+      template {
+        data = file("../config/otel-config.yaml")
+        destination = "secrets/otel-config.yaml"
+      }
+
+      resources {
+        memory = 100
+        cpu = 100
+      }
+    }
+
+    task "apm" {
+      driver = "docker"
+      config {
+        image = "docker.elastic.co/apm/apm-server:8.2.0"
+        network_mode = "host"
+        ports = [ "apm" ]
+        args = [ "--strict.perms=false" ]
+        volumes = [
+          "secrets/apm-config.yaml:/usr/share/apm-server/apm-server.yml:ro"
+        ]
+      }
+
+      template {
+        data = file("../config/apm-config.yaml")
+        destination = "secrets/apm-config.yaml"
+      }
+
+      resources {
+        memory = 100
+        cpu = 100
+      }
+    }
+
+/*
+    task "node_exporter" {
+      driver = "docker"
+      config {
+        image = "quay.io/prometheus/node-exporter:v1.1.2"
+        network_mode = "host"
+        ports = [ "node_exporter" ]
+        volumes = [
+          "/:/host:ro,rslave"
+        ]
+        args = [ "--path.rootfs=/host" ]
+      }
+
+      resources {
+        cpu = 50
+        memory = 40
+      }
+    }
+*/
+
+    task "filebeat" {
+      driver = "docker"
+      config {
+        image = "docker.elastic.co/beats/filebeat:8.2.0"
+        network_mode = "host"
+        volumes = [
+          "/mnt/ssd/telemetry/filebeat:/usr/share/filebeat/data",
+          "secrets/filebeat.yml:/usr/share/filebeat/filebeat.yml",
+          "/var/run/docker.sock:/var/run/docker.sock",
+          "/var/lib/docker/containers/:/var/lib/docker/containers/:ro",
+          "/var/log/:/var/log/:ro",
+        ]
+        args = [ "--strict.perms=false" ]
+        privileged = true
+      }
+      user = "root"
+
+
+      template {
+        data = file("../config/filebeat.yml")
+        destination = "secrets/filebeat.yml"
+      }
+
+      resources {
+        memory = 100
+        cpu = 100
+      }
+    }
+  }
+}
+