aboutsummaryrefslogtreecommitdiff
path: root/app/telemetry
diff options
context:
space:
mode:
authorAlex Auvolat <alex@adnab.me>2022-03-08 14:16:14 +0100
committerAlex Auvolat <alex@adnab.me>2022-03-08 14:16:14 +0100
commite3eca391e071cc0ba1c3ac214f4b40e4835ee8ec (patch)
tree69933f1b8bfdf83eb1f25fd24e3b520e7dabee3c /app/telemetry
parent27ffee95b894a6a8c8af8a4c32453506229c149e (diff)
downloadnixcfg-e3eca391e071cc0ba1c3ac214f4b40e4835ee8ec.tar.gz
nixcfg-e3eca391e071cc0ba1c3ac214f4b40e4835ee8ec.zip
Add filebeat to stream logs into elasticsearch
Diffstat (limited to 'app/telemetry')
-rw-r--r--app/telemetry/config/filebeat.yml46
-rw-r--r--app/telemetry/deploy/telemetry-system.hcl28
2 files changed, 72 insertions, 2 deletions
diff --git a/app/telemetry/config/filebeat.yml b/app/telemetry/config/filebeat.yml
new file mode 100644
index 0000000..6967d97
--- /dev/null
+++ b/app/telemetry/config/filebeat.yml
@@ -0,0 +1,46 @@
+# see https://github.com/elastic/beats/blob/master/filebeat/filebeat.reference.yml
+filebeat.modules:
+- module: system
+ syslog:
+ enabled: true
+ auth:
+ enabled: true
+
+#filebeat.inputs:
+#- type: container
+# enabled: true
+# paths:
+# -/var/lib/docker/containers/*/*.log
+# stream: all # can be all, stdout or stderr
+
+#========================== Filebeat autodiscover ==============================
+filebeat.autodiscover:
+ providers:
+ - type: docker
+ # https://www.elastic.co/guide/en/beats/filebeat/current/configuration-autodiscover-hints.html
+ # This URL alos contains instructions on multi-line logs
+ hints.enabled: true
+
+#================================ Processors ===================================
+processors:
+# - add_cloud_metadata: ~
+- add_docker_metadata: ~
+- add_locale:
+ format: offset
+- add_host_metadata:
+ netinfo.enabled: true
+
+#========================== Elasticsearch output ===============================
+output.elasticsearch:
+ hosts: ["localhost:9200"]
+ username: beats_system
+ password: {{ key "secrets/telemetry/elastic_passwords/beats_system" }}
+
+#============================== Dashboards =====================================
+setup.dashboards:
+ enabled: false
+
+#============================== Xpack Monitoring ===============================
+xpack.monitoring:
+ enabled: true
+ elasticsearch:
diff --git a/app/telemetry/deploy/telemetry-system.hcl b/app/telemetry/deploy/telemetry-system.hcl
index c9b11b1..16f2a5a 100644
--- a/app/telemetry/deploy/telemetry-system.hcl
+++ b/app/telemetry/deploy/telemetry-system.hcl
@@ -74,7 +74,7 @@ EOH
task "otel" {
driver = "docker"
config {
- image = "otel/opentelemetry-collector-contrib:0.44.0"
+ image = "otel/opentelemetry-collector-contrib:0.46.0"
args = [
"--config=/etc/otel-config.yaml",
]
@@ -99,7 +99,7 @@ EOH
task "apm" {
driver = "docker"
config {
- image = "docker.elastic.co/apm/apm-server:7.17.0"
+ image = "docker.elastic.co/apm/apm-server:7.17.1"
network_mode = "host"
ports = [ "apm" ]
args = [ "--strict.perms=false" ]
@@ -136,6 +136,30 @@ EOH
memory = 40
}
}
+
+ task "filebeat" {
+ driver = "docker"
+ config {
+ image = "docker.elastic.co/beats/filebeat:7.17.1"
+ network_mode = "host"
+ volumes = [
+ "/mnt/ssd/telemetry/filebeat:/usr/share/filebeat/data",
+ "secrets/filebeat.yml:/usr/share/filebeat/filebeat.yml",
+ "/var/run/docker.sock:/var/run/docker.sock",
+ "/var/lib/docker/containers/:/var/lib/docker/containers/:ro",
+ "/var/log/:/var/log/:ro",
+ ]
+ args = [ "--strict.perms=false" ]
+ privileged = true
+ }
+ user = "root"
+
+
+ template {
+ data = file("../config/filebeat.yml")
+ destination = "secrets/filebeat.yml"
+ }
+ }
}
}