aboutsummaryrefslogtreecommitdiff
path: root/app/telemetry
diff options
context:
space:
mode:
authorAlex Auvolat <alex@adnab.me>2022-02-26 21:31:58 +0100
committerAlex Auvolat <alex@adnab.me>2022-02-26 21:31:58 +0100
commit41a65b803217d7a3ebdaa5029d58a5cac4ce63da (patch)
tree3853243fc3b3eb4999479ae5fb5d116f047b49d1 /app/telemetry
parent7dcb4072dcced7a4d92ae2c356722ccb6bfd5e2b (diff)
downloadnixcfg-41a65b803217d7a3ebdaa5029d58a5cac4ce63da.tar.gz
nixcfg-41a65b803217d7a3ebdaa5029d58a5cac4ce63da.zip
Allow grafana to move between nodes
Diffstat (limited to 'app/telemetry')
-rw-r--r--app/telemetry/config/grafana-litestream.yml10
-rw-r--r--app/telemetry/deploy/telemetry-system.hcl2
-rw-r--r--app/telemetry/deploy/telemetry.hcl86
3 files changed, 89 insertions, 9 deletions
diff --git a/app/telemetry/config/grafana-litestream.yml b/app/telemetry/config/grafana-litestream.yml
new file mode 100644
index 0000000..a537d9c
--- /dev/null
+++ b/app/telemetry/config/grafana-litestream.yml
@@ -0,0 +1,10 @@
+dbs:
+ - path: /ephemeral/grafana.db
+ replicas:
+ - url: s3://grafana-db/grafana.db
+ region: garage-staging
+ endpoint: http://{{ env "attr.unique.network.ip-address" }}:3990
+ access-key-id: {{ key "secrets/telemetry/grafana/s3_access_key" | trimSpace }}
+ secret-access-key: {{ key "secrets/telemetry/grafana/s3_secret_key" | trimSpace }}
+ force-path-style: true
+ sync-interval: 60s
diff --git a/app/telemetry/deploy/telemetry-system.hcl b/app/telemetry/deploy/telemetry-system.hcl
index af9acff..82103bd 100644
--- a/app/telemetry/deploy/telemetry-system.hcl
+++ b/app/telemetry/deploy/telemetry-system.hcl
@@ -30,7 +30,7 @@ job "telemetry-system" {
}
resources {
- memory = 2500
+ memory = 1500
cpu = 500
}
diff --git a/app/telemetry/deploy/telemetry.hcl b/app/telemetry/deploy/telemetry.hcl
index c3edd4c..b394699 100644
--- a/app/telemetry/deploy/telemetry.hcl
+++ b/app/telemetry/deploy/telemetry.hcl
@@ -31,7 +31,7 @@ EOH
}
resources {
- memory = 1500
+ memory = 1000
cpu = 500
}
@@ -62,18 +62,42 @@ EOH
group "grafana" {
count = 1
- constraint {
- attribute = "${attr.unique.hostname}"
- operator = "="
- value = "carcajou"
- }
-
network {
port "grafana" {
static = 3333
}
}
+ task "restore-db" {
+ lifecycle {
+ hook = "prestart"
+ sidecar = false
+ }
+
+ driver = "docker"
+ config {
+ image = "litestream/litestream:0.3.7"
+ args = [
+ "restore", "-config", "/etc/litestream.yml", "/ephemeral/grafana.db"
+ ]
+ volumes = [
+ "../alloc/data:/ephemeral",
+ "secrets/litestream.yml:/etc/litestream.yml"
+ ]
+ }
+ user = "472"
+
+ template {
+ data = file("../config/grafana-litestream.yml")
+ destination = "secrets/litestream.yml"
+ }
+
+ resources {
+ memory = 200
+ cpu = 1000
+ }
+ }
+
task "grafana" {
driver = "docker"
config {
@@ -81,7 +105,7 @@ EOH
network_mode = "host"
ports = [ "grafana" ]
volumes = [
- "/mnt/ssd/telemetry/grafana:/var/lib/grafana",
+ "../alloc/data:/var/lib/grafana",
"secrets/elastic.yaml:/etc/grafana/provisioning/datasources/elastic.yaml"
]
}
@@ -104,7 +128,53 @@ EOH
memory = 500
cpu = 100
}
+
+ service {
+ tags = [
+ "grafana",
+ "tricot grafana.home.adnab.me",
+ ]
+ port = 3333
+ address_mode = "driver"
+ name = "grafana"
+ check {
+ type = "tcp"
+ port = 3333
+ address_mode = "driver"
+ interval = "60s"
+ timeout = "5s"
+ check_restart {
+ limit = 3
+ grace = "90s"
+ ignore_warnings = false
+ }
+ }
+ }
}
+ task "replicate-db" {
+ driver = "docker"
+ config {
+ image = "litestream/litestream:0.3.7"
+ args = [
+ "replicate", "-config", "/etc/litestream.yml"
+ ]
+ volumes = [
+ "../alloc/data:/ephemeral",
+ "secrets/litestream.yml:/etc/litestream.yml"
+ ]
+ }
+ user = "472"
+
+ template {
+ data = file("../config/grafana-litestream.yml")
+ destination = "secrets/litestream.yml"
+ }
+
+ resources {
+ memory = 200
+ cpu = 100
+ }
+ }
}
}