diff options
| -rw-r--r-- | app/telemetry/config/apm-config.yaml | 4 | ||||
| -rw-r--r-- | app/telemetry/config/grafana/provisioning/datasources/elastic.yaml | 8 | ||||
| -rw-r--r-- | app/telemetry/deploy/telemetry-system.hcl | 18 | ||||
| -rw-r--r-- | app/telemetry/deploy/telemetry.hcl | 2 | ||||
| -rw-r--r-- | doc/telemetry.md | 44 |
5 files changed, 30 insertions, 46 deletions
diff --git a/app/telemetry/config/apm-config.yaml b/app/telemetry/config/apm-config.yaml index 9288036..07a88bd 100644 --- a/app/telemetry/config/apm-config.yaml +++ b/app/telemetry/config/apm-config.yaml @@ -8,8 +8,8 @@ output.elasticsearch: # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`. # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. hosts: ["localhost:9200"] - username: "apm" - password: "{{ key "secrets/telemetry/elastic_passwords/apm" }}" + username: "elastic" + password: "{{ key "secrets/telemetry/elastic_passwords/elastic" }}" instrumentation: enabled: true diff --git a/app/telemetry/config/grafana/provisioning/datasources/elastic.yaml b/app/telemetry/config/grafana/provisioning/datasources/elastic.yaml index a41be6f..7d2277c 100644 --- a/app/telemetry/config/grafana/provisioning/datasources/elastic.yaml +++ b/app/telemetry/config/grafana/provisioning/datasources/elastic.yaml @@ -5,13 +5,13 @@ datasources: type: elasticsearch access: proxy url: http://localhost:9200 - password: '{{ key "secrets/telemetry/elastic_passwords/grafana" }}' - user: 'grafana' - database: apm-* + password: '{{ key "secrets/telemetry/elastic_passwords/elastic" }}' + user: 'elastic' + database: metrics-* basicAuth: false isDefault: true jsonData: - esVersion: "7.10.0" + esVersion: "8.2.0" includeFrozen: false logLevelField: '' logMessageField: '' diff --git a/app/telemetry/deploy/telemetry-system.hcl b/app/telemetry/deploy/telemetry-system.hcl index cb39bac..d5a7241 100644 --- a/app/telemetry/deploy/telemetry-system.hcl +++ b/app/telemetry/deploy/telemetry-system.hcl @@ -15,10 +15,11 @@ job "telemetry-system" { task "elastic" { driver = "docker" config { - image = "docker.elastic.co/elasticsearch/elasticsearch:7.17.0" + image = "docker.elastic.co/elasticsearch/elasticsearch:8.2.0" network_mode = "host" volumes = [ "/mnt/ssd/telemetry/es_data:/usr/share/elasticsearch/data", + "secrets/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12", ] ports = [ "elastic", "elastic_internal" ] sysctl = { @@ -29,12 +30,19 @@ job "telemetry-system" { } } + user = "1000" + resources { memory = 1500 cpu = 500 } template { + data = "{{ key \"secrets/telemetry/elasticsearch/elastic-certificates.p12\" }}" + destination = "secrets/elastic-certificates.p12" + } + + template { data = <<EOH node.name={{ env "attr.unique.hostname" }} http.port=9200 @@ -48,8 +56,8 @@ xpack.security.authc.api_key.enabled=true xpack.security.transport.ssl.enabled=true xpack.security.transport.ssl.verification_mode=certificate xpack.security.transport.ssl.client_authentication=required -xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/data/elastic-certificates.p12 -xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/data/elastic-certificates.p12 +xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12 +xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12 cluster.routing.allocation.disk.watermark.high=75% cluster.routing.allocation.disk.watermark.low=65% ES_JAVA_OPTS=-Xms512M -Xmx512M @@ -101,7 +109,7 @@ EOH task "apm" { driver = "docker" config { - image = "docker.elastic.co/apm/apm-server:7.17.1" + image = "docker.elastic.co/apm/apm-server:8.2.0" network_mode = "host" ports = [ "apm" ] args = [ "--strict.perms=false" ] @@ -144,7 +152,7 @@ EOH task "filebeat" { driver = "docker" config { - image = "docker.elastic.co/beats/filebeat:7.17.1" + image = "docker.elastic.co/beats/filebeat:8.2.0" network_mode = "host" volumes = [ "/mnt/ssd/telemetry/filebeat:/usr/share/filebeat/data", diff --git a/app/telemetry/deploy/telemetry.hcl b/app/telemetry/deploy/telemetry.hcl index fc0c389..afabfa7 100644 --- a/app/telemetry/deploy/telemetry.hcl +++ b/app/telemetry/deploy/telemetry.hcl @@ -14,7 +14,7 @@ job "telemetry" { task "kibana" { driver = "docker" config { - image = "docker.elastic.co/kibana/kibana:7.17.0" + image = "docker.elastic.co/kibana/kibana:8.2.0" network_mode = "host" ports = [ "kibana" ] } diff --git a/doc/telemetry.md b/doc/telemetry.md index b34cfc8..968b1e4 100644 --- a/doc/telemetry.md +++ b/doc/telemetry.md @@ -1,13 +1,6 @@ -# create elasticsearch folders on all nodes - -```bash -mkdir -p /mnt/ssd/telemetry/es_data/nodes -chown 1000 /mnt/ssd/telemetry/es_data/nodes -``` - # generate ca and tls certs for elasticsearch cluster -start a `bash` in an elasticsearch image, such as `docker.elastic.co/elasticsearch/elasticsearch:7.17.0`: `docker run -ti docker.elastic.co/elasticsearch/elasticsearch:7.17.0 bash` +start a `bash` in an elasticsearch image, such as `docker.elastic.co/elasticsearch/elasticsearch:8.2.0`: `docker run -ti docker.elastic.co/elasticsearch/elasticsearch:8.2.0 bash` generate a ca and node certs: @@ -16,46 +9,29 @@ generate a ca and node certs: ./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12 ``` -copy `elastic-certificates.p12` to `/mnt/ssd/telemetry/es_data` in all nodes, and chown it: +write these files in Consul at `secrets/telemetry/elasticsearch/elastic-certificates.p12` and `secrets/telemetry/elasticsearch/elastic-stack-ca.p12` + +# start nomad services ```bash -chown 1000 /mnt/ssd/telemetry/es_data/elastic-certificates.p12 +nomad run telemetry-system.hcl +nomad run telemetry.hcl ``` # create elasticsearch passwords -in elasticsearch container +in an elasticsearch container that was launched by Nomad: ```bash -./bin/elasticsearch-setup-passwords auto +./bin/elasticsearch-reset-password -u elastic +./bin/elasticsearch-reset-password -u kibana ``` save passwords in consul, at: -- `secrets/telemetry/elastic_passwords/apm_system` for user `apm_system` - `secrets/telemetry/elastic_passwords/kibana_system` for user `kibana_system` - `secrets/telemetry/elastic_passwords/elastic` for user `elastic` check kibana works, login to kibana with user `elastic` -# create role and user for apm - -create role `apm_writer`, give privileges: - -- cluster privileges `manage_ilm`, `read_ilm`, `manage_ingest_pipelines`, `manage_index_templates` -- on index `apm-*` privileges `create_doc`, `create_index`, `view_index_metadata`, `manage` -- on index `apm-*sourcemap` privilege `read_cross_cluster` - -create user `apm` with roles `apm_writer` and `apm_system`. give it a randomly generated password that you save in `secrets/telemetry/elastic_passwords/apm` - -check apm data is ingested correctly (visible in kibana) - -# create role and user for grafana - -create role `grafana`, give privileges: - -- on index `apm-*` privileges `read` and `view_index_metadata` - -create user `grafana` with role `grafana`. give it a randomly generated password that you save in `secrets/telemetry/elastic_passwords/grafana` - -check grafana works +grafana and apm-server will use the elastic user (the admin) to write data to elasticsearch |