From 2e8923b383eb06c53261eee8e5c442b857fb67e4 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 24 Aug 2022 15:42:47 +0200 Subject: Move app files into cluster subdirectories; add prod garage --- app/telemetry/config/apm-config.yaml | 20 --- app/telemetry/config/filebeat.yml | 46 ------ app/telemetry/config/grafana-litestream.yml | 10 -- .../grafana/provisioning/datasources/elastic.yaml | 21 --- app/telemetry/config/otel-config.yaml | 56 ------- app/telemetry/deploy/telemetry-system.hcl | 182 --------------------- app/telemetry/deploy/telemetry.hcl | 181 -------------------- 7 files changed, 516 deletions(-) delete mode 100644 app/telemetry/config/apm-config.yaml delete mode 100644 app/telemetry/config/filebeat.yml delete mode 100644 app/telemetry/config/grafana-litestream.yml delete mode 100644 app/telemetry/config/grafana/provisioning/datasources/elastic.yaml delete mode 100644 app/telemetry/config/otel-config.yaml delete mode 100644 app/telemetry/deploy/telemetry-system.hcl delete mode 100644 app/telemetry/deploy/telemetry.hcl (limited to 'app/telemetry') diff --git a/app/telemetry/config/apm-config.yaml b/app/telemetry/config/apm-config.yaml deleted file mode 100644 index 07a88bd..0000000 --- a/app/telemetry/config/apm-config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apm-server: - # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket. - host: "0.0.0.0:8200" -#-------------------------- Elasticsearch output -------------------------- -output.elasticsearch: - # Array of hosts to connect to. - # Scheme and port can be left out and will be set to the default (`http` and `9200`). - # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`. - # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. - hosts: ["localhost:9200"] - username: "elastic" - password: "{{ key "secrets/telemetry/elastic_passwords/elastic" }}" - -instrumentation: - enabled: true - environment: staging - -logging: - level: warning - to_stderr: true diff --git a/app/telemetry/config/filebeat.yml b/app/telemetry/config/filebeat.yml deleted file mode 100644 index 310afd1..0000000 --- a/app/telemetry/config/filebeat.yml +++ /dev/null @@ -1,46 +0,0 @@ -# see https://github.com/elastic/beats/blob/master/filebeat/filebeat.reference.yml -filebeat.modules: -- module: system - syslog: - enabled: true - auth: - enabled: true - -#filebeat.inputs: -#- type: container -# enabled: true -# paths: -# -/var/lib/docker/containers/*/*.log -# stream: all # can be all, stdout or stderr - -#========================== Filebeat autodiscover ============================== -filebeat.autodiscover: - providers: - - type: docker - # https://www.elastic.co/guide/en/beats/filebeat/current/configuration-autodiscover-hints.html - # This URL alos contains instructions on multi-line logs - hints.enabled: true - -#================================ Processors =================================== -processors: -# - add_cloud_metadata: ~ -- add_docker_metadata: ~ -- add_locale: - format: offset -- add_host_metadata: - netinfo.enabled: true - -#========================== Elasticsearch output =============================== -output.elasticsearch: - hosts: ["localhost:9200"] - username: elastic - password: {{ key "secrets/telemetry/elastic_passwords/elastic" }} - -#============================== Dashboards ===================================== -setup.dashboards: - enabled: false - -#============================== Xpack Monitoring =============================== -xpack.monitoring: - enabled: true - elasticsearch: diff --git a/app/telemetry/config/grafana-litestream.yml b/app/telemetry/config/grafana-litestream.yml deleted file mode 100644 index a537d9c..0000000 --- a/app/telemetry/config/grafana-litestream.yml +++ /dev/null @@ -1,10 +0,0 @@ -dbs: - - path: /ephemeral/grafana.db - replicas: - - url: s3://grafana-db/grafana.db - region: garage-staging - endpoint: http://{{ env "attr.unique.network.ip-address" }}:3990 - access-key-id: {{ key "secrets/telemetry/grafana/s3_access_key" | trimSpace }} - secret-access-key: {{ key "secrets/telemetry/grafana/s3_secret_key" | trimSpace }} - force-path-style: true - sync-interval: 60s diff --git a/app/telemetry/config/grafana/provisioning/datasources/elastic.yaml b/app/telemetry/config/grafana/provisioning/datasources/elastic.yaml deleted file mode 100644 index 7d2277c..0000000 --- a/app/telemetry/config/grafana/provisioning/datasources/elastic.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: 1 - -datasources: - - name: DS_ELASTICSEARCH - type: elasticsearch - access: proxy - url: http://localhost:9200 - password: '{{ key "secrets/telemetry/elastic_passwords/elastic" }}' - user: 'elastic' - database: metrics-* - basicAuth: false - isDefault: true - jsonData: - esVersion: "8.2.0" - includeFrozen: false - logLevelField: '' - logMessageField: '' - maxConcurrentShardRequests: 5 - timeField: "@timestamp" - timeInterval: "5s" - readOnly: false diff --git a/app/telemetry/config/otel-config.yaml b/app/telemetry/config/otel-config.yaml deleted file mode 100644 index bcf1baa..0000000 --- a/app/telemetry/config/otel-config.yaml +++ /dev/null @@ -1,56 +0,0 @@ -receivers: - # Data sources: metrics, traces - otlp: - protocols: - grpc: - endpoint: ":4317" - http: - endpoint: ":55681" - # Data sources: metrics - prometheus: - config: - scrape_configs: - - job_name: "garage" - scrape_interval: 5s - static_configs: - - targets: - - "{{ env "attr.unique.network.ip-address" }}:3909" - - job_name: "node_exporter" - scrape_interval: 5s - static_configs: - - targets: - - "{{ env "attr.unique.network.ip-address" }}:9100" - -exporters: - logging: - logLevel: info - # see https://www.elastic.co/guide/en/apm/get-started/current/open-telemetry-elastic.html#open-telemetry-collector - otlp/elastic: - endpoint: "localhost:8200" - tls: - insecure: true - -processors: - batch: - probabilistic_sampler: - hash_seed: 42 - sampling_percentage: 10 - -extensions: - health_check: - pprof: - endpoint: :1888 - zpages: - endpoint: :55679 - -service: - extensions: [pprof, zpages, health_check] - pipelines: - traces: - receivers: [otlp] - processors: [probabilistic_sampler, batch] - exporters: [logging, otlp/elastic] - metrics: - receivers: [otlp, prometheus] - processors: [batch] - exporters: [logging, otlp/elastic] diff --git a/app/telemetry/deploy/telemetry-system.hcl b/app/telemetry/deploy/telemetry-system.hcl deleted file mode 100644 index 3e26c2e..0000000 --- a/app/telemetry/deploy/telemetry-system.hcl +++ /dev/null @@ -1,182 +0,0 @@ -job "telemetry-system" { - datacenters = ["neptune"] - type = "system" - - group "elasticsearch" { - network { - port "elastic" { - static = 9200 - } - port "elastic_internal" { - static = 9300 - } - } - - task "elastic" { - driver = "docker" - config { - image = "docker.elastic.co/elasticsearch/elasticsearch:8.2.0" - network_mode = "host" - volumes = [ - "/mnt/ssd/telemetry/es_data:/usr/share/elasticsearch/data", - "secrets/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12", - ] - ports = [ "elastic", "elastic_internal" ] - sysctl = { - #"vm.max_map_count" = "262144", - } - ulimit = { - memlock = "9223372036854775807:9223372036854775807", - } - } - - user = "1000" - - resources { - memory = 1500 - cpu = 500 - } - - template { - data = "{{ key \"secrets/telemetry/elasticsearch/elastic-certificates.p12\" }}" - destination = "secrets/elastic-certificates.p12" - } - - template { - data = <