Update telemetry to ES 8.2.0 and simplify config a bit

app/telemetry/config/apm-config.yaml

@@ -8,8 +8,8 @@ output.elasticsearch:
   # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`.
   # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`.
   hosts: ["localhost:9200"]
-  username: "apm"
+  username: "elastic"
-  password: "{{ key "secrets/telemetry/elastic_passwords/apm" }}"
+  password: "{{ key "secrets/telemetry/elastic_passwords/elastic" }}"
 
 instrumentation:
   enabled: true

app/telemetry/config/grafana/provisioning/datasources/elastic.yaml

@@ -5,13 +5,13 @@ datasources:
     type: elasticsearch
     access: proxy
     url: http://localhost:9200
-    password: '{{ key "secrets/telemetry/elastic_passwords/grafana" }}'
+    password: '{{ key "secrets/telemetry/elastic_passwords/elastic" }}'
-    user: 'grafana'
+    user: 'elastic'
-    database: apm-*
+    database: metrics-*
     basicAuth: false
     isDefault: true
     jsonData:
-      esVersion: "7.10.0"
+      esVersion: "8.2.0"
       includeFrozen: false
       logLevelField: ''
       logMessageField: ''

app/telemetry/deploy/telemetry-system.hcl

@@ -15,10 +15,11 @@ job "telemetry-system" {
     task "elastic" {
       driver = "docker"
       config {
-        image = "docker.elastic.co/elasticsearch/elasticsearch:7.17.0"
+        image = "docker.elastic.co/elasticsearch/elasticsearch:8.2.0"
         network_mode = "host"
         volumes = [
           "/mnt/ssd/telemetry/es_data:/usr/share/elasticsearch/data",
+          "secrets/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12",
         ]
         ports = [ "elastic", "elastic_internal" ]
         sysctl = {
@@ -29,11 +30,18 @@ job "telemetry-system" {
         }
       }
 
+      user = "1000"
+
       resources {
         memory = 1500
         cpu = 500
       }
 
+      template {
+        data = "{{ key \"secrets/telemetry/elasticsearch/elastic-certificates.p12\" }}"
+        destination = "secrets/elastic-certificates.p12"
+      }
+
       template {
         data = <<EOH
 node.name={{ env "attr.unique.hostname" }}
@@ -48,8 +56,8 @@ xpack.security.authc.api_key.enabled=true
 xpack.security.transport.ssl.enabled=true
 xpack.security.transport.ssl.verification_mode=certificate
 xpack.security.transport.ssl.client_authentication=required
-xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/data/elastic-certificates.p12
+xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
-xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/data/elastic-certificates.p12
+xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
 cluster.routing.allocation.disk.watermark.high=75%
 cluster.routing.allocation.disk.watermark.low=65%
 ES_JAVA_OPTS=-Xms512M -Xmx512M
@@ -101,7 +109,7 @@ EOH
     task "apm" {
       driver = "docker"
       config {
-        image = "docker.elastic.co/apm/apm-server:7.17.1"
+        image = "docker.elastic.co/apm/apm-server:8.2.0"
         network_mode = "host"
         ports = [ "apm" ]
         args = [ "--strict.perms=false" ]
@@ -144,7 +152,7 @@ EOH
     task "filebeat" {
       driver = "docker"
       config {
-        image = "docker.elastic.co/beats/filebeat:7.17.1"
+        image = "docker.elastic.co/beats/filebeat:8.2.0"
         network_mode = "host"
         volumes = [
           "/mnt/ssd/telemetry/filebeat:/usr/share/filebeat/data",

app/telemetry/deploy/telemetry.hcl

@@ -14,7 +14,7 @@ job "telemetry" {
     task "kibana" {
       driver = "docker"
       config {
-        image = "docker.elastic.co/kibana/kibana:7.17.0"
+        image = "docker.elastic.co/kibana/kibana:8.2.0"
         network_mode = "host"
         ports = [ "kibana" ]
       }

doc/telemetry.md

@@ -1,13 +1,6 @@
-# create elasticsearch folders on all nodes
-
-```bash
-mkdir -p /mnt/ssd/telemetry/es_data/nodes
-chown 1000 /mnt/ssd/telemetry/es_data/nodes
-```
-
 # generate ca and tls certs for elasticsearch cluster
 
-start a `bash` in an elasticsearch image, such as `docker.elastic.co/elasticsearch/elasticsearch:7.17.0`: `docker run -ti docker.elastic.co/elasticsearch/elasticsearch:7.17.0 bash`
+start a `bash` in an elasticsearch image, such as `docker.elastic.co/elasticsearch/elasticsearch:8.2.0`: `docker run -ti docker.elastic.co/elasticsearch/elasticsearch:8.2.0 bash`
 
 generate a ca and node certs:
 
@@ -16,46 +9,29 @@ generate a ca and node certs:
 ./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12
 ```
 
-copy `elastic-certificates.p12` to `/mnt/ssd/telemetry/es_data` in all nodes, and chown it:
+write these files in Consul at `secrets/telemetry/elasticsearch/elastic-certificates.p12` and `secrets/telemetry/elasticsearch/elastic-stack-ca.p12`
+
+# start nomad services
 
 ```bash
-chown 1000 /mnt/ssd/telemetry/es_data/elastic-certificates.p12
+nomad run telemetry-system.hcl
+nomad run telemetry.hcl
 ```
 
 # create elasticsearch passwords
 
-in elasticsearch container
+in an elasticsearch container that was launched by Nomad:
 
 ```bash
-./bin/elasticsearch-setup-passwords auto
+./bin/elasticsearch-reset-password -u elastic
+./bin/elasticsearch-reset-password -u kibana
 ```
 
 save passwords in consul, at:
 
-- `secrets/telemetry/elastic_passwords/apm_system` for user `apm_system`
 - `secrets/telemetry/elastic_passwords/kibana_system` for user `kibana_system`
 - `secrets/telemetry/elastic_passwords/elastic` for user `elastic`
 
 check kibana works, login to kibana with user `elastic`
 
-# create role and user for apm
+grafana and apm-server will use the elastic user (the admin) to write data to elasticsearch
-
-create role `apm_writer`, give privileges:
-
-- cluster privileges `manage_ilm`, `read_ilm`, `manage_ingest_pipelines`, `manage_index_templates`
-- on index `apm-*` privileges `create_doc`, `create_index`, `view_index_metadata`, `manage`
-- on index `apm-*sourcemap` privilege `read_cross_cluster`
-
-create user `apm` with roles `apm_writer` and `apm_system`. give it a randomly generated password that you save in `secrets/telemetry/elastic_passwords/apm`
-
-check apm data is ingested correctly (visible in kibana)
-
-# create role and user for grafana
-
-create role `grafana`, give privileges:
-
-- on index `apm-*` privileges `read` and `view_index_metadata`
-
-create user `grafana` with role `grafana`. give it a randomly generated password that you save in `secrets/telemetry/elastic_passwords/grafana`
-
-check grafana works