Add security to telemetry deployment

app/telemetry/config/apm-config.yaml

@@ -8,3 +8,9 @@ output.elasticsearch:
   # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`.
   # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`.
   hosts: ["localhost:9200"]
+  username: "apm"
+  password: "{{ key "secrets/telemetry/elastic_passwords/apm" }}"
+
+logging:
+  level: warning
+  to_stderr: true

app/telemetry/config/grafana/provisioning/datasources/elastic.yaml

@@ -5,8 +5,8 @@ datasources:
     type: elasticsearch
     access: proxy
     url: http://localhost:9200
-    password: ''
+    password: '{{ key "secrets/telemetry/elastic_passwords/grafana" }}'
-    user: ''
+    user: 'grafana'
     database: apm-*
     basicAuth: false
     isDefault: true

app/telemetry/deploy/telemetry.hcl

@@ -77,6 +77,7 @@ http.port=9200
 cluster.name=es-docker-cluster
 discovery.type=single-node
 bootstrap.memory_lock=true
+xpack.security.enabled=true
 ES_JAVA_OPTS=-Xms512m -Xmx512m
 EOH
         destination = "secrets/env"
@@ -96,6 +97,8 @@ EOH
         data = <<EOH
 SERVER_NAME=kibana.local
 ELASTICSEARCH_HOSTS=http://localhost:9200
+ELASTICSEARCH_USERNAME=kibana_system
+ELASTICSEARCH_PASSWORD={{ key "secrets/telemetry/elastic_passwords/kibana_system" }}
 EOH
         destination = "secrets/env"
         env = true

doc/telemetry.md

@@ -0,0 +1,37 @@
+# create elasticsearch passwords
+
+in elasticsearch container
+
+```bash
+./bin/elasticsearch-setup-passwords auto
+```
+
+save passwords in consul, at:
+
+- `secrets/telemetry/elastic_passwords/apm_system` for user `apm_system`
+- `secrets/telemetry/elastic_passwords/kibana_system` for user `kibana_system`
+- `secrets/telemetry/elastic_passwords/elastic` for user `elastic`
+
+check kibana works, login to kibana with user `elastic`
+
+# create role and user for apm
+
+create role `apm_writer`, give privileges:
+
+- cluster privileges `manage_ilm`, `read_ilm`, `manage_ingest_pipelines`
+- on index `apm-*` privileges `create_doc`, `create_index`, `view_index_metadata`
+- on index `apm-*sourcemap` privilege `read_cross_cluster`
+
+create user `apm` with roles `apm_writer` and `apm_system`. give it a randomly generated password that you save in `secrets/telemetry/elastic_passwords/apm`
+
+check apm data is ingested correctly (visible in kibana)
+
+# create role and user for grafana
+
+create role `grafana`, give privileges:
+
+- on index `apm-*` privileges `read` and `view_index_metadata`
+
+create user `grafana` with role `grafana`. give it a randomly generated password that you save in `secrets/telemetry/elastic_passwords/grafana`
+
+check grafana works