From e3eca391e071cc0ba1c3ac214f4b40e4835ee8ec Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 8 Mar 2022 14:16:14 +0100
Subject: [PATCH] Add filebeat to stream logs into elasticsearch

---
 app/telemetry/config/filebeat.yml         | 46 +++++++++++++++++++++++
 app/telemetry/deploy/telemetry-system.hcl | 28 +++++++++++++-
 2 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 app/telemetry/config/filebeat.yml

diff --git a/app/telemetry/config/filebeat.yml b/app/telemetry/config/filebeat.yml
new file mode 100644
index 0000000..6967d97
--- /dev/null
+++ b/app/telemetry/config/filebeat.yml
@@ -0,0 +1,46 @@
+# see https://github.com/elastic/beats/blob/master/filebeat/filebeat.reference.yml
+filebeat.modules:
+- module: system
+  syslog:
+    enabled: true
+  auth:
+    enabled: true
+
+#filebeat.inputs:
+#- type: container
+#  enabled: true
+#  paths:
+#    -/var/lib/docker/containers/*/*.log
+#  stream: all # can be all, stdout or stderr
+
+#========================== Filebeat autodiscover ==============================
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      # https://www.elastic.co/guide/en/beats/filebeat/current/configuration-autodiscover-hints.html
+      # This URL alos contains instructions on multi-line logs
+      hints.enabled: true
+
+#================================ Processors ===================================
+processors:
+# - add_cloud_metadata: ~
+- add_docker_metadata: ~
+- add_locale:
+    format: offset
+- add_host_metadata:
+    netinfo.enabled: true
+
+#========================== Elasticsearch output ===============================
+output.elasticsearch:
+  hosts: ["localhost:9200"]
+  username: beats_system
+  password: {{ key "secrets/telemetry/elastic_passwords/beats_system" }}
+
+#============================== Dashboards =====================================
+setup.dashboards:
+  enabled: false
+
+#============================== Xpack Monitoring ===============================
+xpack.monitoring:
+  enabled: true
+  elasticsearch:
diff --git a/app/telemetry/deploy/telemetry-system.hcl b/app/telemetry/deploy/telemetry-system.hcl
index c9b11b1..16f2a5a 100644
--- a/app/telemetry/deploy/telemetry-system.hcl
+++ b/app/telemetry/deploy/telemetry-system.hcl
@@ -74,7 +74,7 @@ EOH
     task "otel" {
       driver = "docker"
       config {
-        image = "otel/opentelemetry-collector-contrib:0.44.0"
+        image = "otel/opentelemetry-collector-contrib:0.46.0"
         args = [
           "--config=/etc/otel-config.yaml",
         ]
@@ -99,7 +99,7 @@ EOH
     task "apm" {
       driver = "docker"
       config {
-        image = "docker.elastic.co/apm/apm-server:7.17.0"
+        image = "docker.elastic.co/apm/apm-server:7.17.1"
         network_mode = "host"
         ports = [ "apm" ]
         args = [ "--strict.perms=false" ]
@@ -136,6 +136,30 @@ EOH
         memory = 40
       }
     }
+
+    task "filebeat" {
+      driver = "docker"
+      config {
+        image = "docker.elastic.co/beats/filebeat:7.17.1"
+        network_mode = "host"
+        volumes = [
+          "/mnt/ssd/telemetry/filebeat:/usr/share/filebeat/data",
+          "secrets/filebeat.yml:/usr/share/filebeat/filebeat.yml",
+          "/var/run/docker.sock:/var/run/docker.sock",
+          "/var/lib/docker/containers/:/var/lib/docker/containers/:ro",
+          "/var/log/:/var/log/:ro",
+        ]
+        args = [ "--strict.perms=false" ]
+        privileged = true
+      }
+      user = "root"
+
+
+      template {
+        data = file("../config/filebeat.yml")
+        destination = "secrets/filebeat.yml"
+      }
+    }
   }
 }