Add filebeat to stream logs into elasticsearch

This commit is contained in:
Alex 2022-03-08 14:16:14 +01:00
parent 27ffee95b8
commit e3eca391e0
Signed by untrusted user: lx
GPG key ID: 0E496D15096376BE
2 changed files with 72 additions and 2 deletions

View file

@ -0,0 +1,46 @@
# see https://github.com/elastic/beats/blob/master/filebeat/filebeat.reference.yml
filebeat.modules:
- module: system
syslog:
enabled: true
auth:
enabled: true
#filebeat.inputs:
#- type: container
# enabled: true
# paths:
# -/var/lib/docker/containers/*/*.log
# stream: all # can be all, stdout or stderr
#========================== Filebeat autodiscover ==============================
filebeat.autodiscover:
providers:
- type: docker
# https://www.elastic.co/guide/en/beats/filebeat/current/configuration-autodiscover-hints.html
# This URL alos contains instructions on multi-line logs
hints.enabled: true
#================================ Processors ===================================
processors:
# - add_cloud_metadata: ~
- add_docker_metadata: ~
- add_locale:
format: offset
- add_host_metadata:
netinfo.enabled: true
#========================== Elasticsearch output ===============================
output.elasticsearch:
hosts: ["localhost:9200"]
username: beats_system
password: {{ key "secrets/telemetry/elastic_passwords/beats_system" }}
#============================== Dashboards =====================================
setup.dashboards:
enabled: false
#============================== Xpack Monitoring ===============================
xpack.monitoring:
enabled: true
elasticsearch:

View file

@ -74,7 +74,7 @@ EOH
task "otel" {
driver = "docker"
config {
image = "otel/opentelemetry-collector-contrib:0.44.0"
image = "otel/opentelemetry-collector-contrib:0.46.0"
args = [
"--config=/etc/otel-config.yaml",
]
@ -99,7 +99,7 @@ EOH
task "apm" {
driver = "docker"
config {
image = "docker.elastic.co/apm/apm-server:7.17.0"
image = "docker.elastic.co/apm/apm-server:7.17.1"
network_mode = "host"
ports = [ "apm" ]
args = [ "--strict.perms=false" ]
@ -136,6 +136,30 @@ EOH
memory = 40
}
}
task "filebeat" {
driver = "docker"
config {
image = "docker.elastic.co/beats/filebeat:7.17.1"
network_mode = "host"
volumes = [
"/mnt/ssd/telemetry/filebeat:/usr/share/filebeat/data",
"secrets/filebeat.yml:/usr/share/filebeat/filebeat.yml",
"/var/run/docker.sock:/var/run/docker.sock",
"/var/lib/docker/containers/:/var/lib/docker/containers/:ro",
"/var/log/:/var/log/:ro",
]
args = [ "--strict.perms=false" ]
privileged = true
}
user = "root"
template {
data = file("../config/filebeat.yml")
destination = "secrets/filebeat.yml"
}
}
}
}