From b6561f6e1bcb6a8de13a186405a480e356df89d8 Mon Sep 17 00:00:00 2001 From: Maximilien Date: Thu, 17 Feb 2022 09:38:57 +0100 Subject: [PATCH] Add docker-compose for traces & metrics --- script/telemetry/elastic/.env | 3 + script/telemetry/elastic/apm-config.yaml | 10 +++ script/telemetry/elastic/docker-compose.yml | 69 +++++++++++++++++++ .../provisioning/datasources/elastic.yaml | 20 ++++++ script/telemetry/elastic/otel-config.yaml | 47 +++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 script/telemetry/elastic/.env create mode 100644 script/telemetry/elastic/apm-config.yaml create mode 100644 script/telemetry/elastic/docker-compose.yml create mode 100644 script/telemetry/elastic/grafana/provisioning/datasources/elastic.yaml create mode 100644 script/telemetry/elastic/otel-config.yaml diff --git a/script/telemetry/elastic/.env b/script/telemetry/elastic/.env new file mode 100644 index 000000000..c4d06423b --- /dev/null +++ b/script/telemetry/elastic/.env @@ -0,0 +1,3 @@ +COMPOSE_PROJECT_NAME=telemetry +OTEL_COLLECT_TAG=0.44.0 +ELASTIC_BUNDLE_TAG=7.17.0 diff --git a/script/telemetry/elastic/apm-config.yaml b/script/telemetry/elastic/apm-config.yaml new file mode 100644 index 000000000..1c1e64529 --- /dev/null +++ b/script/telemetry/elastic/apm-config.yaml @@ -0,0 +1,10 @@ +apm-server: + # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket. + host: "0.0.0.0:8200" +#-------------------------- Elasticsearch output -------------------------- +output.elasticsearch: + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (`http` and `9200`). + # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`. + # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. + hosts: ["localhost:9200"] diff --git a/script/telemetry/elastic/docker-compose.yml b/script/telemetry/elastic/docker-compose.yml new file mode 100644 index 000000000..c93ab68e9 --- /dev/null +++ b/script/telemetry/elastic/docker-compose.yml @@ -0,0 +1,69 @@ +version: "2" +services: + + otel: + image: otel/opentelemetry-collector-contrib:${OTEL_COLLECT_TAG} + command: [ "--config=/etc/otel-config.yaml" ] + volumes: + - ./otel-config.yaml:/etc/otel-config.yaml + network_mode: "host" + + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_BUNDLE_TAG} + container_name: elastic + environment: + - "node.name=elastic" + - "http.port=9200" + - "cluster.name=es-docker-cluster" + - "discovery.type=single-node" + - "bootstrap.memory_lock=true" + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: 65536 + volumes: + - "es_data:/usr/share/elasticsearch/data" + network_mode: "host" + + # kibana instance and collectors + # see https://www.elastic.co/guide/en/elastic-stack-get-started/current/get-started-docker.html + kibana: + image: docker.elastic.co/kibana/kibana:${ELASTIC_BUNDLE_TAG} + container_name: kibana + environment: + SERVER_NAME: "kibana.local" + # ELASTICSEARCH_URL: "http://localhost:9700" + ELASTICSEARCH_HOSTS: "http://localhost:9200" + depends_on: [ 'elastic' ] + network_mode: "host" + + apm: + image: docker.elastic.co/apm/apm-server:${ELASTIC_BUNDLE_TAG} + container_name: apm + volumes: + - "./apm-config.yaml:/usr/share/apm-server/apm-server.yml:ro" + depends_on: [ 'elastic' ] + network_mode: "host" + + grafana: + # see https://grafana.com/docs/grafana/latest/installation/docker/ + image: "grafana/grafana:8.3.5" + container_name: grafana + # restart: unless-stopped + environment: + - "GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel,grafana-worldmap-panel,grafana-polystat-panel" + network_mode: "host" + volumes: + # chown 472:472 if needed + - grafana:/var/lib/grafana + - ./grafana/provisioning/:/etc/grafana/provisioning/ + +volumes: + es_data: + driver: local + grafana: + driver: local + metricbeat: + driver: local diff --git a/script/telemetry/elastic/grafana/provisioning/datasources/elastic.yaml b/script/telemetry/elastic/grafana/provisioning/datasources/elastic.yaml new file mode 100644 index 000000000..e46e7ac97 --- /dev/null +++ b/script/telemetry/elastic/grafana/provisioning/datasources/elastic.yaml @@ -0,0 +1,20 @@ +apiVersion: 1 + +datasources: + - name: DS_ELASTICSEARCH + type: elasticsearch + access: proxy + url: http://elastic:9700 + password: '' + user: '' + database: metricbeat-* + basicAuth: false + isDefault: true + jsonData: + esVersion: 70 + logLevelField: '' + logMessageField: '' + maxConcurrentShardRequests: 5 + timeField: "@timestamp" + timeInterval: 10s + readOnly: false diff --git a/script/telemetry/elastic/otel-config.yaml b/script/telemetry/elastic/otel-config.yaml new file mode 100644 index 000000000..cd119f0ab --- /dev/null +++ b/script/telemetry/elastic/otel-config.yaml @@ -0,0 +1,47 @@ +receivers: + # Data sources: metrics, traces + otlp: + protocols: + grpc: + endpoint: ":4317" + http: + endpoint: ":55681" + # Data sources: metrics + prometheus: + config: + scrape_configs: + - job_name: "garage" + scrape_interval: 5s + static_configs: + - targets: ["localhost:3909"] + +exporters: + logging: + logLevel: info + # see https://www.elastic.co/guide/en/apm/get-started/current/open-telemetry-elastic.html#open-telemetry-collector + otlp/elastic: + endpoint: "localhost:8200" + tls: + insecure: true + +processors: + batch: + +extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + +service: + extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging, otlp/elastic] + metrics: + receivers: [otlp, prometheus] + processors: [batch] + exporters: [logging, otlp/elastic]