nixcfg/app/telemetry/deploy/telemetry-system.hcl

job "telemetry-system" {
  datacenters = ["neptune"]
  type = "system"

  group "elasticsearch" {
    network {
      port "elastic" {
        static = 9200
      }
      port "elastic_internal" {
        static = 9300
      }
    }

    task "elastic" {
      driver = "docker"
      config {
        image = "docker.elastic.co/elasticsearch/elasticsearch:8.2.0"
        network_mode = "host"
        volumes = [
          "/mnt/ssd/telemetry/es_data:/usr/share/elasticsearch/data",
          "secrets/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12",
        ]
        ports = [ "elastic", "elastic_internal" ]
        sysctl = {
          #"vm.max_map_count" = "262144",
        }
        ulimit = {
          memlock = "9223372036854775807:9223372036854775807", 
        }
      }

      user = "1000"

      resources {
        memory = 1500
        cpu = 500
      }

      template {
        data = "{{ key \"secrets/telemetry/elasticsearch/elastic-certificates.p12\" }}"
        destination = "secrets/elastic-certificates.p12"
      }

      template {
        data = <<EOH
node.name={{ env "attr.unique.hostname" }}
http.port=9200
transport.port=9300
cluster.name=es-deuxfleurs
cluster.initial_master_nodes=carcajou,caribou,cariacou
discovery.seed_hosts=carcajou,caribou,cariacou
bootstrap.memory_lock=true
xpack.security.enabled=true
xpack.security.authc.api_key.enabled=true
xpack.security.transport.ssl.enabled=true
xpack.security.transport.ssl.verification_mode=certificate
xpack.security.transport.ssl.client_authentication=required
xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
cluster.routing.allocation.disk.watermark.high=75%
cluster.routing.allocation.disk.watermark.low=65%
ES_JAVA_OPTS=-Xms512M -Xmx512M
EOH
        destination = "secrets/env"
        env = true
      }
    }
  }

  group "collector" {
    network {
      port "otel_grpc" {
        static = 4317
      }
      port "apm" {
        static = 8200
      }
      port "node_exporter" {
        static = 9100
      }
    }

    task "otel" {
      driver = "docker"
      config {
        image = "otel/opentelemetry-collector-contrib:0.46.0"
        args = [
          "--config=/etc/otel-config.yaml",
        ]
        network_mode = "host"
        ports= [ "otel_grpc" ]
        volumes = [
          "secrets/otel-config.yaml:/etc/otel-config.yaml"
        ]
      }

      template {
        data = file("../config/otel-config.yaml")
        destination = "secrets/otel-config.yaml"
      }

      resources {
        memory = 100
        cpu = 100
      }
    }

    task "apm" {
      driver = "docker"
      config {
        image = "docker.elastic.co/apm/apm-server:8.2.0"
        network_mode = "host"
        ports = [ "apm" ]
        args = [ "--strict.perms=false" ]
        volumes = [
          "secrets/apm-config.yaml:/usr/share/apm-server/apm-server.yml:ro"
        ]
      }

      template {
        data = file("../config/apm-config.yaml")
        destination = "secrets/apm-config.yaml"
      }

      resources {
        memory = 100
        cpu = 100
      }
    }

/*
    task "node_exporter" {
      driver = "docker"
      config {
        image = "quay.io/prometheus/node-exporter:v1.1.2"
        network_mode = "host"
        ports = [ "node_exporter" ]
        volumes = [
          "/:/host:ro,rslave"
        ]
        args = [ "--path.rootfs=/host" ]
      }

      resources {
        cpu = 50
        memory = 40
      }
    }
*/

    task "filebeat" {
      driver = "docker"
      config {
        image = "docker.elastic.co/beats/filebeat:8.2.0"
        network_mode = "host"
        volumes = [
          "/mnt/ssd/telemetry/filebeat:/usr/share/filebeat/data",
          "secrets/filebeat.yml:/usr/share/filebeat/filebeat.yml",
          "/var/run/docker.sock:/var/run/docker.sock",
          "/var/lib/docker/containers/:/var/lib/docker/containers/:ro",
          "/var/log/:/var/log/:ro",
        ]
        args = [ "--strict.perms=false" ]
        privileged = true
      }
      user = "root"


      template {
        data = file("../config/filebeat.yml")
        destination = "secrets/filebeat.yml"
      }

      resources {
        memory = 100
        cpu = 100
      }
    }
  }
}