Complete telemetry configuration

cluster/prod/app/telemetry/config/grafana-ldap.toml

@@ -0,0 +1,49 @@
+[[servers]]
+# Ldap server host (specify multiple hosts space separated)
+host = "bottin.service.prod.consul"
+# Default port is 389 or 636 if use_ssl = true
+port = 389
+# Set to true if LDAP server should use an encrypted TLS connection (either with STARTTLS or LDAPS)
+use_ssl = false
+# If set to true, use LDAP with STARTTLS instead of LDAPS
+start_tls = false
+# set to true if you want to skip SSL cert validation
+ssl_skip_verify = false
+# set to the path to your root CA certificate or leave unset to use system defaults
+# root_ca_cert = "/path/to/certificate.crt"
+# Authentication against LDAP servers requiring client certificates
+# client_cert = "/path/to/client.crt"
+# client_key = "/path/to/client.key"
+
+# Search user bind dn
+bind_dn = "cn=grafana,ou=services,ou=users,dc=deuxfleurs,dc=fr"
+# Search user bind password
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+bind_password = "{{ key "secrets/telemetry/grafana/grafana_ldap_password" | trimSpace }}"
+
+# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)"
+# Allow login from email or username, example "(|(sAMAccountName=%s)(userPrincipalName=%s))"
+search_filter = "(cn=%s)"
+
+# An array of base dns to search through
+search_base_dns = ["ou=users,dc=deuxfleurs,dc=fr"]
+
+# group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))"
+# group_search_filter_user_attribute = "distinguishedName"
+# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"]
+
+[[servers.group_mappings]]
+group_dn = "cn=admin,ou=groups,dc=deuxfleurs,dc=fr"
+org_role = "Admin"
+grafana_admin = true
+
+[[servers.group_mappings]]
+group_dn = "*"
+org_role = "Viewer"
+
+# Specify names of the LDAP attributes your LDAP uses
+[servers.attributes]
+member_of = "memberof"
+email =  "mail"
+username = "cn"
+uid = "cn"

cluster/prod/app/telemetry/config/prometheus.yml

@@ -41,3 +41,27 @@ scrape_configs:
         ca_file: /etc/prometheus/consul.crt
         cert_file: /etc/prometheus/consul-client.crt
         key_file: /etc/prometheus/consul-client.key
+
+  # see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config
+  # and https://www.nomadproject.io/api-docs/metrics
+  # and https://learn.hashicorp.com/tutorials/nomad/prometheus-metrics
+  # dashboard at https://grafana.com/grafana/dashboards/3800
+  - job_name: 'nomad'
+    scrape_interval: 10s
+    metrics_path: "/v1/metrics"
+    params:
+      format: ['prometheus']
+    scheme: 'https'
+    tls_config:
+      ca_file: /etc/prometheus/nomad.crt
+      cert_file: /etc/prometheus/nomad-client.crt
+      key_file: /etc/prometheus/nomad-client.key
+      insecure_skip_verify: true
+    consul_sd_configs:
+    - server: 'https://localhost:8501'
+      services:
+        - 'nomad-client'
+      tls_config:
+        ca_file: /etc/prometheus/consul.crt
+        cert_file: /etc/prometheus/consul-client.crt
+        key_file: /etc/prometheus/consul-client.key

cluster/prod/app/telemetry/deploy/telemetry-system.hcl

@@ -1,5 +1,5 @@
 job "telemetry-system" {
-	datacenters = ["neptune", "orion"]
+	datacenters = ["neptune", "orion", "bespin"]
 	type = "system"
 	priority = "100"
 
@@ -12,7 +12,7 @@ job "telemetry-system" {
 			driver = "docker"
 
 			config {
-				image = "quay.io/prometheus/node-exporter:v1.1.2"
+				image = "quay.io/prometheus/node-exporter:v1.4.0"
 				network_mode = "host"
 				volumes = [
 					"/:/host:ro,rslave"

cluster/prod/app/telemetry/deploy/telemetry.hcl

@@ -1,9 +1,9 @@
 job "telemetry" {
-  datacenters = ["neptune"]
+  datacenters = ["neptune", "bespin"]
   type = "service"
 
   group "prometheus" {
-    count = 1
+    count = 2
 
     network {
       port "prometheus" {
@@ -11,14 +11,26 @@ job "telemetry" {
       }
     }
 
+    constraint {
+      attribute = "${attr.unique.hostname}"
+      operator = "set_contains_any"
+      value = "concombre,df-ymk"
+    }
+
     task "prometheus" {
       driver = "docker"
       config {
-        image = "prom/prometheus:v2.38.0"
+        image = "prom/prometheus:v2.39.0"
         network_mode = "host"
         ports = [ "prometheus" ]
+        args = [
+          "--config.file=/etc/prometheus/prometheus.yml",
+          "--storage.tsdb.path=/data",
+          "--storage.tsdb.retention.size=20GB",
+        ]
         volumes = [
-          "secrets:/etc/prometheus"
+          "secrets:/etc/prometheus",
+          "/mnt/ssd/prometheus:/data"
         ]
       }
 
@@ -42,6 +54,21 @@ job "telemetry" {
         destination = "secrets/consul-client.key"
       }
 
+      template {
+        data = "{{ key \"secrets/nomad/nomad.crt\" }}"
+        destination = "secrets/nomad.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/nomad/nomad-client.crt\" }}"
+        destination = "secrets/nomad-client.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/nomad/nomad-client.key\" }}"
+        destination = "secrets/nomad-client.key"
+      }
+
       resources {
         memory = 501
         cpu = 500
@@ -110,12 +137,13 @@ job "telemetry" {
     task "grafana" {
       driver = "docker"
       config {
-        image = "grafana/grafana:8.4.3"
+        image = "grafana/grafana:9.2.0"
         network_mode = "host"
         ports = [ "grafana" ]
         volumes = [
           "../alloc/data:/var/lib/grafana",
-          "secrets/prometheus.yaml:/etc/grafana/provisioning/datasources/prometheus.yaml"
+          "secrets/prometheus.yaml:/etc/grafana/provisioning/datasources/prometheus.yaml",
+          "secrets/ldap.toml:/etc/grafana/ldap.toml"
         ]
       }
 
@@ -124,10 +152,16 @@ job "telemetry" {
         destination = "secrets/prometheus.yaml"
       }
 
+      template {
+        data = file("../config/grafana-ldap.toml")
+        destination = "secrets/ldap.toml"
+      }
+
       template {
         data = <<EOH
 GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel,grafana-worldmap-panel,grafana-polystat-panel
 GF_SERVER_HTTP_PORT=3719
+GF_AUTH_LDAP_ENABLED=true
 EOH
         destination = "secrets/env"
         env = true

deploy_pki

@@ -18,6 +18,7 @@ cmd systemctl restart consul
 cmd sleep 10
 
 for file in nomad-ca.crt nomad$YEAR.crt nomad$YEAR.key \
+	nomad$YEAR-client.crt nomad$YEAR-client.key \
 	consul$YEAR.crt consul$YEAR-client.crt consul$YEAR-client.key
 do
 	if pass $PKI/$file >/dev/null; then
@@ -37,3 +38,8 @@ cmd "consul kv put secrets/consul/consul-ca.crt - < /var/lib/consul/pki/consul-c
 cmd "consul kv put secrets/consul/consul.crt - < /var/lib/consul/pki/consul$YEAR.crt"
 cmd "consul kv put secrets/consul/consul-client.crt - < /var/lib/consul/pki/consul$YEAR-client.crt"
 cmd "consul kv put secrets/consul/consul-client.key - < /var/lib/consul/pki/consul$YEAR-client.key"
+
+cmd "consul kv put secrets/nomad/nomad-ca.crt - < /var/lib/private/nomad/pki/nomad-ca.crt"
+cmd "consul kv put secrets/nomad/nomad.crt - < /var/lib/private/nomad/pki/nomad$YEAR.crt"
+cmd "consul kv put secrets/nomad/nomad-client.crt - < /var/lib/private/nomad/pki/nomad$YEAR-client.crt"
+cmd "consul kv put secrets/nomad/nomad-client.key - < /var/lib/private/nomad/pki/nomad$YEAR-client.key"

nix/deuxfleurs.nix

@@ -305,6 +305,11 @@ in
           "public_ipv6" = cfg.ipv6;
         };
       };
+      telemetry = {
+        publish_allocation_metrics = true;
+        publish_node_metrics = true;
+        prometheus_metrics = true;
+      };
       tls = {
         http = true;
         rpc = true;