staging: enable telemetry for nomad allocations

cluster/staging/app/telemetry/config/prometheus.yml

@@ -38,3 +38,27 @@ scrape_configs:
         ca_file: /etc/prom/consul.crt
         cert_file: /etc/prom/consul-client.crt
         key_file: /etc/prom/consul-client.key
+
+  # see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config
+  # and https://www.nomadproject.io/api-docs/metrics
+  # and https://learn.hashicorp.com/tutorials/nomad/prometheus-metrics
+  # dashboard at https://grafana.com/grafana/dashboards/3800
+  - job_name: 'nomad'
+    scrape_interval: 10s
+    metrics_path: "/v1/metrics"
+    params:
+      format: ['prometheus']
+    scheme: 'https'
+    tls_config:
+      ca_file: /etc/prom/nomad-ca.crt
+      cert_file: /etc/prom/nomad-client.crt
+      key_file: /etc/prom/nomad-client.key
+      insecure_skip_verify: true
+    consul_sd_configs:
+    - server: 'https://localhost:8501'
+      services:
+        - 'nomad-client'
+      tls_config:
+        ca_file: /etc/prom/consul.crt
+        cert_file: /etc/prom/consul-client.crt
+        key_file: /etc/prom/consul-client.key

cluster/staging/app/telemetry/deploy/telemetry-service.hcl

@@ -53,6 +53,21 @@ job "telemetry-service" {
         destination = "etc/prom/consul-client.key"
       }
 
+      template {
+        data = "{{ key \"secrets/nomad/nomad-ca.crt\" }}"
+        destination = "etc/prom/nomad-ca.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/nomad/nomad-client.crt\" }}"
+        destination = "etc/prom/nomad-client.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/nomad/nomad-client.key\" }}"
+        destination = "etc/prom/nomad-client.key"
+      }
+
       resources {
         memory = 500
         cpu = 200