2023-01-01 19:44:28 +00:00
|
|
|
job "telemetry-service" {
|
2023-01-04 09:06:06 +00:00
|
|
|
datacenters = ["neptune", "jupiter", "corrin", "bespin"]
|
2022-02-17 14:06:09 +00:00
|
|
|
type = "service"
|
|
|
|
|
2022-09-20 15:13:46 +00:00
|
|
|
group "prometheus" {
|
2022-10-18 08:32:41 +00:00
|
|
|
count = 2
|
2022-02-17 14:06:09 +00:00
|
|
|
|
|
|
|
network {
|
2022-09-20 15:13:46 +00:00
|
|
|
port "prometheus" {
|
|
|
|
static = 9090
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-10-18 08:32:41 +00:00
|
|
|
constraint {
|
|
|
|
attribute = "${attr.unique.hostname}"
|
|
|
|
operator = "set_contains_any"
|
2024-01-31 08:33:12 +00:00
|
|
|
value = "df-pw5,origan"
|
2022-10-18 08:32:41 +00:00
|
|
|
}
|
|
|
|
|
2022-09-20 15:13:46 +00:00
|
|
|
task "prometheus" {
|
2022-12-01 22:48:46 +00:00
|
|
|
driver = "nix2"
|
2022-02-17 14:06:09 +00:00
|
|
|
config {
|
2022-12-01 22:48:46 +00:00
|
|
|
nixpkgs = "github:nixos/nixpkgs/nixos-22.11"
|
|
|
|
packages = [ "#prometheus", "#coreutils", "#findutils", "#bash" ]
|
|
|
|
command = "prometheus"
|
2022-10-18 08:32:41 +00:00
|
|
|
args = [
|
2022-12-01 22:48:46 +00:00
|
|
|
"--config.file=/etc/prom/prometheus.yml",
|
2022-10-18 08:32:41 +00:00
|
|
|
"--storage.tsdb.path=/data",
|
2022-11-04 10:09:37 +00:00
|
|
|
"--storage.tsdb.retention.size=5GB",
|
2022-10-18 08:32:41 +00:00
|
|
|
]
|
2022-12-01 22:48:46 +00:00
|
|
|
bind = {
|
|
|
|
"/mnt/ssd/prometheus" = "/data"
|
|
|
|
}
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template {
|
2022-09-20 15:13:46 +00:00
|
|
|
data = file("../config/prometheus.yml")
|
2022-12-01 22:48:46 +00:00
|
|
|
destination = "etc/prom/prometheus.yml"
|
2022-09-20 15:13:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template {
|
2024-03-28 10:55:58 +00:00
|
|
|
data = "{{ key \"secrets/consul/consul-ca.crt\" }}"
|
2022-12-01 22:48:46 +00:00
|
|
|
destination = "etc/prom/consul.crt"
|
2022-09-20 15:13:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template {
|
|
|
|
data = "{{ key \"secrets/consul/consul-client.crt\" }}"
|
2022-12-01 22:48:46 +00:00
|
|
|
destination = "etc/prom/consul-client.crt"
|
2022-09-20 15:13:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template {
|
|
|
|
data = "{{ key \"secrets/consul/consul-client.key\" }}"
|
2022-12-01 22:48:46 +00:00
|
|
|
destination = "etc/prom/consul-client.key"
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
resources {
|
2022-09-20 15:13:46 +00:00
|
|
|
memory = 500
|
2022-09-26 13:53:32 +00:00
|
|
|
cpu = 200
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
|
|
|
|
2022-02-26 19:31:57 +00:00
|
|
|
service {
|
2022-12-01 22:48:46 +00:00
|
|
|
port = "prometheus"
|
2022-09-20 15:13:46 +00:00
|
|
|
name = "prometheus"
|
2022-02-26 19:31:57 +00:00
|
|
|
check {
|
2022-09-20 15:13:46 +00:00
|
|
|
type = "http"
|
|
|
|
path = "/"
|
2022-02-26 19:31:57 +00:00
|
|
|
interval = "60s"
|
|
|
|
timeout = "5s"
|
|
|
|
check_restart {
|
|
|
|
limit = 3
|
|
|
|
grace = "90s"
|
|
|
|
ignore_warnings = false
|
|
|
|
}
|
|
|
|
}
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
2022-02-26 19:31:57 +00:00
|
|
|
}
|
|
|
|
}
|
2022-09-20 15:13:46 +00:00
|
|
|
|
2022-02-26 19:31:57 +00:00
|
|
|
group "grafana" {
|
|
|
|
count = 1
|
2022-02-17 14:06:09 +00:00
|
|
|
|
2024-04-30 22:15:35 +00:00
|
|
|
constraint {
|
|
|
|
attribute = "${attr.unique.hostname}"
|
|
|
|
operator = "!="
|
|
|
|
value = "piranha"
|
|
|
|
}
|
|
|
|
|
2022-02-26 19:31:57 +00:00
|
|
|
network {
|
|
|
|
port "grafana" {
|
2022-09-20 15:13:46 +00:00
|
|
|
static = 3719
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-02-26 20:31:58 +00:00
|
|
|
task "restore-db" {
|
|
|
|
lifecycle {
|
|
|
|
hook = "prestart"
|
|
|
|
sidecar = false
|
|
|
|
}
|
|
|
|
|
2022-12-01 23:14:31 +00:00
|
|
|
driver = "nix2"
|
2022-02-26 20:31:58 +00:00
|
|
|
config {
|
2022-12-01 23:14:31 +00:00
|
|
|
packages = [ "#litestream" ]
|
|
|
|
command = "litestream"
|
2022-02-26 20:31:58 +00:00
|
|
|
args = [
|
|
|
|
"restore", "-config", "/etc/litestream.yml", "/ephemeral/grafana.db"
|
|
|
|
]
|
2022-12-01 23:14:31 +00:00
|
|
|
bind = {
|
|
|
|
"../alloc/data" = "/ephemeral",
|
|
|
|
}
|
2022-02-26 20:31:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template {
|
|
|
|
data = file("../config/grafana-litestream.yml")
|
2022-12-01 23:14:31 +00:00
|
|
|
destination = "etc/litestream.yml"
|
2022-02-26 20:31:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
resources {
|
2022-09-26 13:53:32 +00:00
|
|
|
memory = 100
|
2022-11-04 10:09:37 +00:00
|
|
|
memory_max = 1000
|
2022-09-26 13:53:32 +00:00
|
|
|
cpu = 100
|
2022-02-26 20:31:58 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-02-17 14:06:09 +00:00
|
|
|
task "grafana" {
|
2022-12-01 23:14:31 +00:00
|
|
|
driver = "nix2"
|
2022-02-17 14:06:09 +00:00
|
|
|
config {
|
2022-12-01 23:14:31 +00:00
|
|
|
nixpkgs = "github:nixos/nixpkgs/nixos-22.11"
|
|
|
|
packages = [ "#grafana" ]
|
|
|
|
command = "grafana-server"
|
|
|
|
args = [
|
|
|
|
"-homepath", "/share/grafana",
|
|
|
|
"cfg:default.paths.data=/grafana",
|
|
|
|
"cfg:default.paths.provisioning=/grafana-provisioning"
|
2022-02-17 14:06:09 +00:00
|
|
|
]
|
2022-12-01 23:14:31 +00:00
|
|
|
|
|
|
|
bind = {
|
|
|
|
"../alloc/data" = "/grafana",
|
|
|
|
}
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template {
|
2022-09-20 15:13:46 +00:00
|
|
|
data = file("../config/grafana-datasource-prometheus.yaml")
|
2022-12-01 23:14:31 +00:00
|
|
|
destination = "grafana-provisioning/datasources/prometheus.yaml"
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template {
|
|
|
|
data = <<EOH
|
2023-01-01 19:44:28 +00:00
|
|
|
GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel,grafana-worldmap-panel,grafana-polystat-panel
|
|
|
|
GF_SERVER_HTTP_PORT=3719
|
|
|
|
GF_SECURITY_ADMIN_PASSWORD={{ key "secrets/telemetry/grafana/admin_password" }}
|
2022-09-26 13:53:32 +00:00
|
|
|
EOH
|
|
|
|
destination = "secrets/env"
|
|
|
|
env = true
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
resources {
|
2022-09-26 13:53:32 +00:00
|
|
|
memory = 300
|
2023-01-03 17:25:32 +00:00
|
|
|
cpu = 300
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
2022-02-26 20:31:58 +00:00
|
|
|
|
2023-01-01 19:44:28 +00:00
|
|
|
restart {
|
|
|
|
interval = "30s"
|
|
|
|
attempts = 10
|
|
|
|
delay = "1m"
|
|
|
|
mode = "delay"
|
|
|
|
}
|
|
|
|
|
2022-02-26 20:31:58 +00:00
|
|
|
service {
|
2022-12-01 23:14:31 +00:00
|
|
|
name = "grafana"
|
2022-02-26 20:31:58 +00:00
|
|
|
tags = [
|
2022-09-26 13:53:32 +00:00
|
|
|
"grafana",
|
|
|
|
"tricot grafana.staging.deuxfleurs.org",
|
2022-12-11 20:27:16 +00:00
|
|
|
"d53-cname grafana.staging.deuxfleurs.org",
|
2022-02-26 20:31:58 +00:00
|
|
|
]
|
2022-12-01 23:14:31 +00:00
|
|
|
port = "grafana"
|
2022-02-26 20:31:58 +00:00
|
|
|
check {
|
|
|
|
type = "tcp"
|
|
|
|
interval = "60s"
|
|
|
|
timeout = "5s"
|
|
|
|
check_restart {
|
|
|
|
limit = 3
|
|
|
|
grace = "90s"
|
|
|
|
ignore_warnings = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
|
|
|
|
2022-02-26 20:31:58 +00:00
|
|
|
task "replicate-db" {
|
2022-12-01 23:14:31 +00:00
|
|
|
driver = "nix2"
|
2022-02-26 20:31:58 +00:00
|
|
|
config {
|
2022-12-01 23:14:31 +00:00
|
|
|
packages = [ "#litestream" ]
|
|
|
|
command = "litestream"
|
2022-02-26 20:31:58 +00:00
|
|
|
args = [
|
|
|
|
"replicate", "-config", "/etc/litestream.yml"
|
|
|
|
]
|
2022-12-01 23:14:31 +00:00
|
|
|
bind = {
|
|
|
|
"../alloc/data" = "/ephemeral",
|
|
|
|
}
|
2022-02-26 20:31:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template {
|
|
|
|
data = file("../config/grafana-litestream.yml")
|
2022-12-01 23:14:31 +00:00
|
|
|
destination = "etc/litestream.yml"
|
2022-02-26 20:31:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
resources {
|
2022-09-26 13:53:32 +00:00
|
|
|
memory = 100
|
2022-11-04 10:09:37 +00:00
|
|
|
memory_max = 500
|
2022-02-26 20:31:58 +00:00
|
|
|
cpu = 100
|
|
|
|
}
|
|
|
|
}
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|
2022-09-26 13:53:32 +00:00
|
|
|
|
2023-10-03 20:56:33 +00:00
|
|
|
/*
|
2022-09-26 13:53:32 +00:00
|
|
|
group "jaeger" {
|
|
|
|
count = 1
|
|
|
|
|
|
|
|
network {
|
|
|
|
port "jaeger-frontend" {
|
|
|
|
to = 16686
|
|
|
|
}
|
|
|
|
port "jaeger-otlp-grpc" {
|
|
|
|
static = 4317
|
|
|
|
to = 4317
|
|
|
|
}
|
|
|
|
port "jaeger-otlp-http" {
|
|
|
|
static = 4318
|
|
|
|
to = 4318
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
task "jaeger" {
|
|
|
|
driver = "docker"
|
|
|
|
config {
|
|
|
|
image = "jaegertracing/all-in-one:1.36"
|
|
|
|
ports = [ "jaeger-frontend", "jaeger-otlp-grpc", "jaeger-otlp-http" ]
|
|
|
|
}
|
|
|
|
resources {
|
2023-01-03 17:25:32 +00:00
|
|
|
memory = 1000
|
|
|
|
cpu = 500
|
2022-09-26 13:53:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
template {
|
|
|
|
data = <<EOH
|
|
|
|
COLLECTOR_OTLP_ENABLED=true
|
|
|
|
EOH
|
|
|
|
destination = "secrets/env"
|
|
|
|
env = true
|
|
|
|
}
|
|
|
|
|
|
|
|
service {
|
|
|
|
port = "jaeger-frontend"
|
|
|
|
address_mode = "host"
|
|
|
|
name = "jaeger-frontend"
|
2022-12-07 15:35:21 +00:00
|
|
|
tags = [
|
|
|
|
"tricot jaeger.staging.deuxfleurs.org",
|
2022-12-11 20:27:16 +00:00
|
|
|
"d53-cname jaeger.staging.deuxfleurs.org",
|
2022-12-07 15:35:21 +00:00
|
|
|
]
|
2022-09-26 13:53:32 +00:00
|
|
|
check {
|
|
|
|
type = "tcp"
|
|
|
|
port = "jaeger-frontend"
|
|
|
|
address_mode = "host"
|
|
|
|
interval = "60s"
|
|
|
|
timeout = "5s"
|
|
|
|
check_restart {
|
|
|
|
limit = 3
|
|
|
|
grace = "90s"
|
|
|
|
ignore_warnings = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
service {
|
|
|
|
port = "jaeger-otlp-grpc"
|
|
|
|
address_mode = "host"
|
|
|
|
name = "jaeger-otlp-grpc"
|
|
|
|
check {
|
|
|
|
type = "tcp"
|
|
|
|
port = "jaeger-otlp-grpc"
|
|
|
|
address_mode = "host"
|
|
|
|
interval = "60s"
|
|
|
|
timeout = "5s"
|
|
|
|
check_restart {
|
|
|
|
limit = 3
|
|
|
|
grace = "90s"
|
|
|
|
ignore_warnings = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
service {
|
|
|
|
port = "jaeger-otlp-http"
|
|
|
|
address_mode = "host"
|
|
|
|
name = "jaeger-otlp-http"
|
|
|
|
check {
|
|
|
|
type = "tcp"
|
|
|
|
port = "jaeger-otlp-grpc"
|
|
|
|
address_mode = "host"
|
|
|
|
interval = "60s"
|
|
|
|
timeout = "5s"
|
|
|
|
check_restart {
|
|
|
|
limit = 3
|
|
|
|
grace = "90s"
|
|
|
|
ignore_warnings = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-10-03 20:56:33 +00:00
|
|
|
*/
|
2022-02-17 14:06:09 +00:00
|
|
|
}
|