Run Garage as a Nomad Nix job on staging cluster
This commit is contained in:
parent
3c5f4b55e6
commit
945dd4fa9a
GPG key ID:
0E496D15096376BE
3 changed files with 238 additions and 18 deletions
218
cluster/staging/app/garage/deploy/garage-nix.hcl
Normal file
218
cluster/staging/app/garage/deploy/garage-nix.hcl
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
job "garage-staging" {
|
||||
type = "system"
|
||||
#datacenters = [ "neptune", "pluton" ]
|
||||
datacenters = [ "neptune" ]
|
||||
|
||||
priority = 80
|
||||
|
||||
constraint {
|
||||
attribute = "${attr.cpu.arch}"
|
||||
value = "amd64"
|
||||
}
|
||||
|
||||
group "garage-staging" {
|
||||
network {
|
||||
port "s3" { static = 3990 }
|
||||
port "rpc" { static = 3991 }
|
||||
port "web" { static = 3992 }
|
||||
port "k2v" { static = 3993 }
|
||||
port "admin" { static = 3909 }
|
||||
}
|
||||
|
||||
update {
|
||||
max_parallel = 1
|
||||
min_healthy_time = "30s"
|
||||
healthy_deadline = "5m"
|
||||
}
|
||||
|
||||
# task "repair" {
|
||||
# lifecycle {
|
||||
# hook = "prestart"
|
||||
# sidecar = false
|
||||
# }
|
||||
|
||||
# driver = "docker"
|
||||
|
||||
# config {
|
||||
# image = "dxflrs/amd64_garage:v0.7.99-k2v"
|
||||
# command = "/garage"
|
||||
# args = [ "offline-repair", "--yes", "object_counters" ]
|
||||
# network_mode = "host"
|
||||
# volumes = [
|
||||
# "/mnt/storage/garage-staging/data:/data",
|
||||
# "/mnt/ssd/garage-staging/meta:/meta",
|
||||
# "secrets/garage.toml:/etc/garage.toml",
|
||||
# ]
|
||||
# }
|
||||
|
||||
# template {
|
||||
# data = file("../config/garage.toml")
|
||||
# destination = "secrets/garage.toml"
|
||||
# }
|
||||
|
||||
# resources {
|
||||
# memory = 2000
|
||||
# cpu = 1000
|
||||
# }
|
||||
# }
|
||||
|
||||
task "server" {
|
||||
driver = "nix"
|
||||
|
||||
config {
|
||||
packages = [
|
||||
"git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c"
|
||||
]
|
||||
command = [ "/bin/garage", "server" ]
|
||||
bind = {
|
||||
"/mnt/storage/garage-staging/data" = "/data",
|
||||
"/mnt/ssd/garage-staging/meta" = "/meta",
|
||||
}
|
||||
}
|
||||
|
||||
template {
|
||||
data = file("../config/garage.toml")
|
||||
destination = "etc/garage.toml"
|
||||
}
|
||||
|
||||
template {
|
||||
data = "{{ key \"secrets/consul/consul-ca.crt\" }}"
|
||||
destination = "etc/garage/consul-ca.crt"
|
||||
}
|
||||
|
||||
template {
|
||||
data = "{{ key \"secrets/consul/consul-client.crt\" }}"
|
||||
destination = "etc/garage/consul-client.crt"
|
||||
}
|
||||
|
||||
template {
|
||||
data = "{{ key \"secrets/consul/consul-client.key\" }}"
|
||||
destination = "etc/garage/consul-client.key"
|
||||
}
|
||||
|
||||
template {
|
||||
data = <<EOH
|
||||
RUST_LOG=garage=debug
|
||||
EOH
|
||||
destination = "secrets/env"
|
||||
env = true
|
||||
}
|
||||
|
||||
resources {
|
||||
memory = 2000
|
||||
memory_max = 3000
|
||||
cpu = 500
|
||||
}
|
||||
|
||||
kill_signal = "SIGINT"
|
||||
kill_timeout = "20s"
|
||||
|
||||
service {
|
||||
tags = [
|
||||
"garage-staging-api",
|
||||
"tricot garage-staging.home.adnab.me",
|
||||
"tricot garage.staging.deuxfleurs.org",
|
||||
"tricot-add-header Access-Control-Allow-Origin *",
|
||||
]
|
||||
port = 3990
|
||||
address_mode = "driver"
|
||||
name = "garage-staging-api"
|
||||
check {
|
||||
type = "tcp"
|
||||
port = 3990
|
||||
address_mode = "driver"
|
||||
interval = "60s"
|
||||
timeout = "5s"
|
||||
check_restart {
|
||||
limit = 3
|
||||
grace = "90s"
|
||||
ignore_warnings = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
service {
|
||||
tags = [
|
||||
"garage-staging-k2v-api",
|
||||
"tricot k2v.staging.deuxfleurs.org",
|
||||
"tricot-add-header Access-Control-Allow-Origin *",
|
||||
]
|
||||
port = 3993
|
||||
address_mode = "driver"
|
||||
name = "garage-staging-k2v-api"
|
||||
# check {
|
||||
# type = "tcp"
|
||||
# port = 3993
|
||||
# address_mode = "driver"
|
||||
# interval = "60s"
|
||||
# timeout = "5s"
|
||||
# check_restart {
|
||||
# limit = 3
|
||||
# grace = "90s"
|
||||
# ignore_warnings = false
|
||||
# }
|
||||
# }
|
||||
}
|
||||
|
||||
service {
|
||||
tags = ["garage-staging-rpc"]
|
||||
port = 3991
|
||||
address_mode = "driver"
|
||||
name = "garage-staging-rpc"
|
||||
check {
|
||||
type = "tcp"
|
||||
port = 3991
|
||||
address_mode = "driver"
|
||||
interval = "60s"
|
||||
timeout = "5s"
|
||||
check_restart {
|
||||
limit = 3
|
||||
grace = "90s"
|
||||
ignore_warnings = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
service {
|
||||
tags = [
|
||||
"garage-staging-web",
|
||||
"tricot *.web.staging.deuxfleurs.org",
|
||||
"tricot staging.deuxfleurs.org",
|
||||
"tricot matrix.home.adnab.me/.well-known/matrix/server",
|
||||
"tricot-add-header Access-Control-Allow-Origin *",
|
||||
]
|
||||
port = 3992
|
||||
address_mode = "driver"
|
||||
name = "garage-staging-web"
|
||||
check {
|
||||
type = "tcp"
|
||||
port = 3992
|
||||
address_mode = "driver"
|
||||
interval = "60s"
|
||||
timeout = "5s"
|
||||
check_restart {
|
||||
limit = 3
|
||||
grace = "90s"
|
||||
ignore_warnings = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
service {
|
||||
tags = [
|
||||
"garage-staging-admin",
|
||||
]
|
||||
port = 3909
|
||||
address_mode = "driver"
|
||||
name = "garage-staging-admin"
|
||||
}
|
||||
|
||||
restart {
|
||||
interval = "30m"
|
||||
attempts = 10
|
||||
delay = "15s"
|
||||
mode = "delay"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -68,4 +68,24 @@
|
|||
|
||||
# For Garage ipv6 communication
|
||||
networking.firewall.allowedTCPPorts = [ 3991 ];
|
||||
|
||||
## -----
|
||||
|
||||
## EXPERIMENTAL ON STAGING: NIX NOMAD DRIVER
|
||||
services.nomad.dropPrivileges = false;
|
||||
services.nomad.extraPackages = [
|
||||
pkgs.nix
|
||||
pkgs.git
|
||||
];
|
||||
services.nomad.extraSettingsPlugins =
|
||||
let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
|
||||
in [ nomad_driver_nix ];
|
||||
|
||||
# the nix driver requires flakes to be enabled
|
||||
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||||
|
||||
# use our cache as additionnal substituer (we put precompiled packages there,
|
||||
# like we used to do on the docker hub)
|
||||
nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ];
|
||||
nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,22 +21,4 @@
|
|||
|
||||
deuxfleurs.cluster_ip = "10.14.1.1";
|
||||
deuxfleurs.is_raft_server = true;
|
||||
|
||||
# ----
|
||||
|
||||
# experimental: config for nomad nix driver
|
||||
services.nomad.dropPrivileges = false;
|
||||
services.nomad.extraPackages = [
|
||||
pkgs.nix
|
||||
pkgs.git
|
||||
];
|
||||
services.nomad.extraSettingsPlugins =
|
||||
let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
|
||||
in [ nomad_driver_nix ];
|
||||
# the nix driver requires flakes to be enabled
|
||||
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||||
|
||||
# use our cache as additionnal substituer
|
||||
nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ];
|
||||
nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ];
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue