Deuxfleurs/nixcfg
14
7
Fork 2
Code Issues 19 Pull requests 2 Projects Releases Wiki Activity

staging: harden config of smartctl exporter

Browse source 
It currently requires all nodes to have /dev/sda (the device passthrough is hardcoded for now)
This commit is contained in:
Baptiste Jonglez 2025-03-19 23:46:32 +01:00
parent 5790453ff1
commit 8d9aa00de5
3 changed files with 249 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

170
cluster/staging/app/telemetry/config/smartctl-seccomp.json Normal file
View file

@ -0,0 +1,170 @@
{
"defaultAction": "SCMP_ACT_ERRNO",
"defaultErrnoRet": 1,
"architectures": [
"SCMP_ARCH_X86_64"
],
"syscalls": [
{
"names": [
"rt_sigaction",
"rt_sigprocmask",
"getpid",
"fcntl",
"fstatfs",
"gettid",
"futex",
"getdents64",
"epoll_ctl",
"tgkill",
"openat",
"read",
"close",
"nanosleep",
"getsockname",
"setsockopt",
"chdir",
"capget",
"prctl",
"accept4",
"fstat",
"getcwd",
"setuid",
"setgid",
"setgroups",
"capset",
"newfstatat",
"write",
"writev",
"mmap",
"brk",
"rt_sigreturn",
"access",
"execve",
"getppid",
"exit_group",
"faccessat2",
"mprotect",
"pread64",
"arch_prctl",
"set_tid_address",
"set_robust_list",
"rseq",
"munmap",
"madvise",
"sigaltstack",
"statfs",
"waitid",
"readlinkat",
"eventfd2",
"epoll_create1",
"pipe2",
"pidfd_send_signal",
"pidfd_open",
"readlink",
"epoll_pwait",
"dup3",
"bind",
"listen",
"getrliimt",
"sched_getaffinity",
"sched_yield"
],
"action": "SCMP_ACT_ALLOW",
"comment": "globally needed by the go runtime"
},
{
"names": [
"open",
"uname"
],
"action": "SCMP_ACT_ALLOW",
"comment": "Used by smartctl"
},
{
"names": [
"ioctl"
],
"action": "SCMP_ACT_ALLOW",
"comment": "allow SG_IO (aka SCSCI commands) on ioctl as it's what's used to read SMART data",
"args": [
{
"index": 1,
"value": 8837,
"op": "SCMP_CMP_EQ"
}
]
},
{
"names": [
"ioctl"
],
"action": "SCMP_ACT_ALLOW",
"comment": "allow NVME_IOCTL_ID command (0x4e40) on ioctl as it's what's used to read data on NVMe devices",
"args": [
{
"index": 1,
"value": 20032,
"op": "SCMP_CMP_EQ"
}
]
},
{
"names": [
"ioctl"
],
"action": "SCMP_ACT_ALLOW",
"comment": "allow NVME_IOCTL_ADMIN_CMD command (0xc0484e41) on ioctl as it's what's used to read data on NVMe devices. For some reason, it needs to be encoded as 0xffffffffc0484e41",
"args": [
{
"index": 1,
"value": 18446744072640548417,
"op": "SCMP_CMP_EQ"
}
]
},
{
"names": [
"ioctl"
],
"action": "SCMP_ACT_ERRNO",
"comment": "Debug to allow/deny all ioctl (change to _LOG, _ALLOW, or _ERRNO appropriately)"
},
{
"names": [
"clone"
],
"action": "SCMP_ACT_ALLOW",
"comment": "partially allow clone as per docker config",
"args": [
{
"index": 0,
"value": 2114060288,
"op": "SCMP_CMP_MASKED_EQ"
}
]
},
{
"names": [
"clone3"
],
"action": "SCMP_ACT_ERRNO",
"comment": "disable clone3 in a specific way as per docker's default config",
"errnoRet": 38
},
{
"names": [
"socket"
],
"action": "SCMP_ACT_ALLOW",
"comment": "allow IPv4 sockets",
"args": [
{
"index": 0,
"value": 2,
"op": "SCMP_CMP_EQ"
}
]
}
]
}

47
cluster/staging/app/telemetry/deploy/telemetry-system.hcl → cluster/staging/app/telemetry/deploy/telemetry-node-exporter.hcl
View file

@ -1,4 +1,4 @@
job "telemetry-system" {
job "telemetry-node-exporter" {
datacenters = ["neptune", "dathomir", "corrin", "bespin"]
type = "system"
priority = "100"
@ -46,49 +46,4 @@ job "telemetry-system" {
}
}
}
group "smartctl_exporter" {
network {
port "smartctl_exporter" { static = 9101 }
}
task "smartctl_exporter" {
driver = "docker"
user = "root"
config {
image = "prometheuscommunity/smartctl-exporter:v0.13.0"
network_mode = "host"
privileged = true
args = [
"--web.listen-address=0.0.0.0:9101"
]
}
resources {
cpu = 50
memory = 40
}
service {
tags = [ "telemetry" ]
port = 9101
address_mode = "driver"
name = "smartctl-exporter"
check {
type = "http"
path = "/"
port = 9101
address_mode = "driver"
interval = "60s"
timeout = "5s"
check_restart {
limit = 3
grace = "90s"
ignore_warnings = false
}
}
}
}
}
}

78
cluster/staging/app/telemetry/deploy/telemetry-smartctl-exporter.hcl Normal file
View file

@ -0,0 +1,78 @@
job "telemetry-smartctl-exporter" {
datacenters = ["neptune", "dathomir", "corrin", "bespin"]
type = "system"
priority = "100"
group "smartctl_exporter" {
network {
port "smartctl_exporter" { static = 9101 }
}
task "smartctl_exporter" {
driver = "docker"
# Necessary to use low-level SMART and NVMe commands
user = "root"
config {
image = "prometheuscommunity/smartctl-exporter:v0.13.0"
args = [
"--web.listen-address=0.0.0.0:9101"
]
network_mode = "host"
# CAP_SYS_RAWIO is needed for SMART requests, while CAPS_SYS_ADMIN
# is needed for NVMe requests.
cap_drop = ["all"]
cap_add = ["CAP_SYS_RAWIO", "CAP_SYS_ADMIN"]
# Hardening options to avoid running the container as privileged,
# while still allowing just enough syscalls so that smartctl can query the disks.
security_opt = [
"no-new-privileges",
# Apparently there is no variable to determine the path to the allocation, hence this hack
"seccomp=/var/lib/nomad/alloc/${NOMAD_ALLOC_ID}/${NOMAD_TASK_NAME}/local/smartctl-seccomp.json",
]
readonly_rootfs = true
# Sadly, devices must exist for Docker to accept this option, so
# we can't declare all possible devices.
# This may help: https://docs.docker.com/reference/cli/docker/container/run/#device-cgroup-rule
devices = [
{
host_path = "/dev/sda"
container_path = "/dev/sda"
cgroup_permissions = "r"
}
]
}
template {
data = file("../config/smartctl-seccomp.json")
destination = "local/smartctl-seccomp.json"
perms = 444
}
resources {
cpu = 50
memory = 40
}
service {
tags = [ "telemetry" ]
port = 9101
address_mode = "driver"
name = "smartctl-exporter"
check {
type = "http"
path = "/"
port = 9101
address_mode = "driver"
interval = "60s"
timeout = "5s"
check_restart {
limit = 3
grace = "90s"
ignore_warnings = false
}
}
}
}
}
}