Compare commits
No commits in common. "main" and "driver-exec2" have entirely different histories.
main
...
driver-exe
16 changed files with 236 additions and 465 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -1,2 +1,2 @@
|
|||
nomad-driver-*
|
||||
*-driver
|
||||
nomad-driver-exec2
|
||||
exec2-driver
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
PLUGIN_BINARY=nix2-driver
|
||||
PLUGIN_BINARY=hello-driver
|
||||
export GO111MODULE=on
|
||||
|
||||
default: build
|
||||
|
|
|
|||
79
README.md
79
README.md
|
|
@ -1,33 +1,84 @@
|
|||
Nomad Nix Driver Plugin
|
||||
Nomad Skeleton Driver Plugin
|
||||
==========
|
||||
|
||||
A Nomad driver to run Nix jobs.
|
||||
Uses the same isolation mechanism as the `exec` driver.
|
||||
Partially based on [`nomad-driver-nix`](https://github.com/input-output-hk/nomad-driver-nix)
|
||||
Skeleton project for
|
||||
[Nomad task driver plugins](https://www.nomadproject.io/docs/drivers/index.html).
|
||||
|
||||
This project is intended for bootstrapping development of a new task driver
|
||||
plugin.
|
||||
|
||||
- Website: [https://www.nomadproject.io](https://www.nomadproject.io)
|
||||
- Mailing list: [Google Groups](http://groups.google.com/group/nomad-tool)
|
||||
|
||||
Requirements
|
||||
-------------------
|
||||
|
||||
- [Go](https://golang.org/doc/install) v1.19 or later (to compile the plugin)
|
||||
- [Nomad](https://www.nomadproject.io/downloads.html) v1.3 or later (to run the plugin)
|
||||
- [Nix](https://nixos.org/download.html) v2.11 or later (to run the plugin), either through NixOS or installed in root mode
|
||||
- [Go](https://golang.org/doc/install) v1.18 or later (to compile the plugin)
|
||||
- [Nomad](https://www.nomadproject.io/downloads.html) v0.9+ (to run the plugin)
|
||||
|
||||
Building and using the Nix driver plugin
|
||||
Building the Skeleton Plugin
|
||||
-------------------
|
||||
|
||||
To build the plugin and run a dev agent:
|
||||
[Generate](https://github.com/hashicorp/nomad-skeleton-driver-plugin/generate)
|
||||
a new repository in your account from this template by clicking the `Use this
|
||||
template` button above.
|
||||
|
||||
Clone the repository somewhere in your computer. This project uses
|
||||
[Go modules](https://blog.golang.org/using-go-modules) so you will need to set
|
||||
the environment variable `GO111MODULE=on` or work outside your `GOPATH` if it
|
||||
is set to `auto` or not declared.
|
||||
|
||||
```sh
|
||||
$ git clone git@github.com:<ORG>/<REPO>git
|
||||
```
|
||||
|
||||
Enter the plugin directory and update the paths in `go.mod` and `main.go` to
|
||||
match your repository path.
|
||||
|
||||
```diff
|
||||
// go.mod
|
||||
|
||||
- module github.com/hashicorp/nomad-skeleton-driver-plugin
|
||||
+ module github.com/<ORG>/<REPO>
|
||||
...
|
||||
```
|
||||
|
||||
```diff
|
||||
// main.go
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
log "github.com/hashicorp/go-hclog"
|
||||
- "github.com/hashicorp/nomad-skeleton-driver-plugin/hello"
|
||||
+. "github.com/<REPO>/<ORG>/hello"
|
||||
...
|
||||
|
||||
```
|
||||
|
||||
Build the skeleton plugin.
|
||||
|
||||
```sh
|
||||
$ make build
|
||||
```
|
||||
|
||||
## Deploying Driver Plugins in Nomad
|
||||
|
||||
The initial version of the skeleton is a simple task that outputs a greeting.
|
||||
You can try it out by starting a Nomad agent and running the job provided in
|
||||
the `example` folder:
|
||||
|
||||
```sh
|
||||
$ make build
|
||||
$ nomad agent -dev -config=./example/agent.hcl -plugin-dir=$(pwd)
|
||||
|
||||
# in another shell
|
||||
$ nomad run ./example/example-batch.hcl
|
||||
$ nomad run ./example/example-service.hcl
|
||||
$ nomad run ./example/example.nomad
|
||||
$ nomad logs <ALLOCATION ID>
|
||||
```
|
||||
|
||||
Writing Nix job specifications
|
||||
Code Organization
|
||||
-------------------
|
||||
|
||||
See documentation comments in example HCL files.
|
||||
Follow the comments marked with a `TODO` tag to implement your driver's logic.
|
||||
For more information check the
|
||||
[Nomad documentation on plugins](https://www.nomadproject.io/docs/internals/plugins/index.html).
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@
|
|||
client {
|
||||
}
|
||||
|
||||
plugin "nix2-driver" {
|
||||
plugin "exec2-driver" {
|
||||
config {
|
||||
default_nixpkgs = "github:nixos/nixpkgs/nixos-22.05"
|
||||
bind_read_only = {
|
||||
"/etc" = "/etc",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,69 +0,0 @@
|
|||
job "nix2-example-batch" {
|
||||
datacenters = ["dc1"]
|
||||
type = "batch"
|
||||
|
||||
group "example" {
|
||||
# Simple example: how to run a binary from a Nixpkgs package
|
||||
# By default, this will use nixpkgs from github:nixos/nixpkgs/nixos-22.05
|
||||
# as a base system, as defined in the agent config file.
|
||||
# This could be overridden by setting nixpkgs = "another flake"
|
||||
# inside the config {} block
|
||||
task "nix-hello" {
|
||||
driver = "nix2"
|
||||
|
||||
config {
|
||||
# Packages contains a list of Nix flakes to include in the environement.
|
||||
# Entries that start with # will be relative to nixpkgs.
|
||||
# Otherwise, they are flake names that are passed directly to Nix build
|
||||
packages = [
|
||||
"#hello" # equivalent to "github:nixos/nixpkgs/nixos-22.05#hello"
|
||||
]
|
||||
command = "hello"
|
||||
}
|
||||
}
|
||||
|
||||
# This example show how to setup root CA certificates so that jobs
|
||||
# can do TLS connections
|
||||
# Here, a Nix profile is built using packages curl and cacert from nixpkgs.
|
||||
# Because the cacert package is included, the ca-bundle.crt file is added to
|
||||
# /etc in that profile. Then, the nix2 driver binds all files from that
|
||||
# profile in the root directory, making ca-bundle.crt available directly under /etc.
|
||||
# Reference: see https://gist.github.com/CMCDragonkai/1ae4f4b5edeb021ca7bb1d271caca999
|
||||
task "nix-curl-ssl" {
|
||||
driver = "nix2"
|
||||
|
||||
config {
|
||||
packages = [
|
||||
"#curl", "#cacert"
|
||||
]
|
||||
command = "curl"
|
||||
args = [
|
||||
"https://nixos.org"
|
||||
]
|
||||
}
|
||||
env = {
|
||||
SSL_CERT_FILE = "/etc/ssl/certs/ca-bundle.crt"
|
||||
}
|
||||
}
|
||||
|
||||
# This example show how to use a flake defined from a file
|
||||
task "nix-hello-flake" {
|
||||
driver = "nix2"
|
||||
|
||||
config {
|
||||
# Packages contains a list of Nix flakes to include in the environement.
|
||||
# Entries that start with # will be relative to nixpkgs.
|
||||
# Otherwise, they are flake names that are passed directly to Nix build
|
||||
packages = [
|
||||
".#hello"
|
||||
]
|
||||
command = "hello"
|
||||
}
|
||||
|
||||
template {
|
||||
data = file("flake.nix")
|
||||
destination = "flake.nix"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
job "nix2-example-service" {
|
||||
datacenters = ["dc1"]
|
||||
type = "service"
|
||||
|
||||
group "example" {
|
||||
# This task defines a server that runs a simple python file server on port 8080,
|
||||
# which allows to explore the contents of the filesystem namespace as visible
|
||||
# by processes that run inside the task.
|
||||
# A bunch of utilities are included as well, so that you can exec into the container
|
||||
# and explore what's inside by yourself.
|
||||
task "nix-python-serve-http" {
|
||||
driver = "nix2"
|
||||
|
||||
config {
|
||||
packages = [
|
||||
"#python3",
|
||||
"#bash",
|
||||
"#coreutils",
|
||||
"#curl",
|
||||
"#nix",
|
||||
"#git",
|
||||
"#cacert",
|
||||
"#strace",
|
||||
"#gnugrep",
|
||||
"#findutils",
|
||||
"#mount",
|
||||
]
|
||||
command = "python3"
|
||||
args = [ "-m", "http.server", "8080" ]
|
||||
}
|
||||
env = {
|
||||
SSL_CERT_FILE = "/etc/ssl/certs/ca-bundle.crt"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
57
example/example.hcl
Normal file
57
example/example.hcl
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
job "example" {
|
||||
datacenters = ["dc1"]
|
||||
type = "batch"
|
||||
|
||||
group "example" {
|
||||
task "test-host-bin" {
|
||||
driver = "exec2"
|
||||
|
||||
config {
|
||||
command = "/bin/sh"
|
||||
args = ["-c", "echo hello world"]
|
||||
bind_read_only = {
|
||||
"/bin" = "/bin",
|
||||
"/lib" = "/lib",
|
||||
"/lib64" = "/lib64",
|
||||
"/usr" = "/usr",
|
||||
"/nix" = "/nix",
|
||||
}
|
||||
}
|
||||
user = "lx"
|
||||
}
|
||||
|
||||
task "test-nix-hello" {
|
||||
driver = "exec2"
|
||||
|
||||
config {
|
||||
command = "/sw/bin/nix"
|
||||
args = [
|
||||
"--extra-experimental-features", "flakes",
|
||||
"--extra-experimental-features", "nix-command",
|
||||
"run",
|
||||
"github:NixOS/nixpkgs#hello"
|
||||
]
|
||||
bind = {
|
||||
"/nix" = "/nix",
|
||||
}
|
||||
bind_read_only = {
|
||||
"/home/lx/.nix-profile" = "/sw",
|
||||
}
|
||||
}
|
||||
user = "lx"
|
||||
}
|
||||
|
||||
task "test-nix-store" {
|
||||
driver = "exec2"
|
||||
|
||||
config {
|
||||
command = "/nix/store/30j23057fqnnc1p4jqmq73p0gxgn0frq-bash-5.1-p16/bin/sh"
|
||||
args = ["-c", "/nix/store/y41s1vcn0irn9ahn9wh62yx2cygs7qjj-coreutils-8.32/bin/ls /*; /nix/store/y41s1vcn0irn9ahn9wh62yx2cygs7qjj-coreutils-8.32/bin/id"]
|
||||
bind_read_only = {
|
||||
"/nix" = "/nix",
|
||||
}
|
||||
}
|
||||
user = "lx"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
{
|
||||
description = "A very basic flake";
|
||||
|
||||
outputs = { self, nixpkgs }:
|
||||
let
|
||||
pkgs = import nixpkgs { system = "x86_64-linux"; };
|
||||
hello = pkgs.writeScriptBin "hello" ''
|
||||
#!${pkgs.bash}/bin/bash
|
||||
echo "Hello from bash script!"
|
||||
'';
|
||||
in {
|
||||
|
||||
packages.x86_64-linux.hello = hello;
|
||||
|
||||
packages.x86_64-linux.default = self.packages.x86_64-linux.hello;
|
||||
|
||||
};
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package nix2
|
||||
package exec2
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
|
@ -6,11 +6,10 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.deuxfleurs.fr/Deuxfleurs/nomad-driver-nix2/executor"
|
||||
"github.com/Alexis211/nomad-driver-exec2/executor"
|
||||
"github.com/hashicorp/consul-template/signals"
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/nomad/client/lib/cgutil"
|
||||
|
|
@ -29,7 +28,7 @@ import (
|
|||
|
||||
const (
|
||||
// pluginName is the name of the plugin
|
||||
pluginName = "nix2"
|
||||
pluginName = "exec2"
|
||||
|
||||
// fingerprintPeriod is the interval at which the driver will send fingerprint responses
|
||||
fingerprintPeriod = 30 * time.Second
|
||||
|
|
@ -69,17 +68,18 @@ var (
|
|||
hclspec.NewAttr("default_ipc_mode", "string", false),
|
||||
hclspec.NewLiteral(`"private"`),
|
||||
),
|
||||
"default_nixpkgs": hclspec.NewDefault(
|
||||
hclspec.NewAttr("default_nixpkgs", "string", false),
|
||||
hclspec.NewLiteral(`"github:nixos/nixpkgs/nixos-23.11"`),
|
||||
),
|
||||
"allow_caps": hclspec.NewDefault(
|
||||
hclspec.NewAttr("allow_caps", "list(string)", false),
|
||||
hclspec.NewLiteral(capabilities.HCLSpecLiteral),
|
||||
),
|
||||
"allow_bind": hclspec.NewDefault(
|
||||
hclspec.NewAttr("allow_bind", "bool", false),
|
||||
hclspec.NewLiteral("true"),
|
||||
// Default host directories to bind in tasks
|
||||
"bind": hclspec.NewDefault(
|
||||
hclspec.NewAttr("bind", "list(map(string))", false),
|
||||
hclspec.NewLiteral("{}"),
|
||||
),
|
||||
"bind_read_only": hclspec.NewDefault(
|
||||
hclspec.NewAttr("bind_read_only", "list(map(string))", false),
|
||||
hclspec.NewLiteral("{}"),
|
||||
),
|
||||
})
|
||||
|
||||
|
|
@ -94,8 +94,6 @@ var (
|
|||
"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
|
||||
"cap_add": hclspec.NewAttr("cap_add", "list(string)", false),
|
||||
"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
|
||||
"nixpkgs": hclspec.NewAttr("nixpkgs", "string", false),
|
||||
"packages": hclspec.NewAttr("packages", "list(string)", false),
|
||||
})
|
||||
|
||||
// driverCapabilities represents the RPC response for what features are
|
||||
|
|
@ -159,15 +157,15 @@ type Config struct {
|
|||
// exec-based task drivers.
|
||||
DefaultModeIPC string `codec:"default_ipc_mode"`
|
||||
|
||||
// Nixpkgs flake to use
|
||||
DefaultNixpkgs string `codec:"default_nixpkgs"`
|
||||
|
||||
// AllowCaps configures which Linux Capabilities are enabled for tasks
|
||||
// running on this node.
|
||||
AllowCaps []string `codec:"allow_caps"`
|
||||
|
||||
// AllowBind defines whether users may bind host directories
|
||||
AllowBind bool `codec:"allow_bind"`
|
||||
// Paths to bind for read-write acess in all jobs
|
||||
Bind hclutils.MapStrStr `codec:"bind"`
|
||||
|
||||
// Paths to bind for read-only acess in all jobs
|
||||
BindReadOnly hclutils.MapStrStr `codec:"bind_read_only"`
|
||||
}
|
||||
|
||||
func (c *Config) validate() error {
|
||||
|
|
@ -213,20 +211,14 @@ type TaskConfig struct {
|
|||
// Must be "private" or "host" if set.
|
||||
ModeIPC string `codec:"ipc_mode"`
|
||||
|
||||
// Nixpkgs flake to use
|
||||
Nixpkgs string `codec:"nixpkgs"`
|
||||
|
||||
// CapAdd is a set of linux capabilities to enable.
|
||||
CapAdd []string `codec:"cap_add"`
|
||||
|
||||
// CapDrop is a set of linux capabilities to disable.
|
||||
CapDrop []string `codec:"cap_drop"`
|
||||
|
||||
// List of Nix packages to add to environment
|
||||
Packages []string `codec:"packages"`
|
||||
}
|
||||
|
||||
func (tc *TaskConfig) validate(dc *Config) error {
|
||||
func (tc *TaskConfig) validate() error {
|
||||
switch tc.ModePID {
|
||||
case "", executor.IsolationModePrivate, executor.IsolationModeHost:
|
||||
default:
|
||||
|
|
@ -249,12 +241,6 @@ func (tc *TaskConfig) validate(dc *Config) error {
|
|||
return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops)
|
||||
}
|
||||
|
||||
if !dc.AllowBind {
|
||||
if len(tc.Bind) > 0 || len(tc.BindReadOnly) > 0 {
|
||||
return fmt.Errorf("bind and bind_read_only are deactivated for the %s driver", pluginName)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -469,14 +455,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
|
|||
if err := cfg.DecodeDriverConfig(&driverConfig); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
|
||||
}
|
||||
if driverConfig.Bind == nil {
|
||||
driverConfig.Bind = make(hclutils.MapStrStr)
|
||||
}
|
||||
if driverConfig.BindReadOnly == nil {
|
||||
driverConfig.BindReadOnly = make(hclutils.MapStrStr)
|
||||
}
|
||||
|
||||
if err := driverConfig.validate(&d.config); err != nil {
|
||||
if err := driverConfig.validate(); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed driver config validation: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -500,46 +480,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
|
|||
|
||||
user := cfg.User
|
||||
if user == "" {
|
||||
user = "nobody"
|
||||
}
|
||||
|
||||
// Determine the nixpkgs version to use.
|
||||
nixpkgs := driverConfig.Nixpkgs
|
||||
if nixpkgs == "" {
|
||||
nixpkgs = d.config.DefaultNixpkgs
|
||||
}
|
||||
// Use that repo for all packages not specified from a flake already.
|
||||
for i := range driverConfig.Packages {
|
||||
if strings.HasPrefix(driverConfig.Packages[i], "#") {
|
||||
driverConfig.Packages[i] = nixpkgs + driverConfig.Packages[i]
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare NixOS packages and setup a bunch of read-only mounts
|
||||
// for system stuff and required NixOS packages
|
||||
d.eventer.EmitEvent(&drivers.TaskEvent{
|
||||
TaskID: cfg.ID,
|
||||
AllocID: cfg.AllocID,
|
||||
TaskName: cfg.Name,
|
||||
Timestamp: time.Now(),
|
||||
Message: fmt.Sprintf(
|
||||
"Building Nix packages and preparing NixOS state (using nixpkgs from flake: %s)",
|
||||
nixpkgs,
|
||||
),
|
||||
Annotations: map[string]string{
|
||||
"packages": strings.Join(driverConfig.Packages, " "),
|
||||
},
|
||||
})
|
||||
taskDirs := cfg.TaskDir()
|
||||
systemMounts, err := prepareNixPackages(taskDirs.Dir, driverConfig.Packages, nixpkgs)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Some files are necessary and should be taken from outside if not present already
|
||||
etcpaths := []string{
|
||||
"/etc/nsswitch.conf", // Necessary for many things
|
||||
"/etc/passwd", // Necessary for username/UID lookup
|
||||
user = "0"
|
||||
}
|
||||
|
||||
if cfg.DNS != nil {
|
||||
|
|
@ -548,59 +489,64 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
|
|||
return nil, nil, fmt.Errorf("failed to build mount for resolv.conf: %v", err)
|
||||
}
|
||||
cfg.Mounts = append(cfg.Mounts, dnsMount)
|
||||
} else {
|
||||
// Inherit nameserver configuration from host
|
||||
etcpaths = append(etcpaths, "/etc/resolv.conf")
|
||||
}
|
||||
|
||||
for _, f := range etcpaths {
|
||||
if _, ok := systemMounts[f]; !ok {
|
||||
systemMounts[f] = f
|
||||
// Bind mounts specified in driver config
|
||||
if d.config.Bind != nil {
|
||||
for host, task := range d.config.Bind {
|
||||
mount_config := drivers.MountConfig{
|
||||
TaskPath: task,
|
||||
HostPath: host,
|
||||
Readonly: false,
|
||||
PropagationMode: "private",
|
||||
}
|
||||
d.logger.Info("adding RW mount from driver config", "mount_config", hclog.Fmt("%+v", mount_config))
|
||||
cfg.Mounts = append(cfg.Mounts, &mount_config)
|
||||
}
|
||||
}
|
||||
|
||||
d.logger.Info("adding RO system mounts for Nix stuff / system stuff", "system_mounts", hclog.Fmt("%+v", systemMounts))
|
||||
|
||||
for host, task := range systemMounts {
|
||||
mount_config := drivers.MountConfig{
|
||||
TaskPath: task,
|
||||
HostPath: host,
|
||||
Readonly: true,
|
||||
PropagationMode: "private",
|
||||
if d.config.BindReadOnly != nil {
|
||||
for host, task := range d.config.BindReadOnly {
|
||||
mount_config := drivers.MountConfig{
|
||||
TaskPath: task,
|
||||
HostPath: host,
|
||||
Readonly: true,
|
||||
PropagationMode: "private",
|
||||
}
|
||||
d.logger.Info("adding RO mount from driver config", "mount_config", hclog.Fmt("%+v", mount_config))
|
||||
cfg.Mounts = append(cfg.Mounts, &mount_config)
|
||||
}
|
||||
cfg.Mounts = append(cfg.Mounts, &mount_config)
|
||||
}
|
||||
|
||||
// Set PATH to /bin
|
||||
cfg.Env["PATH"] = "/bin"
|
||||
|
||||
// Bind mounts specified in task config
|
||||
for host, task := range driverConfig.Bind {
|
||||
mount_config := drivers.MountConfig{
|
||||
TaskPath: task,
|
||||
HostPath: host,
|
||||
Readonly: false,
|
||||
PropagationMode: "private",
|
||||
if driverConfig.Bind != nil {
|
||||
for host, task := range driverConfig.Bind {
|
||||
mount_config := drivers.MountConfig{
|
||||
TaskPath: task,
|
||||
HostPath: host,
|
||||
Readonly: false,
|
||||
PropagationMode: "private",
|
||||
}
|
||||
d.logger.Info("adding RW mount from task spec", "mount_config", hclog.Fmt("%+v", mount_config))
|
||||
cfg.Mounts = append(cfg.Mounts, &mount_config)
|
||||
}
|
||||
d.logger.Info("adding RW mount from task spec", "mount_config", hclog.Fmt("%+v", mount_config))
|
||||
cfg.Mounts = append(cfg.Mounts, &mount_config)
|
||||
}
|
||||
for host, task := range driverConfig.BindReadOnly {
|
||||
mount_config := drivers.MountConfig{
|
||||
TaskPath: task,
|
||||
HostPath: host,
|
||||
Readonly: true,
|
||||
PropagationMode: "private",
|
||||
if driverConfig.BindReadOnly != nil {
|
||||
for host, task := range driverConfig.BindReadOnly {
|
||||
mount_config := drivers.MountConfig{
|
||||
TaskPath: task,
|
||||
HostPath: host,
|
||||
Readonly: true,
|
||||
PropagationMode: "private",
|
||||
}
|
||||
d.logger.Info("adding RO mount from task spec", "mount_config", hclog.Fmt("%+v", mount_config))
|
||||
cfg.Mounts = append(cfg.Mounts, &mount_config)
|
||||
}
|
||||
d.logger.Info("adding RO mount from task spec", "mount_config", hclog.Fmt("%+v", mount_config))
|
||||
cfg.Mounts = append(cfg.Mounts, &mount_config)
|
||||
}
|
||||
|
||||
caps, err := capabilities.Calculate(
|
||||
capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop,
|
||||
)
|
||||
if err != nil {
|
||||
pluginClient.Kill()
|
||||
return nil, nil, err
|
||||
}
|
||||
d.logger.Debug("task capabilities", "capabilities", caps)
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package nix2
|
||||
package exec2
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"git.deuxfleurs.fr/Deuxfleurs/nomad-driver-nix2/executor"
|
||||
"github.com/Alexis211/nomad-driver-exec2/executor"
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package nix2
|
||||
package exec2
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
|
@ -801,41 +801,18 @@ func cmdMounts(mounts []*drivers.MountConfig) []*lconfigs.Mount {
|
|||
//
|
||||
// See also executor.lookupBin for a version used by non-isolated drivers.
|
||||
func lookupTaskBin(command *ExecCommand) (string, string, error) {
|
||||
cmd := command.Cmd
|
||||
|
||||
taskPath, hostPath, err := lookupBinFile(command, cmd)
|
||||
if err == nil {
|
||||
return taskPath, hostPath, nil
|
||||
}
|
||||
|
||||
if !strings.Contains(cmd, "/") {
|
||||
// Look up also in /bin
|
||||
bin := filepath.Join("/bin", cmd)
|
||||
taskPath, hostPath, err = lookupBinFile(command, bin)
|
||||
if err == nil {
|
||||
return taskPath, hostPath, nil
|
||||
}
|
||||
|
||||
return "", "", fmt.Errorf("file %s not found in task dir or in mounts, even when looking up /bin", cmd)
|
||||
} else {
|
||||
// If there's a / in the binary's path, we can't fallback to a PATH search
|
||||
return "", "", fmt.Errorf("file %s not found in task dir or in mounts", cmd)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func lookupBinFile(command *ExecCommand, bin string) (string, string, error) {
|
||||
taskDir := command.TaskDir
|
||||
bin := command.Cmd
|
||||
|
||||
// Check in the local directory
|
||||
localDir := filepath.Join(taskDir, allocdir.TaskLocal)
|
||||
taskPath, hostPath, err := getPathInTaskDir(taskDir, localDir, bin)
|
||||
taskPath, hostPath, err := getPathInTaskDir(command.TaskDir, localDir, bin)
|
||||
if err == nil {
|
||||
return taskPath, hostPath, nil
|
||||
}
|
||||
|
||||
// Check at the root of the task's directory
|
||||
taskPath, hostPath, err = getPathInTaskDir(taskDir, taskDir, bin)
|
||||
taskPath, hostPath, err = getPathInTaskDir(command.TaskDir, command.TaskDir, bin)
|
||||
if err == nil {
|
||||
return taskPath, hostPath, nil
|
||||
}
|
||||
|
|
@ -848,7 +825,31 @@ func lookupBinFile(command *ExecCommand, bin string) (string, string, error) {
|
|||
}
|
||||
}
|
||||
|
||||
return "", "", fmt.Errorf("file %s not found in task dir or in mounts", bin)
|
||||
// If there's a / in the binary's path, we can't fallback to a PATH search
|
||||
if strings.Contains(bin, "/") {
|
||||
return "", "", fmt.Errorf("file %s not found under path %s", bin, taskDir)
|
||||
}
|
||||
|
||||
// look for a file using a PATH-style lookup inside the directory
|
||||
// root. Similar to the stdlib's exec.LookPath except:
|
||||
// - uses a restricted lookup PATH rather than the agent process's PATH env var.
|
||||
// - does not require that the file is already executable (this will be ensured
|
||||
// by the caller)
|
||||
// - does not prevent using relative path as added to exec.LookPath in go1.19
|
||||
// (this gets fixed-up in the caller)
|
||||
|
||||
// This is a fake PATH so that we're not using the agent's PATH
|
||||
restrictedPaths := []string{"/usr/local/bin", "/usr/bin", "/bin"}
|
||||
|
||||
for _, dir := range restrictedPaths {
|
||||
pathDir := filepath.Join(command.TaskDir, dir)
|
||||
taskPath, hostPath, err = getPathInTaskDir(command.TaskDir, pathDir, bin)
|
||||
if err == nil {
|
||||
return taskPath, hostPath, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", "", fmt.Errorf("file %s not found under path", bin)
|
||||
}
|
||||
|
||||
// getPathInTaskDir searches for the binary in the task directory and nested
|
||||
|
|
|
|||
28
go.mod
28
go.mod
|
|
@ -1,34 +1,23 @@
|
|||
module git.deuxfleurs.fr/Deuxfleurs/nomad-driver-nix2
|
||||
module github.com/Alexis211/nomad-driver-exec2
|
||||
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/LK4D4/joincontext v0.0.0-20171026170139-1724345da6d5
|
||||
github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2
|
||||
github.com/containernetworking/plugins v1.1.1
|
||||
github.com/creack/pty v1.1.18
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/hashicorp/consul-template v0.29.6-0.20221026140134-90370e07bf62
|
||||
github.com/hashicorp/go-hclog v1.3.1
|
||||
github.com/hashicorp/go-multierror v1.1.1
|
||||
github.com/hashicorp/go-plugin v1.4.4
|
||||
github.com/hashicorp/nomad v1.4.2
|
||||
github.com/mitchellh/go-ps v1.0.0
|
||||
github.com/opencontainers/runc v1.1.4
|
||||
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
|
||||
github.com/shirou/gopsutil/v3 v3.22.8
|
||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
|
||||
golang.org/x/sys v0.1.0
|
||||
google.golang.org/grpc v1.48.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/LK4D4/joincontext v0.0.0-20171026170139-1724345da6d5 // indirect
|
||||
github.com/Masterminds/goutils v1.1.1 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.1.1 // indirect
|
||||
github.com/Masterminds/sprig/v3 v3.2.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||
github.com/agext/levenshtein v1.2.1 // indirect
|
||||
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
|
||||
github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect
|
||||
github.com/armon/go-metrics v0.4.1 // indirect
|
||||
github.com/armon/go-radix v1.0.0 // indirect
|
||||
github.com/bgentry/speakeasy v0.1.0 // indirect
|
||||
|
|
@ -37,7 +26,9 @@ require (
|
|||
github.com/cilium/ebpf v0.9.1 // indirect
|
||||
github.com/container-storage-interface/spec v1.6.0 // indirect
|
||||
github.com/containerd/console v1.0.3 // indirect
|
||||
github.com/containernetworking/plugins v1.1.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
|
||||
github.com/creack/pty v1.1.18 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/docker/docker v20.10.17+incompatible // indirect
|
||||
|
|
@ -47,6 +38,7 @@ require (
|
|||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/godbus/dbus/v5 v5.1.0 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/btree v1.0.0 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
|
|
@ -57,6 +49,7 @@ require (
|
|||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v1.1.5 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.4 // indirect
|
||||
|
|
@ -91,6 +84,7 @@ require (
|
|||
github.com/mitchellh/cli v1.1.4 // indirect
|
||||
github.com/mitchellh/copystructure v1.2.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/go-ps v1.0.0 // indirect
|
||||
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
|
||||
github.com/mitchellh/hashstructure v1.1.0 // indirect
|
||||
|
|
@ -102,6 +96,8 @@ require (
|
|||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
|
||||
github.com/oklog/run v1.1.0 // indirect
|
||||
github.com/onsi/gomega v1.17.0 // indirect
|
||||
github.com/opencontainers/runc v1.1.4 // indirect
|
||||
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
|
||||
github.com/opencontainers/selinux v1.10.1 // indirect
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
|
|
@ -110,10 +106,12 @@ require (
|
|||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
|
||||
github.com/seccomp/libseccomp-golang v0.10.0 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.22.8 // indirect
|
||||
github.com/shopspring/decimal v1.2.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/spf13/cast v1.3.1 // indirect
|
||||
github.com/stretchr/testify v1.8.1 // indirect
|
||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.10 // indirect
|
||||
github.com/tklauser/numcpus v0.5.0 // indirect
|
||||
github.com/vishvananda/netlink v1.2.1-beta.2 // indirect
|
||||
|
|
@ -128,11 +126,13 @@ require (
|
|||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
|
||||
golang.org/x/sys v0.1.0 // indirect
|
||||
golang.org/x/text v0.3.8 // indirect
|
||||
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
|
||||
golang.org/x/tools v0.1.12 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220712132514-bdd2acd4974d // indirect
|
||||
google.golang.org/grpc v1.48.0 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
|
||||
gopkg.in/fsnotify.v1 v1.4.7 // indirect
|
||||
|
|
|
|||
4
main.go
4
main.go
|
|
@ -1,7 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"git.deuxfleurs.fr/Deuxfleurs/nomad-driver-nix2/nix2"
|
||||
"github.com/Alexis211/nomad-driver-exec2/exec2"
|
||||
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/nomad/plugins"
|
||||
|
|
@ -14,5 +14,5 @@ func main() {
|
|||
|
||||
// factory returns a new instance of a nomad driver plugin
|
||||
func factory(log hclog.Logger) interface{} {
|
||||
return nix2.NewPlugin(log)
|
||||
return exec2.NewPlugin(log)
|
||||
}
|
||||
|
|
|
|||
163
nix2/nix.go
163
nix2/nix.go
|
|
@ -1,163 +0,0 @@
|
|||
package nix2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
|
||||
)
|
||||
|
||||
const (
|
||||
closureNix = `
|
||||
{ path }:
|
||||
let
|
||||
nixpkgs = builtins.getFlake "%s";
|
||||
inherit (nixpkgs.legacyPackages.x86_64-linux) buildPackages;
|
||||
in buildPackages.closureInfo { rootPaths = builtins.storePath path; }
|
||||
`
|
||||
)
|
||||
|
||||
func prepareNixPackages(taskDir string, packages []string, nixpkgs string) (hclutils.MapStrStr, error) {
|
||||
mounts := make(hclutils.MapStrStr)
|
||||
|
||||
profileLink := filepath.Join(taskDir, "current-profile")
|
||||
profile, err := nixBuildProfile(taskDir, packages, profileLink)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Build of the flakes failed: %v", err)
|
||||
}
|
||||
|
||||
closureLink := filepath.Join(taskDir, "current-closure")
|
||||
closure, err := nixBuildClosure(profileLink, closureLink, nixpkgs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Build of the flakes failed: %v", err)
|
||||
}
|
||||
|
||||
mounts[profile] = profile
|
||||
|
||||
if entries, err := os.ReadDir(profile); err != nil {
|
||||
return nil, fmt.Errorf("Couldn't read profile directory: %w", err)
|
||||
} else {
|
||||
for _, entry := range entries {
|
||||
if name := entry.Name(); name != "etc" {
|
||||
mounts[filepath.Join(profile, name)] = "/" + name
|
||||
continue
|
||||
}
|
||||
|
||||
etcEntries, err := os.ReadDir(filepath.Join(profile, "etc"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Couldn't read profile's /etc directory: %w", err)
|
||||
}
|
||||
|
||||
for _, etcEntry := range etcEntries {
|
||||
etcName := etcEntry.Name()
|
||||
mounts[filepath.Join(profile, "etc", etcName)] = "/etc/" + etcName
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
requisites, err := nixRequisites(closure)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Couldn't determine flake requisites: %v", err)
|
||||
}
|
||||
|
||||
for _, requisite := range requisites {
|
||||
mounts[requisite] = requisite
|
||||
}
|
||||
|
||||
return mounts, nil
|
||||
}
|
||||
|
||||
func nixBuildProfile(taskDir string, flakes []string, link string) (string, error) {
|
||||
cmd := exec.Command("nix", append(
|
||||
[]string{
|
||||
"--extra-experimental-features", "nix-command",
|
||||
"--extra-experimental-features", "flakes",
|
||||
"profile",
|
||||
"install",
|
||||
"--no-write-lock-file",
|
||||
"--profile",
|
||||
link},
|
||||
flakes...)...)
|
||||
stderr := &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
cmd.Dir = taskDir
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("%v failed: %s. Err: %v", cmd.Args, stderr.String(), err)
|
||||
}
|
||||
|
||||
if target, err := os.Readlink(link); err == nil {
|
||||
return os.Readlink(filepath.Join(filepath.Dir(link), target))
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
func nixBuildClosure(profile string, link string, nixpkgs string) (string, error) {
|
||||
cmd := exec.Command(
|
||||
"nix",
|
||||
"--extra-experimental-features", "nix-command",
|
||||
"--extra-experimental-features", "flakes",
|
||||
"build",
|
||||
"--out-link", link,
|
||||
"--expr", fmt.Sprintf(closureNix, nixpkgs),
|
||||
"--impure",
|
||||
"--no-write-lock-file",
|
||||
"--argstr", "path", profile)
|
||||
|
||||
stderr := &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("%v failed: %s. Err: %v", cmd.Args, stderr.String(), err)
|
||||
}
|
||||
|
||||
return os.Readlink(link)
|
||||
}
|
||||
|
||||
type nixPathInfo struct {
|
||||
Path string `json:"path"`
|
||||
NarHash string `json:"narHash"`
|
||||
NarSize uint64 `json:"narSize"`
|
||||
References []string `json:"references"`
|
||||
Deriver string `json:"deriver"`
|
||||
RegistrationTime uint64 `json:"registrationTime"`
|
||||
Signatures []string `json:"signatures"`
|
||||
}
|
||||
|
||||
func nixRequisites(path string) ([]string, error) {
|
||||
cmd := exec.Command(
|
||||
"nix",
|
||||
"--extra-experimental-features", "nix-command",
|
||||
"--extra-experimental-features", "flakes",
|
||||
"path-info",
|
||||
"--json",
|
||||
"--recursive",
|
||||
path)
|
||||
|
||||
stdout := &bytes.Buffer{}
|
||||
cmd.Stdout = stdout
|
||||
|
||||
stderr := &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("%v failed: %s. Err: %v", cmd.Args, stderr.String(), err)
|
||||
}
|
||||
|
||||
result := []*nixPathInfo{}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
requisites := []string{}
|
||||
for _, result := range result {
|
||||
requisites = append(requisites, result.Path)
|
||||
}
|
||||
|
||||
return requisites, nil
|
||||
}
|
||||
Loading…
Reference in a new issue