Ability to run Nix jobs

example/agent.hcl

@@ -3,10 +3,7 @@
 client {
 }
 
-plugin "exec2-driver" {
+plugin "nix2-driver" {
   config {
-    bind_read_only = {
-      "/etc" = "/etc",
-    }
   }
 }

example/example.hcl

@@ -3,53 +3,20 @@ job "example" {
   type        = "batch"
 
   group "example" {
-    task "test-host-bin" {
-      driver = "exec2"
-
-      config {
-        command =  "/bin/sh"
-        args = ["-c", "echo hello world"]
-        bind_read_only = {
-          "/bin" = "/bin",
-          "/lib" = "/lib",
-          "/lib64" = "/lib64",
-          "/usr" = "/usr",
-          "/nix" = "/nix",
-        }
-      }
-      user = "lx"
-    }
-
     task "test-nix-hello" {
-      driver = "exec2"
+      driver = "nix2"
 
       config {
-        command = "/sw/bin/nix"
+        command = "sh"
         args = [
-          "--extra-experimental-features", "flakes",
+          "-c",
-          "--extra-experimental-features", "nix-command",
+          "pwd; ls -l *; mount; hello"
-          "run",
+        ]
+        packages = [
+          "github:NixOS/nixpkgs#coreutils",
+          "github:NixOS/nixpkgs#bash",
           "github:NixOS/nixpkgs#hello"
           ]
-        bind = {
-          "/nix" = "/nix",
-        }
-        bind_read_only = {
-          "/home/lx/.nix-profile" = "/sw",
-        }
-      }
-      user = "lx"
-    }
-
-    task "test-nix-store" {
-      driver = "exec2"
-
-      config {
-        command =  "/nix/store/30j23057fqnnc1p4jqmq73p0gxgn0frq-bash-5.1-p16/bin/sh"
-        args = ["-c", "/nix/store/y41s1vcn0irn9ahn9wh62yx2cygs7qjj-coreutils-8.32/bin/ls /*; /nix/store/y41s1vcn0irn9ahn9wh62yx2cygs7qjj-coreutils-8.32/bin/id"]
-        bind_read_only = {
-          "/nix" = "/nix",
-        }
       }
       user = "lx"
     }

example/example2.hcl

@@ -0,0 +1,28 @@
+job "example2" {
+  datacenters = ["dc1"]
+  type        = "service"
+
+  group "example" {
+    task "server" {
+      driver = "nix2"
+
+      config {
+        packages = [
+          "github:nixos/nixpkgs#python3",
+          "github:nixos/nixpkgs#bash",
+          "github:nixos/nixpkgs#coreutils",
+          "github:nixos/nixpkgs#curl",
+          "github:nixos/nixpkgs#nix",
+          "github:nixos/nixpkgs#git",
+          "github:nixos/nixpkgs#cacert",
+          "github:nixos/nixpkgs#strace",
+          "github:nixos/nixpkgs#gnugrep",
+          "github:nixos/nixpkgs#mount",
+        ]
+        command = "python3"
+        args = [ "-m", "http.server", "8080" ]
+      }
+      user = "lx"
+    }
+  }
+}

executor/executor_linux.go

@@ -801,18 +801,41 @@ func cmdMounts(mounts []*drivers.MountConfig) []*lconfigs.Mount {
 //
 // See also executor.lookupBin for a version used by non-isolated drivers.
 func lookupTaskBin(command *ExecCommand) (string, string, error) {
+	cmd := command.Cmd
+
+	taskPath, hostPath, err := lookupBinFile(command, cmd)
+	if err == nil {
+		return taskPath, hostPath, nil
+	}
+
+	if !strings.Contains(cmd, "/") {
+		// Look up also in /bin
+		bin := filepath.Join("/bin", cmd)
+		taskPath, hostPath, err = lookupBinFile(command, bin)
+		if err == nil {
+			return taskPath, hostPath, nil
+		}
+
+		return "", "", fmt.Errorf("file %s not found in task dir or in mounts, even when looking up /bin", cmd)
+	} else {
+		// If there's a / in the binary's path, we can't fallback to a PATH search
+		return "", "", fmt.Errorf("file %s not found in task dir or in mounts", cmd)
+	}
+
+}
+
+func lookupBinFile(command *ExecCommand, bin string) (string, string, error) {
 	taskDir := command.TaskDir
-	bin := command.Cmd
 
 	// Check in the local directory
 	localDir := filepath.Join(taskDir, allocdir.TaskLocal)
-	taskPath, hostPath, err := getPathInTaskDir(command.TaskDir, localDir, bin)
+	taskPath, hostPath, err := getPathInTaskDir(taskDir, localDir, bin)
 	if err == nil {
 		return taskPath, hostPath, nil
 	}
 
 	// Check at the root of the task's directory
-	taskPath, hostPath, err = getPathInTaskDir(command.TaskDir, command.TaskDir, bin)
+	taskPath, hostPath, err = getPathInTaskDir(taskDir, taskDir, bin)
 	if err == nil {
 		return taskPath, hostPath, nil
 	}
@@ -825,31 +848,7 @@ func lookupTaskBin(command *ExecCommand) (string, string, error) {
 		}
 	}
 
-	// If there's a / in the binary's path, we can't fallback to a PATH search
+	return "", "", fmt.Errorf("file %s not found in task dir or in mounts", bin)
-	if strings.Contains(bin, "/") {
-		return "", "", fmt.Errorf("file %s not found under path %s", bin, taskDir)
-	}
-
-	// look for a file using a PATH-style lookup inside the directory
-	// root. Similar to the stdlib's exec.LookPath except:
-	//   - uses a restricted lookup PATH rather than the agent process's PATH env var.
-	//   - does not require that the file is already executable (this will be ensured
-	//     by the caller)
-	//   - does not prevent using relative path as added to exec.LookPath in go1.19
-	//     (this gets fixed-up in the caller)
-
-	// This is a fake PATH so that we're not using the agent's PATH
-	restrictedPaths := []string{"/usr/local/bin", "/usr/bin", "/bin"}
-
-	for _, dir := range restrictedPaths {
-		pathDir := filepath.Join(command.TaskDir, dir)
-		taskPath, hostPath, err = getPathInTaskDir(command.TaskDir, pathDir, bin)
-		if err == nil {
-			return taskPath, hostPath, nil
-		}
-	}
-
-	return "", "", fmt.Errorf("file %s not found under path", bin)
 }
 
 // getPathInTaskDir searches for the binary in the task directory and nested

nix2/driver.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync"
 	"time"
 
@@ -15,7 +16,6 @@ import (
 	"github.com/hashicorp/nomad/client/lib/cgutil"
 	"github.com/hashicorp/nomad/drivers/shared/capabilities"
 	"github.com/hashicorp/nomad/drivers/shared/eventer"
-	"github.com/hashicorp/nomad/drivers/shared/resolvconf"
 	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
 	"github.com/hashicorp/nomad/helper/pointer"
@@ -89,6 +89,7 @@ var (
 		"ipc_mode":       hclspec.NewAttr("ipc_mode", "string", false),
 		"cap_add":        hclspec.NewAttr("cap_add", "list(string)", false),
 		"cap_drop":       hclspec.NewAttr("cap_drop", "list(string)", false),
+		"packages":       hclspec.NewAttr("packages", "list(string)", false),
 	})
 
 	// driverCapabilities represents the RPC response for what features are
@@ -208,9 +209,12 @@ type TaskConfig struct {
 
 	// CapDrop is a set of linux capabilities to disable.
 	CapDrop []string `codec:"cap_drop"`
+
+	// List of Nix packages to add to environment
+	Packages []string `codec:"packages"`
 }
 
-func (tc *TaskConfig) validate() error {
+func (tc *TaskConfig) validate(dc *Config) error {
 	switch tc.ModePID {
 	case "", executor.IsolationModePrivate, executor.IsolationModeHost:
 	default:
@@ -233,6 +237,12 @@ func (tc *TaskConfig) validate() error {
 		return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops)
 	}
 
+	if !dc.AllowBind {
+		if len(tc.Bind) > 0 || len(tc.BindReadOnly) > 0 {
+			return fmt.Errorf("bind and bind_read_only are deactivated for the %s driver", pluginName)
+		}
+	}
+
 	return nil
 }
 
@@ -447,8 +457,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 	if err := cfg.DecodeDriverConfig(&driverConfig); err != nil {
 		return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
 	}
+	if driverConfig.Bind == nil {
+		driverConfig.Bind = make(hclutils.MapStrStr)
+	}
+	if driverConfig.BindReadOnly == nil {
+		driverConfig.BindReadOnly = make(hclutils.MapStrStr)
+	}
 
-	if err := driverConfig.validate(); err != nil {
+	if err := driverConfig.validate(&d.config); err != nil {
 		return nil, nil, fmt.Errorf("failed driver config validation: %v", err)
 	}
 
@@ -475,52 +491,73 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		user = "0"
 	}
 
-	if cfg.DNS != nil {
+	// Prepare NixOS packages and setup a bunch of read-only mounts
-		dnsMount, err := resolvconf.GenerateDNSMount(cfg.TaskDir().Dir, cfg.DNS)
+	// for system stuff and required NixOS packages
-		if err != nil {
+	d.eventer.EmitEvent(&drivers.TaskEvent{
-			return nil, nil, fmt.Errorf("failed to build mount for resolv.conf: %v", err)
+		TaskID:    cfg.ID,
-		}
+		AllocID:   cfg.AllocID,
-		cfg.Mounts = append(cfg.Mounts, dnsMount)
+		TaskName:  cfg.Name,
+		Timestamp: time.Now(),
+		Message:   "Building Nix packages and preparing NixOS state",
+		Annotations: map[string]string{
+			"packages": strings.Join(driverConfig.Packages, " "),
+		},
+	})
+	taskDirs := cfg.TaskDir()
+	systemMounts, err := prepareNixPackages(taskDirs.Dir, driverConfig.Packages)
+	if err != nil {
+		return nil, nil, err
 	}
 
-	// Bind mounts specified in driver config
+	// Some files are necessary and should be taken from outside if not present already
+	for _, f := range []string{ "/etc/resolv.conf", "/etc/passwd", "/etc/nsswitch.conf" } {
+		if _, ok := systemMounts[f]; !ok {
+			systemMounts[f] = f
+		}
+	}
+
+	d.logger.Info("adding RO system mounts for Nix stuff / system stuff", "system_mounts", hclog.Fmt("%+v", systemMounts))
+
+	for host, task := range systemMounts {
+		mount_config := drivers.MountConfig{
+			TaskPath:        task,
+			HostPath:        host,
+			Readonly:        true,
+			PropagationMode: "private",
+		}
+		cfg.Mounts = append(cfg.Mounts, &mount_config)
+	}
+
+	// Set PATH to /bin
+	cfg.Env["PATH"] = "/bin"
 
 	// Bind mounts specified in task config
-	if d.config.AllowBind {
+	for host, task := range driverConfig.Bind {
-		if driverConfig.Bind != nil {
+		mount_config := drivers.MountConfig{
-			for host, task := range driverConfig.Bind {
+			TaskPath:        task,
-				mount_config := drivers.MountConfig{
+			HostPath:        host,
-					TaskPath:        task,
+			Readonly:        false,
-					HostPath:        host,
+			PropagationMode: "private",
-					Readonly:        false,
-					PropagationMode: "private",
-				}
-				d.logger.Info("adding RW mount from task spec", "mount_config", hclog.Fmt("%+v", mount_config))
-				cfg.Mounts = append(cfg.Mounts, &mount_config)
-			}
 		}
-		if driverConfig.BindReadOnly != nil {
+		d.logger.Info("adding RW mount from task spec", "mount_config", hclog.Fmt("%+v", mount_config))
-			for host, task := range driverConfig.BindReadOnly {
+		cfg.Mounts = append(cfg.Mounts, &mount_config)
-				mount_config := drivers.MountConfig{
+	}
-					TaskPath:        task,
+	for host, task := range driverConfig.BindReadOnly {
-					HostPath:        host,
+		mount_config := drivers.MountConfig{
-					Readonly:        true,
+			TaskPath:        task,
-					PropagationMode: "private",
+			HostPath:        host,
-				}
+			Readonly:        true,
-				d.logger.Info("adding RO mount from task spec", "mount_config", hclog.Fmt("%+v", mount_config))
+			PropagationMode: "private",
-				cfg.Mounts = append(cfg.Mounts, &mount_config)
-			}
-		}
-	} else {
-		if len(driverConfig.Bind) > 0 || len(driverConfig.BindReadOnly) > 0 {
-			return nil, nil, fmt.Errorf("bind and bind_read_only are deactivated for the %s driver", pluginName)
 		}
+		d.logger.Info("adding RO mount from task spec", "mount_config", hclog.Fmt("%+v", mount_config))
+		cfg.Mounts = append(cfg.Mounts, &mount_config)
 	}
 
 	caps, err := capabilities.Calculate(
 		capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop,
 	)
 	if err != nil {
+		pluginClient.Kill()
 		return nil, nil, err
 	}
 	d.logger.Debug("task capabilities", "capabilities", caps)

nix2/nix.go

@@ -0,0 +1,164 @@
+package nix2
+
+import (
+	"bytes"
+	"path/filepath"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
+)
+
+const (
+	closureNix = `
+{ path }:
+let
+  nixpkgs = builtins.getFlake "github:nixos/nixpkgs/nixos-22.05";
+  inherit (nixpkgs.legacyPackages.x86_64-linux) buildPackages;
+in buildPackages.closureInfo { rootPaths = builtins.storePath path; }
+`
+)
+
+func prepareNixPackages(taskDir string, packages []string) (hclutils.MapStrStr, error) {
+	mounts := make(hclutils.MapStrStr)
+
+	profileLink := filepath.Join(taskDir, "current-profile")
+	profile, err := nixBuildProfile(packages, profileLink)
+	if err != nil {
+		return nil, fmt.Errorf("Build of the flakes failed: %v", err)
+	}
+
+	closureLink := filepath.Join(taskDir, "current-closure")
+	closure, err := nixBuildClosure(profileLink, closureLink)
+	if err != nil {
+		return nil, fmt.Errorf("Build of the flakes failed: %v", err)
+	}
+
+	mounts[profile] = profile
+
+	if entries, err := os.ReadDir(profile); err != nil {
+		return nil, fmt.Errorf("Couldn't read profile directory: %w", err)
+	} else {
+		for _, entry := range entries {
+			if name := entry.Name(); name != "etc" {
+				mounts[filepath.Join(profile, name)] = "/" + name
+				continue
+			}
+
+			etcEntries, err := os.ReadDir(filepath.Join(profile, "etc"))
+			if err != nil {
+				return nil, fmt.Errorf("Couldn't read profile's /etc directory: %w", err)
+			}
+
+			for _, etcEntry := range etcEntries {
+				etcName := etcEntry.Name()
+				mounts[filepath.Join(profile, "etc", etcName)] = "/etc/" + etcName
+			}
+		}
+	}
+
+	mounts[filepath.Join(closure, "registration")] = "/registration"
+
+	requisites, err := nixRequisites(closure)
+	if err != nil {
+		return nil, fmt.Errorf("Couldn't determine flake requisites: %v", err)
+	}
+
+	for _, requisite := range requisites {
+		mounts[requisite] = requisite
+	}
+
+	return mounts, nil
+}
+
+func nixBuildProfile(flakes []string, link string) (string, error) {
+	cmd := exec.Command("nix", append(
+		[]string{
+			"--extra-experimental-features", "nix-command",
+			"--extra-experimental-features", "flakes",
+			"profile",
+			"install",
+			"--no-write-lock-file",
+			"--profile",
+			link},
+		flakes...)...)
+	stderr := &bytes.Buffer{}
+	cmd.Stderr = stderr
+
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("%v failed: %s. Err: %v", cmd.Args, stderr.String(), err)
+	}
+
+	if target, err := os.Readlink(link); err == nil {
+		return os.Readlink(filepath.Join(filepath.Dir(link), target))
+	} else {
+		return "", err
+	}
+}
+
+func nixBuildClosure(profile string, link string) (string, error) {
+	cmd := exec.Command(
+		"nix",
+		"--extra-experimental-features", "nix-command",
+		"--extra-experimental-features", "flakes",
+		"build",
+		"--out-link", link,
+		"--expr", closureNix,
+		"--impure",
+		"--no-write-lock-file",
+		"--argstr", "path", profile)
+
+	stderr := &bytes.Buffer{}
+	cmd.Stderr = stderr
+
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("%v failed: %s. Err: %v", cmd.Args, stderr.String(), err)
+	}
+
+	return os.Readlink(link)
+}
+
+type nixPathInfo struct {
+	Path             string   `json:"path"`
+	NarHash          string   `json:"narHash"`
+	NarSize          uint64   `json:"narSize"`
+	References       []string `json:"references"`
+	Deriver          string   `json:"deriver"`
+	RegistrationTime uint64   `json:"registrationTime"`
+	Signatures       []string `json:"signatures"`
+}
+
+func nixRequisites(path string) ([]string, error) {
+	cmd := exec.Command(
+		"nix",
+		"--extra-experimental-features", "nix-command",
+		"--extra-experimental-features", "flakes",
+		"path-info",
+		"--json",
+		"--recursive",
+		path)
+
+	stdout := &bytes.Buffer{}
+	cmd.Stdout = stdout
+
+	stderr := &bytes.Buffer{}
+	cmd.Stderr = stderr
+
+	if err := cmd.Run(); err != nil {
+		return nil, fmt.Errorf("%v failed: %s. Err: %v", cmd.Args, stderr.String(), err)
+	}
+
+	result := []*nixPathInfo{}
+	if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
+		return nil, err
+	}
+
+	requisites := []string{}
+	for _, result := range result {
+		requisites = append(requisites, result.Path)
+	}
+
+	return requisites, nil
+}