Works better and better, write some examples

This commit is contained in:
Alex 2022-11-29 13:01:46 +01:00
parent ec3eba576a
commit fa49c13513
Signed by: lx
GPG key ID: 0E496D15096376BE
7 changed files with 123 additions and 64 deletions

View file

@ -5,5 +5,6 @@ client {
plugin "nix2-driver" { plugin "nix2-driver" {
config { config {
default_nixpkgs = "github:nixos/nixpkgs/nixos-22.05"
} }
} }

46
example/example-batch.hcl Normal file
View file

@ -0,0 +1,46 @@
job "nix2-example-batch" {
datacenters = ["dc1"]
type = "batch"
group "example" {
# Simple example: how to run a binary from a Nixpkgs package
# By default, this will use nixpkgs from github:nixos/nixpkgs/nixos-22.05
# as a base system, as defined in the agent config file.
# This could be overridden by setting nixpkgs = "another flake"
# inside the config {} block
task "nix-hello" {
driver = "nix2"
config {
packages = [
"hello" # equivalent to "github:nixos/nixpkgs/nixos-22.05#hello"
]
command = "hello"
}
}
# This example show how to setup root CA certificates so that jobs
# can do TLS connections
# Here, a Nix profile is built using packages curl and cacert from nixpkgs.
# Because the cacert package is included, the ca-bundle.crt file is added to
# /etc in that profile. Then, the nix2 driver binds all files from that
# profile in the root directory, making ca-bundle.crt available directly under /etc.
# Reference: see https://gist.github.com/CMCDragonkai/1ae4f4b5edeb021ca7bb1d271caca999
task "nix-curl-ssl" {
driver = "nix2"
config {
packages = [
"curl", "cacert"
]
command = "curl"
args = [
"https://nixos.org"
]
}
env = {
SSL_CERT_FILE = "/etc/ssl/certs/ca-bundle.crt"
}
}
}
}

View file

@ -0,0 +1,35 @@
job "nix2-example-service" {
datacenters = ["dc1"]
type = "service"
group "example" {
# This task defines a server that runs a simple python file server on port 8080,
# which allows to explore the contents of the filesystem namespace as visible
# by processes that run inside the task.
# A bunch of utilities are included as well, so that you can exec into the container
# and explore what's inside by yourself.
task "nix-python-serve-http" {
driver = "nix2"
config {
packages = [
"python3",
"bash",
"coreutils",
"curl",
"nix",
"git",
"cacert",
"strace",
"gnugrep",
"mount",
]
command = "python3"
args = [ "-m", "http.server", "8080" ]
}
env = {
SSL_CERT_FILE = "/etc/ssl/certs/ca-bundle.crt"
}
}
}
}

View file

@ -1,24 +0,0 @@
job "example" {
datacenters = ["dc1"]
type = "batch"
group "example" {
task "test-nix-hello" {
driver = "nix2"
config {
command = "sh"
args = [
"-c",
"pwd; ls -l *; mount; hello"
]
packages = [
"github:NixOS/nixpkgs#coreutils",
"github:NixOS/nixpkgs#bash",
"github:NixOS/nixpkgs#hello"
]
}
user = "lx"
}
}
}

View file

@ -1,28 +0,0 @@
job "example2" {
datacenters = ["dc1"]
type = "service"
group "example" {
task "server" {
driver = "nix2"
config {
packages = [
"github:nixos/nixpkgs#python3",
"github:nixos/nixpkgs#bash",
"github:nixos/nixpkgs#coreutils",
"github:nixos/nixpkgs#curl",
"github:nixos/nixpkgs#nix",
"github:nixos/nixpkgs#git",
"github:nixos/nixpkgs#cacert",
"github:nixos/nixpkgs#strace",
"github:nixos/nixpkgs#gnugrep",
"github:nixos/nixpkgs#mount",
]
command = "python3"
args = [ "-m", "http.server", "8080" ]
}
user = "lx"
}
}
}

View file

@ -68,6 +68,10 @@ var (
hclspec.NewAttr("default_ipc_mode", "string", false), hclspec.NewAttr("default_ipc_mode", "string", false),
hclspec.NewLiteral(`"private"`), hclspec.NewLiteral(`"private"`),
), ),
"default_nixpkgs": hclspec.NewDefault(
hclspec.NewAttr("default_nixpkgs", "string", false),
hclspec.NewLiteral(`"github:nixos/nixpkgs/nixos-22.05"`),
),
"allow_caps": hclspec.NewDefault( "allow_caps": hclspec.NewDefault(
hclspec.NewAttr("allow_caps", "list(string)", false), hclspec.NewAttr("allow_caps", "list(string)", false),
hclspec.NewLiteral(capabilities.HCLSpecLiteral), hclspec.NewLiteral(capabilities.HCLSpecLiteral),
@ -89,6 +93,7 @@ var (
"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false), "ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
"cap_add": hclspec.NewAttr("cap_add", "list(string)", false), "cap_add": hclspec.NewAttr("cap_add", "list(string)", false),
"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
"nixpkgs": hclspec.NewAttr("nixpkgs", "string", false),
"packages": hclspec.NewAttr("packages", "list(string)", false), "packages": hclspec.NewAttr("packages", "list(string)", false),
}) })
@ -153,6 +158,9 @@ type Config struct {
// exec-based task drivers. // exec-based task drivers.
DefaultModeIPC string `codec:"default_ipc_mode"` DefaultModeIPC string `codec:"default_ipc_mode"`
// Nixpkgs flake to use
DefaultNixpkgs string `codec:"default_nixpkgs"`
// AllowCaps configures which Linux Capabilities are enabled for tasks // AllowCaps configures which Linux Capabilities are enabled for tasks
// running on this node. // running on this node.
AllowCaps []string `codec:"allow_caps"` AllowCaps []string `codec:"allow_caps"`
@ -204,6 +212,9 @@ type TaskConfig struct {
// Must be "private" or "host" if set. // Must be "private" or "host" if set.
ModeIPC string `codec:"ipc_mode"` ModeIPC string `codec:"ipc_mode"`
// Nixpkgs flake to use
Nixpkgs string `codec:"nixpkgs"`
// CapAdd is a set of linux capabilities to enable. // CapAdd is a set of linux capabilities to enable.
CapAdd []string `codec:"cap_add"` CapAdd []string `codec:"cap_add"`
@ -488,7 +499,19 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
user := cfg.User user := cfg.User
if user == "" { if user == "" {
user = "0" user = "nobody"
}
// Determine the nixpkgs version to use.
nixpkgs := driverConfig.Nixpkgs
if nixpkgs == "" {
nixpkgs = d.config.DefaultNixpkgs
}
// Use that repo for all packages not specified from a flake already.
for i := range driverConfig.Packages {
if !strings.Contains(driverConfig.Packages[i], "#") {
driverConfig.Packages[i] = nixpkgs + "#" + driverConfig.Packages[i]
}
} }
// Prepare NixOS packages and setup a bunch of read-only mounts // Prepare NixOS packages and setup a bunch of read-only mounts
@ -498,19 +521,27 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
AllocID: cfg.AllocID, AllocID: cfg.AllocID,
TaskName: cfg.Name, TaskName: cfg.Name,
Timestamp: time.Now(), Timestamp: time.Now(),
Message: "Building Nix packages and preparing NixOS state", Message: fmt.Sprintf(
"Building Nix packages and preparing NixOS state (using nixpkgs from flake: %s)",
nixpkgs,
),
Annotations: map[string]string{ Annotations: map[string]string{
"packages": strings.Join(driverConfig.Packages, " "), "packages": strings.Join(driverConfig.Packages, " "),
}, },
}) })
taskDirs := cfg.TaskDir() taskDirs := cfg.TaskDir()
systemMounts, err := prepareNixPackages(taskDirs.Dir, driverConfig.Packages) systemMounts, err := prepareNixPackages(taskDirs.Dir, driverConfig.Packages, nixpkgs)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
// Some files are necessary and should be taken from outside if not present already // Some files are necessary and should be taken from outside if not present already
for _, f := range []string{ "/etc/resolv.conf", "/etc/passwd", "/etc/nsswitch.conf" } { etcpaths := []string{
"/etc/nsswitch.conf", // Necessary for most things
"/etc/passwd", // Necessary for username/UID lookup
"/etc/resolv.conf", // Necessary for DNS resolution
}
for _, f := range etcpaths {
if _, ok := systemMounts[f]; !ok { if _, ok := systemMounts[f]; !ok {
systemMounts[f] = f systemMounts[f] = f
} }

View file

@ -2,11 +2,11 @@ package nix2
import ( import (
"bytes" "bytes"
"path/filepath"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
"os/exec" "os/exec"
"path/filepath"
"github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/pluginutils/hclutils"
) )
@ -15,13 +15,13 @@ const (
closureNix = ` closureNix = `
{ path }: { path }:
let let
nixpkgs = builtins.getFlake "github:nixos/nixpkgs/nixos-22.05"; nixpkgs = builtins.getFlake "%s";
inherit (nixpkgs.legacyPackages.x86_64-linux) buildPackages; inherit (nixpkgs.legacyPackages.x86_64-linux) buildPackages;
in buildPackages.closureInfo { rootPaths = builtins.storePath path; } in buildPackages.closureInfo { rootPaths = builtins.storePath path; }
` `
) )
func prepareNixPackages(taskDir string, packages []string) (hclutils.MapStrStr, error) { func prepareNixPackages(taskDir string, packages []string, nixpkgs string) (hclutils.MapStrStr, error) {
mounts := make(hclutils.MapStrStr) mounts := make(hclutils.MapStrStr)
profileLink := filepath.Join(taskDir, "current-profile") profileLink := filepath.Join(taskDir, "current-profile")
@ -31,7 +31,7 @@ func prepareNixPackages(taskDir string, packages []string) (hclutils.MapStrStr,
} }
closureLink := filepath.Join(taskDir, "current-closure") closureLink := filepath.Join(taskDir, "current-closure")
closure, err := nixBuildClosure(profileLink, closureLink) closure, err := nixBuildClosure(profileLink, closureLink, nixpkgs)
if err != nil { if err != nil {
return nil, fmt.Errorf("Build of the flakes failed: %v", err) return nil, fmt.Errorf("Build of the flakes failed: %v", err)
} }
@ -59,8 +59,6 @@ func prepareNixPackages(taskDir string, packages []string) (hclutils.MapStrStr,
} }
} }
mounts[filepath.Join(closure, "registration")] = "/registration"
requisites, err := nixRequisites(closure) requisites, err := nixRequisites(closure)
if err != nil { if err != nil {
return nil, fmt.Errorf("Couldn't determine flake requisites: %v", err) return nil, fmt.Errorf("Couldn't determine flake requisites: %v", err)
@ -98,14 +96,14 @@ func nixBuildProfile(flakes []string, link string) (string, error) {
} }
} }
func nixBuildClosure(profile string, link string) (string, error) { func nixBuildClosure(profile string, link string, nixpkgs string) (string, error) {
cmd := exec.Command( cmd := exec.Command(
"nix", "nix",
"--extra-experimental-features", "nix-command", "--extra-experimental-features", "nix-command",
"--extra-experimental-features", "flakes", "--extra-experimental-features", "flakes",
"build", "build",
"--out-link", link, "--out-link", link,
"--expr", closureNix, "--expr", fmt.Sprintf(closureNix, nixpkgs),
"--impure", "--impure",
"--no-write-lock-file", "--no-write-lock-file",
"--argstr", "path", profile) "--argstr", "path", profile)