initial networking setup

example/agent.hcl

@@ -5,6 +5,16 @@ log_level = "TRACE"
 
 plugin "hello-driver" {
   config {
-    shell = "bash"
+    bridge = "mybridge"
   }
 }
+
+plugin "raw_exec" {
+  config {
+    enabled = true
+  }
+}
+
+consul {
+  address = "127.0.0.1:8500"
+}

example/example.nomad

@@ -3,15 +3,33 @@
 
 job "example" {
   datacenters = ["dc1"]
-  type        = "batch"
+  type        = "service"
 
   group "example" {
+    network {
+#      mode = "driver"
+      port "http" {
+        to = 8080
+      }
+    }
+
     task "hello-world" {
       driver = "hello-world-example"
 
       config {
-        greeting = "hello"
+        unikernel = "/tmp/hello-key.hvt"
+        # network "device1" {
+        #   ports = ["http", ...]
+        #   ip_arg = "--monitoring"
+        #   gateway_arg = "--xxx"
+        # }
+        # args = [...]
       }
     }
+
+    service {
+      port = "http"
+      name = "hello-service"
+    }
   }
 }

hello/driver.go

@@ -5,12 +5,17 @@ package hello
 
 import (
 	"context"
+	"crypto/md5"
+	// "encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/hashicorp/consul-template/signals"
@@ -42,6 +47,15 @@ const (
 	// this is used to allow modification and migration of the task schema
 	// used by the plugin
 	taskHandleVersion = 1
+
+	// TODO: should these be made configurable?
+	tapDeviceBaseName = "miragetap"
+	iptablesChainBase = "MIRAGE"
+
+	// TODO: make this configurable
+	// addresses in the bridge are <bridgeRangeHi>.<bridgeRangeLo>.0.0/16
+	bridgeRangeHi = 10
+	bridgeRangeLo = 99
 )
 
 var (
@@ -71,9 +85,9 @@ var (
 		//       shell = "fish"
 		//     }
 		//   }
-		"shell": hclspec.NewDefault(
-			hclspec.NewAttr("shell", "string", false),
-			hclspec.NewLiteral(`"bash"`),
+		"bridge": hclspec.NewDefault(
+			hclspec.NewAttr("bridge", "string", false),
+			hclspec.NewLiteral(`"mirage"`),
 		),
 	})
 
@@ -99,9 +113,9 @@ var (
 		//       }
 		//     }
 		//   }
-		"greeting": hclspec.NewDefault(
-			hclspec.NewAttr("greeting", "string", false),
-			hclspec.NewLiteral(`"Hello, World!"`),
+		"unikernel": hclspec.NewDefault(
+			hclspec.NewAttr("unikernel", "string", true),
+			hclspec.NewLiteral(`""`),
 		),
 	})
 
@@ -115,6 +129,11 @@ var (
 		// https://godoc.org/github.com/hashicorp/nomad/plugins/drivers#Capabilities
 		SendSignals: true,
 		Exec:        false,
+		FSIsolation: drivers.FSIsolationNone,
+		NetIsolationModes: []drivers.NetIsolationMode{
+			drivers.NetIsolationModeTask,
+		},
+		MountConfigs: drivers.MountConfigSupportNone,
 	}
 )
 
@@ -125,7 +144,7 @@ type Config struct {
 	// This struct is the decoded version of the schema defined in the
 	// configSpec variable above. It's used to convert the HCL configuration
 	// passed by the Nomad agent into Go contructs.
-	Shell string `codec:"shell"`
+	Bridge string `codec:"bridge"`
 }
 
 // TaskConfig contains configuration information for a task that runs with
@@ -136,7 +155,7 @@ type TaskConfig struct {
 	// This struct is the decoded version of the schema defined in the
 	// taskConfigSpec variable above. It's used to convert the string
 	// configuration for the task into Go contructs.
-	Greeting string `codec:"greeting"`
+	Unikernel string `codec:"unikernel"`
 }
 
 // TaskState is the runtime state which is encoded in the handle returned to
@@ -157,6 +176,19 @@ type TaskState struct {
 	// in-memory representation of the running tasks using the RecoverTask()
 	// method below.
 	Pid        int
+	NetDevices map[string]NetDeviceConfig
+}
+
+type NetDeviceConfig struct {
+	TapDevice string
+	Ip        string
+	Mac       string
+	Ports     []NetDevicePortConfig
+}
+
+type NetDevicePortConfig struct {
+	From int // port on the host
+	To   int // port inside the bridge
 }
 
 // HelloDriverPlugin is an example driver plugin. When provisioned in a job,
@@ -187,6 +219,18 @@ type HelloDriverPlugin struct {
 	logger hclog.Logger
 }
 
+// A unikernel manifest. Obtained by deserializing JSON returned by solo5-elftool.
+type unikernelManifest struct {
+	Type    string            `json:"type"`
+	Version int               `json:"version"`
+	Devices []unikernelDevice `json:"devices"`
+}
+
+type unikernelDevice struct {
+	Name string `json:"name"`
+	Type string `json:"type"` // BLOCK_BASIC, NET_BASIC
+}
+
 // NewPlugin returns a new example driver plugin
 func NewPlugin(logger hclog.Logger) drivers.DriverPlugin {
 	ctx, cancel := context.WithCancel(context.Background())
@@ -212,6 +256,22 @@ func (d *HelloDriverPlugin) ConfigSchema() (*hclspec.Spec, error) {
 	return configSpec, nil
 }
 
+func gatewayIp() string {
+	return fmt.Sprintf("%d.%d.0.1", bridgeRangeHi, bridgeRangeLo)
+}
+
+func broadcastIp() string {
+	return fmt.Sprintf("%d.%d.0.255", bridgeRangeHi, bridgeRangeLo)
+}
+
+func iptablesChainOut() string {
+	return iptablesChainBase + "_OUT"
+}
+
+func iptablesChainIn() string {
+	return iptablesChainBase + "_IN"
+}
+
 // SetConfig is called by the client to pass the configuration for the plugin.
 func (d *HelloDriverPlugin) SetConfig(cfg *base.Config) error {
 	var config Config
@@ -233,10 +293,10 @@ func (d *HelloDriverPlugin) SetConfig(cfg *base.Config) error {
 	//
 	// In the example below we check if the shell specified by the user is
 	// supported by the plugin.
-	shell := d.config.Shell
-	if shell != "bash" && shell != "fish" {
-		return fmt.Errorf("invalid shell %s", d.config.Shell)
-	}
+	// shell := d.config.Shell
+	// if shell != "bash" && shell != "fish" {
+	// 	return fmt.Errorf("invalid shell %s", d.config.Shell)
+	// }
 
 	// Save the Nomad agent configuration
 	if cfg.AgentConfig != nil {
@@ -248,6 +308,40 @@ func (d *HelloDriverPlugin) SetConfig(cfg *base.Config) error {
 	// Here you can use the config values to initialize any resources that are
 	// shared by all tasks that use this driver, such as a daemon process.
 
+	// create the bridge and iptables chains if they do not exist
+	// TODO: handle errors
+
+	// echo "1" > /proc/sys/net/ipv4/ip_forward
+	// ip link add name mybridge type bridge
+	// ip link set dev mybridge up
+	// ip address add 10.0.0.1/24 broadcast 10.0.0.255 dev mybridge
+	exec.Command("sh", "-c", `echo 1 > /proc/sys/net/ipv4/ip_forward`).Run()
+	exec.Command("ip", "link", "add", "name", d.config.Bridge, "type", "bridge").Run()
+	exec.Command("ip", "link", "set", "dev", d.config.Bridge, "up").Run()
+	// TODO: remove broadcast?
+	exec.Command("ip", "address", "add", gatewayIp()+"/16", "broadcast",
+		broadcastIp(), "dev", d.config.Bridge).Run()
+
+	// iptables -t nat -N MIRAGE_OUT
+	// iptables -t nat -A POSTROUTING -j MIRAGE_OUT
+	// + a rule for each unikernel...
+	exec.Command("iptables", "-t", "nat", "-N", iptablesChainOut()).Run()
+	// FIXME: this one gets added unconditionally even if it already exists
+	exec.Command("iptables", "-t", "nat", "-A", "POSTROUTING", "-j", iptablesChainOut()).Run()
+
+	// iptables -t nat -N MIRAGE_IN
+	// iptables -t nat -A PREROUTING -m addrtype --dst-type LOCAL -j MIRAGE_IN
+	// + a rule for each unikernel...
+	exec.Command("iptables", "-t", "nat", "-N", iptablesChainIn()).Run()
+	// FIXME: this one gets added unconditionally even if it already exists
+	exec.Command("iptables", "-t", "nat", "-A", "PREROUTING",
+		"-m", "addrtype", "--dst-type", "LOCAL",
+		"-j", iptablesChainIn()).Run()
+
+	// TODO:
+	// iptables -t nat -I OUTPUT -o lo -j MIRAGE_IN
+	// sysctl -w net.ipv4.conf.all.route_localnet=1
+
 	return nil
 }
 
@@ -311,34 +405,392 @@ func (d *HelloDriverPlugin) buildFingerprint() *drivers.Fingerprint {
 	// the node in which the plugin is running (specific library availability,
 	// installed versions of a software etc.). These attributes can then be
 	// used by an operator to set job constrains.
-	//
-	// In the example below we check if the shell specified by the user exists
-	// in the node.
-	shell := d.config.Shell
 
-	cmd := exec.Command("which", shell)
-	if err := cmd.Run(); err != nil {
+	// FIXME: is this the right place to check this?
+	// do we need to check for the binaries existence every N seconds?
+	// should we do these checks in SetConfig() instead?
+
+	// check that solo5-hvt exists
+	hvtCmd := exec.Command("which", "solo5-hvt")
+	hvtPath, hvtErr := hvtCmd.Output()
+	if hvtErr != nil {
 		return &drivers.Fingerprint{
 			Health:            drivers.HealthStateUndetected,
-			HealthDescription: fmt.Sprintf("shell %s not found", shell),
+			HealthDescription: "solo5-hvt not found",
 		}
 	}
 
-	// We also set the shell and its version as attributes
-	cmd = exec.Command(shell, "--version")
-	if out, err := cmd.Output(); err != nil {
-		d.logger.Warn("failed to find shell version: %v", err)
-	} else {
-		re := regexp.MustCompile("[0-9]\\.[0-9]\\.[0-9]")
-		version := re.FindString(string(out))
-
-		fp.Attributes["driver.hello.shell_version"] = structs.NewStringAttribute(version)
-		fp.Attributes["driver.hello.shell"] = structs.NewStringAttribute(shell)
+	// check that solo5-elftool exists
+	elftoolCmd := exec.Command("which", "solo5-elftool")
+	elftoolPath, elftoolErr := elftoolCmd.Output()
+	if elftoolErr != nil {
+		return &drivers.Fingerprint{
+			Health:            drivers.HealthStateUndetected,
+			HealthDescription: fmt.Sprintf("solo5-elftool not found"),
 		}
+	}
+
+	// We set their paths as attributes
+	fp.Attributes["driver.hello.solo5-hvt"] = structs.NewStringAttribute(string(hvtPath))
+	fp.Attributes["driver.hello.solo5-elftool"] = structs.NewStringAttribute(string(elftoolPath))
 
 	return fp
 }
 
+func getFreshTapDeviceName() (string, error) {
+	cmd := exec.Command("ip", "tuntap", "show")
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("running '%s' failed: %v", cmd.String(), err)
+	}
+	tuntapOut, _ := cmd.Output()
+
+	var tuntapLines []string
+	if string(tuntapOut) == "" {
+		tuntapLines = []string{}
+	} else {
+		tuntapLines = strings.Split(string(tuntapOut), "\n")
+	}
+
+	maxId := 0
+	for _, line := range(tuntapLines) {
+		tapName, _, found := strings.Cut(line, ":")
+		if !found {
+			break
+		}
+		tapIdStr, found := strings.CutPrefix(tapName, tapDeviceBaseName)
+		if !found {
+			break
+		}
+		tapId, err := strconv.Atoi(tapIdStr)
+		if err != nil {
+			break
+		}
+		if tapId > maxId {
+			maxId = tapId
+		}
+	}
+
+	return fmt.Sprintf("%s%d", tapDeviceBaseName, maxId+1), nil
+}
+
+func destroyTapDevice(name string) {
+	exec.Command("ip", "tuntap", "del", name, "mode", "tap").Run()
+}
+
+func createTapDevice(name string, bridge string) error {
+	// ip tuntap add <tap> mode tap
+	cmd := exec.Command("ip", "tuntap", "add", name, "mode", "tap")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("'%s' failed: %v", cmd.String(), err)
+	}
+
+	// ip link set dev <tap> up
+	cmd = exec.Command("ip", "link", "set", "dev", name, "up")
+	if err := cmd.Run(); err != nil {
+		destroyTapDevice(name)
+		return fmt.Errorf("'%s' failed: %v", cmd.String(), err)
+	}
+
+	// ip link set dev <tap> master <bridge>
+	cmd = exec.Command("ip", "link", "set", "dev", name, "master", bridge)
+	if err := cmd.Run(); err != nil {
+		destroyTapDevice(name)
+		return fmt.Errorf("'%s' failed: %v", cmd.String(), err)
+	}
+
+	return nil
+}
+
+func getMacAddr(unikernel string, bridge string) string {
+	// deterministic mac address computation
+	// follow Albatross and use the VEB Kombinat Robotron prefix
+	// (thanks again, friends!)
+	ours := md5.Sum([]byte(bridge + unikernel))
+	return fmt.Sprintf(
+		"%x:%x:%x:%x:%x:%x",
+		0x00, 0x80, 0x41,
+		ours[0], ours[1], ours[2],
+	)
+}
+
+func nextIp(ip [2]int) ([2]int, error) {
+	nip := [2]int{ip[0], ip[1]}
+	// FIX: 255.255 is for broadcast
+	// .255 is for broadcast
+	if nip[1] < 255 {
+		nip[1]++
+		return nip, nil
+	}
+	nip[1] = 0
+	nip[0]++
+
+	if nip[0] < 256 {
+		return nip, nil
+	}
+	return [2]int{}, fmt.Errorf("nextIp")
+}
+
+func getFreshIp() (string, error) {
+	// iptables -t nat -L MIRAGE_OUT
+	// then grep for: XX.XX.XX.XX to find existing ips
+	// (where the first two bytes correspond to the range of our bridge)
+	cmd := exec.Command("iptables", "-t", "nat", "-L", iptablesChainOut())
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("'%s' failed: %v", cmd.String(), err)
+	}
+	chainStr, _ := cmd.Output()
+	chainLines := strings.Split(string(chainStr), "\n")
+
+	r, _ := regexp.Compile(
+		strconv.Itoa(bridgeRangeHi) + "." +
+        strconv.Itoa(bridgeRangeLo) + `.(\d+).(\d+)`)
+
+	usedIps := make(map[[2]int]bool)
+	for i := 2; i < len(chainLines); i++ {
+		line := chainLines[i]
+		matches := r.FindStringSubmatch(line)
+		b1, _ := strconv.Atoi(matches[1])
+		b2, _ := strconv.Atoi(matches[2])
+		usedIps[[2]int{b1, b2}] = true
+	}
+
+	// XX.XX.0.1 is the gateway; start at XX.XX.0.2
+	ip, err := [2]int{0, 2}, error(nil)
+	for err == nil {
+		if !usedIps[ip] {
+			break
+		}
+		ip, err = nextIp(ip)
+	}
+
+	if err != nil {
+		return "", fmt.Errorf("no available ip found")
+	}
+	return fmt.Sprintf("%d.%d.%d.%d", bridgeRangeHi, bridgeRangeLo, ip[0], ip[1]), nil
+}
+
+func getNetDevicesPorts(
+	netDevicesNames []string,
+	taskCfg TaskConfig,
+	cfg *drivers.TaskConfig,
+) map[string][]NetDevicePortConfig {
+	ports := make(map[string][]NetDevicePortConfig)
+
+	if len(netDevicesNames) == 0 {
+		return ports
+	}
+
+	// TODO: extract port<->device assignation from taskCfg
+	// for now, assume that this config is empty, and implement the default
+	// strategy
+
+	for _, name := range netDevicesNames {
+		ports[name] = []NetDevicePortConfig{}
+	}
+
+	var defaultDevice string
+	if _, found := ports["service"]; found {
+		// use 'service' as the default device name when it exists
+		defaultDevice = "service"
+	} else {
+		// otherwise use the first one in the list
+		defaultDevice = netDevicesNames[0]
+	}
+
+	for _, portMapping := range *cfg.Resources.Ports {
+		// only the default case for now
+		ports[defaultDevice] = append(ports[defaultDevice],
+			NetDevicePortConfig{
+				From: portMapping.Value,
+				To:   portMapping.To,
+			})
+	}
+
+	return ports
+}
+
+// fields are "" if not specified
+type NetDeviceArgs struct {
+	Ip      string
+	Gateway string
+}
+
+func getNetDevicesArgs(
+	netDevicesNames []string,
+	taskCfg TaskConfig,
+) map[string]NetDeviceArgs {
+	args := make(map[string]NetDeviceArgs)
+	// TODO: extract the flag<->device assignation from taskCfg
+	// for now, always assign --ipv4 to service and fail if there is another device
+	for _, name := range netDevicesNames {
+		if name == "service" {
+			args["service"] = NetDeviceArgs{
+				Ip:      "--ipv4",
+				Gateway: "--ipv4-gateway",
+			}
+		}
+	}
+	return args
+}
+
+func rollbackIptablesRules(rules [][]string) {
+	for j := len(rules) - 1; j >= 0; j-- {
+		args := append([]string{"-t", "nat", "-D"}, rules[j]...)
+		exec.Command("iptables", args...).Run()
+	}
+}
+
+// run a list of iptables commands of the form "iptables -t nat -A ..." as a
+// single transaction: if one rule fails, we roll back earlier rules before
+// returning an error
+func execIptablesRules(rules [][]string) error {
+	for i, rule := range rules {
+		args := append([]string{"-t", "nat", "-A"}, rule...)
+		cmd := exec.Command("iptables", args...)
+		if err := cmd.Run(); err != nil {
+			rollbackIptablesRules(rules[0:i])
+			return fmt.Errorf("'%s' failed: %v", cmd.String(), err)
+		}
+	}
+	return nil
+}
+
+func netDeviceIptablesRules(ip string, ports []NetDevicePortConfig) [][]string {
+	// iptables rules to talk to the outside and setup incoming port redirections
+	rr := [][]string{}
+	addrule := func(args ...string) { rr = append(rr, args) }
+
+	// allow the ip to talk to the outside
+	// iptables -t nat -A MIRAGE_OUT -s <ip>/32 -j MASQUERADE
+	addrule(iptablesChainOut(), "-s", ip+"/32", "-j", "MASQUERADE")
+
+	// NAT port redirections
+	// iptables -t nat -A MIRAGE_IN ! -s <ip>/32 -p tcp -m tcp --dport 53
+	//   -j DNAT --to-destination <ip>:53
+	// iptables -t nat -A MIRAGE_IN ! -s <ip>/32 -p udp -m udp --dport 53
+	//   -j DNAT --to-destination <ip>:53
+	for _, port := range ports {
+		addrule(iptablesChainIn(), "!", "-s", ip+"/32", "-p", "tcp",
+			"--dport", strconv.Itoa(port.From), /* TODO:check */
+			"-j", "DNAT", "--to-destination", ip+":"+strconv.Itoa(port.To))
+		addrule(iptablesChainIn(), "!", "-s", ip+"/32", "-p", "udp",
+			"--dport", strconv.Itoa(port.From), /* TODO:check from/to */
+			"-j", "DNAT", "--to-destination", ip+":"+strconv.Itoa(port.To))
+	}
+	return rr
+}
+
+func setupNetDevice(
+	deviceName string,
+	unikernel string,
+	bridge string,
+	ports []NetDevicePortConfig,
+	logger hclog.Logger,
+) (NetDeviceConfig, error) {
+	// list existing tap interfaces, and get a fresh tap interface number
+	tapDevice, err := getFreshTapDeviceName()
+	if err != nil {
+		return NetDeviceConfig{}, fmt.Errorf("could not get a fresh tap device name: %v", err)
+	}
+	logger.Debug("got tap device name", tapDevice)
+
+	// create new tap device
+	if err := createTapDevice(tapDevice, bridge); err != nil {
+		return NetDeviceConfig{}, fmt.Errorf("could not create tap device: %v", err)
+	}
+	logger.Debug("successfully created tap device")
+
+	// get a mac address for the unikernel
+	macAddr := getMacAddr(unikernel, bridge)
+	logger.Debug("mac address", macAddr)
+
+	// find an unused ip address
+	ipAddr, err := getFreshIp()
+	if err != nil {
+		destroyTapDevice(tapDevice)
+		return NetDeviceConfig{}, fmt.Errorf("could not get a fresh ip address: %v", err)
+	}
+	logger.Debug("got ip address", ipAddr)
+
+	// execute iptables rules
+	if err := execIptablesRules(netDeviceIptablesRules(ipAddr, ports)); err != nil {
+		destroyTapDevice(tapDevice)
+		return NetDeviceConfig{}, err
+	}
+
+	return NetDeviceConfig{
+		TapDevice: tapDevice,
+		Ip:        ipAddr,
+		Mac:       macAddr,
+		Ports:     ports,
+	}, nil
+}
+
+func destroyNetDevice(device NetDeviceConfig) {
+	rollbackIptablesRules(netDeviceIptablesRules(device.Ip, device.Ports))
+	destroyTapDevice(device.TapDevice)
+}
+
+func destroyNetDevices(devices map[string]NetDeviceConfig) {
+	for _, d := range devices {
+		destroyNetDevice(d)
+	}
+}
+
+func setupNetDevices(
+	netDevicesPorts map[string][]NetDevicePortConfig,
+	unikernel string,
+	bridge string,
+	logger hclog.Logger,
+) (map[string]NetDeviceConfig, error) {
+	netDevices := make(map[string]NetDeviceConfig)
+	for deviceName, devicePorts := range netDevicesPorts {
+		deviceCfg, err :=
+			setupNetDevice(deviceName, unikernel, bridge, devicePorts, logger)
+		if err != nil {
+			destroyNetDevices(netDevices)
+			return nil,
+				fmt.Errorf("failed to setup net device %s: %v",
+					deviceName, err)
+		}
+		netDevices[deviceName] = deviceCfg
+	}
+	return netDevices, nil
+}
+
+// TODO: also return the list of block device names, when we handle block
+// devices
+func readUnikernelManifest(
+	unikernel string,
+	logger hclog.Logger,
+) ([]string /* net devices */, error) {
+	manifestCmd := exec.Command("solo5-elftool", "query-manifest", unikernel)
+	manifestStr, manifestErr := manifestCmd.Output()
+	if manifestErr != nil {
+		return nil, fmt.Errorf("failed to query unikernel manifest: %v", manifestErr)
+	}
+
+	logger.Debug("unikernel manifest", manifestStr)
+
+	var manifest unikernelManifest
+	if err := json.Unmarshal(manifestStr, &manifest); err != nil {
+		return nil, fmt.Errorf("could not parse manifest: %v", err)
+	}
+
+	netDevicesNames := []string{}
+	for _, d := range manifest.Devices {
+		switch d.Type {
+		case "NET_BASIC":
+			netDevicesNames = append(netDevicesNames, d.Name)
+		// case "BLOCK_BASIC":
+		// TODO
+		default:
+			return nil, fmt.Errorf("unhandled device type: %s", d.Type)
+		}
+	}
+	return netDevicesNames, nil
+}
+
 // StartTask returns a task handle and a driver network if necessary.
 func (d *HelloDriverPlugin) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) {
 	if _, ok := d.tasks.Get(cfg.ID); ok {
@@ -375,28 +827,80 @@ func (d *HelloDriverPlugin) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHan
 		LogLevel: "debug",
 	}
 
-	exec, pluginClient, err := executor.CreateExecutor(d.logger, d.nomadConfig, executorConfig)
+	logger := d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID)
+
+	netDevicesNames, err := readUnikernelManifest(driverConfig.Unikernel, logger)
+	if err != nil {
+		return nil, nil, err
+	}
+	netDevicesPorts := getNetDevicesPorts(netDevicesNames, driverConfig, cfg)
+	netDevicesArgs := getNetDevicesArgs(netDevicesNames, driverConfig)
+	netDevices, err :=
+		setupNetDevices(netDevicesPorts, driverConfig.Unikernel,
+			d.config.Bridge, logger)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	solo5Args := []string{}
+	unikernelArgs := []string{}
+	// --net:<service from manifest>=<tapDevice>
+	// --net-mac:<service from manifest>=<macAddr>
+	for name, device := range netDevices {
+		solo5Args = append(solo5Args,
+			fmt.Sprintf("--net:%s=%s", name, device.TapDevice))
+		solo5Args = append(solo5Args,
+			fmt.Sprintf("--net-mac:%s=%s", name, device.Mac))
+	}
+	// --ipv4=<ipAddr>/16 --ipv4-gateway=<gatewayIp()> (by default)
+	// + user-specified equivalent arguments for additional net devices
+	for name, arg := range netDevicesArgs {
+		if arg.Ip != "" {
+			unikernelArgs = append(unikernelArgs,
+				fmt.Sprintf("%s=%s/16", arg.Ip, netDevices[name].Ip))
+		}
+		if arg.Gateway != "" {
+			unikernelArgs = append(unikernelArgs,
+				fmt.Sprintf("%s=%s", arg.Gateway, gatewayIp()))
+		}
+	}
+
+	// TODO: collect additional arguments from the config file
+
+	execu, pluginClient, err := executor.CreateExecutor(logger, d.nomadConfig, executorConfig)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to create executor: %v", err)
 	}
 
-	echoCmd := fmt.Sprintf(`echo "%s"`, driverConfig.Greeting)
+	cfgJson, _ := json.MarshalIndent(cfg, "", "  ")
+	// cfgStr := base64.StdEncoding.EncodeToString(cfgJson)
+	logger.Debug("task config", string(cfgJson))
+
+	netCfgStr, _ := exec.Command("ip", "a").Output()
+	logger.Debug("host network config", string(netCfgStr))
+
+	args := append(solo5Args, []string{"--", driverConfig.Unikernel}...)
+	args = append(args, unikernelArgs...)
+
+	logger.Debug("solo5 command", exec.Command("solo5-hvt", args...).String())
+
 	execCmd := &executor.ExecCommand{
-		Cmd:        d.config.Shell,
-		Args:       []string{"-c", echoCmd},
+		Cmd:        "solo5-hvt",
+		Args:       args,
 		StdoutPath: cfg.StdoutPath,
 		StderrPath: cfg.StderrPath,
 	}
 
-	ps, err := exec.Launch(execCmd)
+	ps, err := execu.Launch(execCmd)
 	if err != nil {
 		pluginClient.Kill()
 		return nil, nil, fmt.Errorf("failed to launch command with executor: %v", err)
 	}
 
 	h := &taskHandle{
-		exec:         exec,
+		exec:         execu,
 		pid:          ps.Pid,
+		netDevices:   netDevices,
 		pluginClient: pluginClient,
 		taskConfig:   cfg,
 		procState:    drivers.TaskStateRunning,
@@ -407,11 +911,13 @@ func (d *HelloDriverPlugin) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHan
 	driverState := TaskState{
 		ReattachConfig: structs.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()),
 		Pid:            ps.Pid,
+		NetDevices:     netDevices,
 		TaskConfig:     cfg,
 		StartedAt:      h.startedAt,
 	}
 
 	if err := handle.SetDriverState(&driverState); err != nil {
+		d.logger.Error("failed to start task, error setting driver state", "error", err)
 		return nil, nil, fmt.Errorf("failed to set driver state: %v", err)
 	}
 
@@ -460,6 +966,7 @@ func (d *HelloDriverPlugin) RecoverTask(handle *drivers.TaskHandle) error {
 	h := &taskHandle{
 		exec:         execImpl,
 		pid:          taskState.Pid,
+		netDevices:   taskState.NetDevices,
 		pluginClient: pluginClient,
 		taskConfig:   taskState.TaskConfig,
 		procState:    drivers.TaskStateRunning,
@@ -575,6 +1082,12 @@ func (d *HelloDriverPlugin) DestroyTask(taskID string, force bool) error {
 		handle.pluginClient.Kill()
 	}
 
+	// remove the relevant iptables rules and the tap device
+	destroyNetDevices(handle.netDevices)
+
+	// TODO: destroy the bridge and the iptables chains if no one is using them
+	// anymore
+
 	d.tasks.Delete(taskID)
 	return nil
 }

hello/handle.go

@@ -33,6 +33,7 @@ type taskHandle struct {
 
 	// TODO: add any extra relevant information about the task.
 	pid        int
+	netDevices map[string]NetDeviceConfig
 }
 
 func (h *taskHandle) TaskStatus() *drivers.TaskStatus {
@@ -48,6 +49,7 @@ func (h *taskHandle) TaskStatus() *drivers.TaskStatus {
 		ExitResult:  h.exitResult,
 		DriverAttributes: map[string]string{
 			"pid": strconv.Itoa(h.pid),
+			// TODO: emit more attributes?
 		},
 	}
 }