From 14476852d40d1d94f51bac536e44d36b01a25679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arma=C3=ABl=20Gu=C3=A9neau?= Date: Fri, 30 Jun 2023 19:08:08 +0200 Subject: [PATCH] initial networking setup --- example/agent.hcl | 12 +- example/example.nomad | 22 +- hello/driver.go | 585 +++++++++++++++++++++++++++++++++++++++--- hello/handle.go | 4 +- 4 files changed, 583 insertions(+), 40 deletions(-) diff --git a/example/agent.hcl b/example/agent.hcl index 0baae0e..25b0efd 100644 --- a/example/agent.hcl +++ b/example/agent.hcl @@ -5,6 +5,16 @@ log_level = "TRACE" plugin "hello-driver" { config { - shell = "bash" + bridge = "mybridge" } } + +plugin "raw_exec" { + config { + enabled = true + } +} + +consul { + address = "127.0.0.1:8500" +} diff --git a/example/example.nomad b/example/example.nomad index 6742c90..e93d0e8 100644 --- a/example/example.nomad +++ b/example/example.nomad @@ -3,15 +3,33 @@ job "example" { datacenters = ["dc1"] - type = "batch" + type = "service" group "example" { + network { +# mode = "driver" + port "http" { + to = 8080 + } + } + task "hello-world" { driver = "hello-world-example" config { - greeting = "hello" + unikernel = "/tmp/hello-key.hvt" + # network "device1" { + # ports = ["http", ...] + # ip_arg = "--monitoring" + # gateway_arg = "--xxx" + # } + # args = [...] } } + + service { + port = "http" + name = "hello-service" + } } } diff --git a/hello/driver.go b/hello/driver.go index 71bac54..678a78b 100644 --- a/hello/driver.go +++ b/hello/driver.go @@ -5,12 +5,17 @@ package hello import ( "context" + "crypto/md5" + // "encoding/base64" + "encoding/json" "errors" "fmt" "os" "os/exec" "path/filepath" "regexp" + "strconv" + "strings" "time" "github.com/hashicorp/consul-template/signals" @@ -42,6 +47,15 @@ const ( // this is used to allow modification and migration of the task schema // used by the plugin taskHandleVersion = 1 + + // TODO: should these be made configurable? + tapDeviceBaseName = "miragetap" + iptablesChainBase = "MIRAGE" + + // TODO: make this configurable + // addresses in the bridge are ..0.0/16 + bridgeRangeHi = 10 + bridgeRangeLo = 99 ) var ( @@ -71,9 +85,9 @@ var ( // shell = "fish" // } // } - "shell": hclspec.NewDefault( - hclspec.NewAttr("shell", "string", false), - hclspec.NewLiteral(`"bash"`), + "bridge": hclspec.NewDefault( + hclspec.NewAttr("bridge", "string", false), + hclspec.NewLiteral(`"mirage"`), ), }) @@ -99,9 +113,9 @@ var ( // } // } // } - "greeting": hclspec.NewDefault( - hclspec.NewAttr("greeting", "string", false), - hclspec.NewLiteral(`"Hello, World!"`), + "unikernel": hclspec.NewDefault( + hclspec.NewAttr("unikernel", "string", true), + hclspec.NewLiteral(`""`), ), }) @@ -115,6 +129,11 @@ var ( // https://godoc.org/github.com/hashicorp/nomad/plugins/drivers#Capabilities SendSignals: true, Exec: false, + FSIsolation: drivers.FSIsolationNone, + NetIsolationModes: []drivers.NetIsolationMode{ + drivers.NetIsolationModeTask, + }, + MountConfigs: drivers.MountConfigSupportNone, } ) @@ -125,7 +144,7 @@ type Config struct { // This struct is the decoded version of the schema defined in the // configSpec variable above. It's used to convert the HCL configuration // passed by the Nomad agent into Go contructs. - Shell string `codec:"shell"` + Bridge string `codec:"bridge"` } // TaskConfig contains configuration information for a task that runs with @@ -136,7 +155,7 @@ type TaskConfig struct { // This struct is the decoded version of the schema defined in the // taskConfigSpec variable above. It's used to convert the string // configuration for the task into Go contructs. - Greeting string `codec:"greeting"` + Unikernel string `codec:"unikernel"` } // TaskState is the runtime state which is encoded in the handle returned to @@ -156,7 +175,20 @@ type TaskState struct { // will respawn a new instance of the plugin and try to restore its // in-memory representation of the running tasks using the RecoverTask() // method below. - Pid int + Pid int + NetDevices map[string]NetDeviceConfig +} + +type NetDeviceConfig struct { + TapDevice string + Ip string + Mac string + Ports []NetDevicePortConfig +} + +type NetDevicePortConfig struct { + From int // port on the host + To int // port inside the bridge } // HelloDriverPlugin is an example driver plugin. When provisioned in a job, @@ -187,6 +219,18 @@ type HelloDriverPlugin struct { logger hclog.Logger } +// A unikernel manifest. Obtained by deserializing JSON returned by solo5-elftool. +type unikernelManifest struct { + Type string `json:"type"` + Version int `json:"version"` + Devices []unikernelDevice `json:"devices"` +} + +type unikernelDevice struct { + Name string `json:"name"` + Type string `json:"type"` // BLOCK_BASIC, NET_BASIC +} + // NewPlugin returns a new example driver plugin func NewPlugin(logger hclog.Logger) drivers.DriverPlugin { ctx, cancel := context.WithCancel(context.Background()) @@ -212,6 +256,22 @@ func (d *HelloDriverPlugin) ConfigSchema() (*hclspec.Spec, error) { return configSpec, nil } +func gatewayIp() string { + return fmt.Sprintf("%d.%d.0.1", bridgeRangeHi, bridgeRangeLo) +} + +func broadcastIp() string { + return fmt.Sprintf("%d.%d.0.255", bridgeRangeHi, bridgeRangeLo) +} + +func iptablesChainOut() string { + return iptablesChainBase + "_OUT" +} + +func iptablesChainIn() string { + return iptablesChainBase + "_IN" +} + // SetConfig is called by the client to pass the configuration for the plugin. func (d *HelloDriverPlugin) SetConfig(cfg *base.Config) error { var config Config @@ -233,10 +293,10 @@ func (d *HelloDriverPlugin) SetConfig(cfg *base.Config) error { // // In the example below we check if the shell specified by the user is // supported by the plugin. - shell := d.config.Shell - if shell != "bash" && shell != "fish" { - return fmt.Errorf("invalid shell %s", d.config.Shell) - } + // shell := d.config.Shell + // if shell != "bash" && shell != "fish" { + // return fmt.Errorf("invalid shell %s", d.config.Shell) + // } // Save the Nomad agent configuration if cfg.AgentConfig != nil { @@ -248,6 +308,40 @@ func (d *HelloDriverPlugin) SetConfig(cfg *base.Config) error { // Here you can use the config values to initialize any resources that are // shared by all tasks that use this driver, such as a daemon process. + // create the bridge and iptables chains if they do not exist + // TODO: handle errors + + // echo "1" > /proc/sys/net/ipv4/ip_forward + // ip link add name mybridge type bridge + // ip link set dev mybridge up + // ip address add 10.0.0.1/24 broadcast 10.0.0.255 dev mybridge + exec.Command("sh", "-c", `echo 1 > /proc/sys/net/ipv4/ip_forward`).Run() + exec.Command("ip", "link", "add", "name", d.config.Bridge, "type", "bridge").Run() + exec.Command("ip", "link", "set", "dev", d.config.Bridge, "up").Run() + // TODO: remove broadcast? + exec.Command("ip", "address", "add", gatewayIp()+"/16", "broadcast", + broadcastIp(), "dev", d.config.Bridge).Run() + + // iptables -t nat -N MIRAGE_OUT + // iptables -t nat -A POSTROUTING -j MIRAGE_OUT + // + a rule for each unikernel... + exec.Command("iptables", "-t", "nat", "-N", iptablesChainOut()).Run() + // FIXME: this one gets added unconditionally even if it already exists + exec.Command("iptables", "-t", "nat", "-A", "POSTROUTING", "-j", iptablesChainOut()).Run() + + // iptables -t nat -N MIRAGE_IN + // iptables -t nat -A PREROUTING -m addrtype --dst-type LOCAL -j MIRAGE_IN + // + a rule for each unikernel... + exec.Command("iptables", "-t", "nat", "-N", iptablesChainIn()).Run() + // FIXME: this one gets added unconditionally even if it already exists + exec.Command("iptables", "-t", "nat", "-A", "PREROUTING", + "-m", "addrtype", "--dst-type", "LOCAL", + "-j", iptablesChainIn()).Run() + + // TODO: + // iptables -t nat -I OUTPUT -o lo -j MIRAGE_IN + // sysctl -w net.ipv4.conf.all.route_localnet=1 + return nil } @@ -311,34 +405,392 @@ func (d *HelloDriverPlugin) buildFingerprint() *drivers.Fingerprint { // the node in which the plugin is running (specific library availability, // installed versions of a software etc.). These attributes can then be // used by an operator to set job constrains. - // - // In the example below we check if the shell specified by the user exists - // in the node. - shell := d.config.Shell - cmd := exec.Command("which", shell) - if err := cmd.Run(); err != nil { + // FIXME: is this the right place to check this? + // do we need to check for the binaries existence every N seconds? + // should we do these checks in SetConfig() instead? + + // check that solo5-hvt exists + hvtCmd := exec.Command("which", "solo5-hvt") + hvtPath, hvtErr := hvtCmd.Output() + if hvtErr != nil { return &drivers.Fingerprint{ Health: drivers.HealthStateUndetected, - HealthDescription: fmt.Sprintf("shell %s not found", shell), + HealthDescription: "solo5-hvt not found", } } - // We also set the shell and its version as attributes - cmd = exec.Command(shell, "--version") - if out, err := cmd.Output(); err != nil { - d.logger.Warn("failed to find shell version: %v", err) - } else { - re := regexp.MustCompile("[0-9]\\.[0-9]\\.[0-9]") - version := re.FindString(string(out)) - - fp.Attributes["driver.hello.shell_version"] = structs.NewStringAttribute(version) - fp.Attributes["driver.hello.shell"] = structs.NewStringAttribute(shell) + // check that solo5-elftool exists + elftoolCmd := exec.Command("which", "solo5-elftool") + elftoolPath, elftoolErr := elftoolCmd.Output() + if elftoolErr != nil { + return &drivers.Fingerprint{ + Health: drivers.HealthStateUndetected, + HealthDescription: fmt.Sprintf("solo5-elftool not found"), + } } + // We set their paths as attributes + fp.Attributes["driver.hello.solo5-hvt"] = structs.NewStringAttribute(string(hvtPath)) + fp.Attributes["driver.hello.solo5-elftool"] = structs.NewStringAttribute(string(elftoolPath)) + return fp } +func getFreshTapDeviceName() (string, error) { + cmd := exec.Command("ip", "tuntap", "show") + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("running '%s' failed: %v", cmd.String(), err) + } + tuntapOut, _ := cmd.Output() + + var tuntapLines []string + if string(tuntapOut) == "" { + tuntapLines = []string{} + } else { + tuntapLines = strings.Split(string(tuntapOut), "\n") + } + + maxId := 0 + for _, line := range(tuntapLines) { + tapName, _, found := strings.Cut(line, ":") + if !found { + break + } + tapIdStr, found := strings.CutPrefix(tapName, tapDeviceBaseName) + if !found { + break + } + tapId, err := strconv.Atoi(tapIdStr) + if err != nil { + break + } + if tapId > maxId { + maxId = tapId + } + } + + return fmt.Sprintf("%s%d", tapDeviceBaseName, maxId+1), nil +} + +func destroyTapDevice(name string) { + exec.Command("ip", "tuntap", "del", name, "mode", "tap").Run() +} + +func createTapDevice(name string, bridge string) error { + // ip tuntap add mode tap + cmd := exec.Command("ip", "tuntap", "add", name, "mode", "tap") + if err := cmd.Run(); err != nil { + return fmt.Errorf("'%s' failed: %v", cmd.String(), err) + } + + // ip link set dev up + cmd = exec.Command("ip", "link", "set", "dev", name, "up") + if err := cmd.Run(); err != nil { + destroyTapDevice(name) + return fmt.Errorf("'%s' failed: %v", cmd.String(), err) + } + + // ip link set dev master + cmd = exec.Command("ip", "link", "set", "dev", name, "master", bridge) + if err := cmd.Run(); err != nil { + destroyTapDevice(name) + return fmt.Errorf("'%s' failed: %v", cmd.String(), err) + } + + return nil +} + +func getMacAddr(unikernel string, bridge string) string { + // deterministic mac address computation + // follow Albatross and use the VEB Kombinat Robotron prefix + // (thanks again, friends!) + ours := md5.Sum([]byte(bridge + unikernel)) + return fmt.Sprintf( + "%x:%x:%x:%x:%x:%x", + 0x00, 0x80, 0x41, + ours[0], ours[1], ours[2], + ) +} + +func nextIp(ip [2]int) ([2]int, error) { + nip := [2]int{ip[0], ip[1]} + // FIX: 255.255 is for broadcast + // .255 is for broadcast + if nip[1] < 255 { + nip[1]++ + return nip, nil + } + nip[1] = 0 + nip[0]++ + + if nip[0] < 256 { + return nip, nil + } + return [2]int{}, fmt.Errorf("nextIp") +} + +func getFreshIp() (string, error) { + // iptables -t nat -L MIRAGE_OUT + // then grep for: XX.XX.XX.XX to find existing ips + // (where the first two bytes correspond to the range of our bridge) + cmd := exec.Command("iptables", "-t", "nat", "-L", iptablesChainOut()) + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("'%s' failed: %v", cmd.String(), err) + } + chainStr, _ := cmd.Output() + chainLines := strings.Split(string(chainStr), "\n") + + r, _ := regexp.Compile( + strconv.Itoa(bridgeRangeHi) + "." + + strconv.Itoa(bridgeRangeLo) + `.(\d+).(\d+)`) + + usedIps := make(map[[2]int]bool) + for i := 2; i < len(chainLines); i++ { + line := chainLines[i] + matches := r.FindStringSubmatch(line) + b1, _ := strconv.Atoi(matches[1]) + b2, _ := strconv.Atoi(matches[2]) + usedIps[[2]int{b1, b2}] = true + } + + // XX.XX.0.1 is the gateway; start at XX.XX.0.2 + ip, err := [2]int{0, 2}, error(nil) + for err == nil { + if !usedIps[ip] { + break + } + ip, err = nextIp(ip) + } + + if err != nil { + return "", fmt.Errorf("no available ip found") + } + return fmt.Sprintf("%d.%d.%d.%d", bridgeRangeHi, bridgeRangeLo, ip[0], ip[1]), nil +} + +func getNetDevicesPorts( + netDevicesNames []string, + taskCfg TaskConfig, + cfg *drivers.TaskConfig, +) map[string][]NetDevicePortConfig { + ports := make(map[string][]NetDevicePortConfig) + + if len(netDevicesNames) == 0 { + return ports + } + + // TODO: extract port<->device assignation from taskCfg + // for now, assume that this config is empty, and implement the default + // strategy + + for _, name := range netDevicesNames { + ports[name] = []NetDevicePortConfig{} + } + + var defaultDevice string + if _, found := ports["service"]; found { + // use 'service' as the default device name when it exists + defaultDevice = "service" + } else { + // otherwise use the first one in the list + defaultDevice = netDevicesNames[0] + } + + for _, portMapping := range *cfg.Resources.Ports { + // only the default case for now + ports[defaultDevice] = append(ports[defaultDevice], + NetDevicePortConfig{ + From: portMapping.Value, + To: portMapping.To, + }) + } + + return ports +} + +// fields are "" if not specified +type NetDeviceArgs struct { + Ip string + Gateway string +} + +func getNetDevicesArgs( + netDevicesNames []string, + taskCfg TaskConfig, +) map[string]NetDeviceArgs { + args := make(map[string]NetDeviceArgs) + // TODO: extract the flag<->device assignation from taskCfg + // for now, always assign --ipv4 to service and fail if there is another device + for _, name := range netDevicesNames { + if name == "service" { + args["service"] = NetDeviceArgs{ + Ip: "--ipv4", + Gateway: "--ipv4-gateway", + } + } + } + return args +} + +func rollbackIptablesRules(rules [][]string) { + for j := len(rules) - 1; j >= 0; j-- { + args := append([]string{"-t", "nat", "-D"}, rules[j]...) + exec.Command("iptables", args...).Run() + } +} + +// run a list of iptables commands of the form "iptables -t nat -A ..." as a +// single transaction: if one rule fails, we roll back earlier rules before +// returning an error +func execIptablesRules(rules [][]string) error { + for i, rule := range rules { + args := append([]string{"-t", "nat", "-A"}, rule...) + cmd := exec.Command("iptables", args...) + if err := cmd.Run(); err != nil { + rollbackIptablesRules(rules[0:i]) + return fmt.Errorf("'%s' failed: %v", cmd.String(), err) + } + } + return nil +} + +func netDeviceIptablesRules(ip string, ports []NetDevicePortConfig) [][]string { + // iptables rules to talk to the outside and setup incoming port redirections + rr := [][]string{} + addrule := func(args ...string) { rr = append(rr, args) } + + // allow the ip to talk to the outside + // iptables -t nat -A MIRAGE_OUT -s /32 -j MASQUERADE + addrule(iptablesChainOut(), "-s", ip+"/32", "-j", "MASQUERADE") + + // NAT port redirections + // iptables -t nat -A MIRAGE_IN ! -s /32 -p tcp -m tcp --dport 53 + // -j DNAT --to-destination :53 + // iptables -t nat -A MIRAGE_IN ! -s /32 -p udp -m udp --dport 53 + // -j DNAT --to-destination :53 + for _, port := range ports { + addrule(iptablesChainIn(), "!", "-s", ip+"/32", "-p", "tcp", + "--dport", strconv.Itoa(port.From), /* TODO:check */ + "-j", "DNAT", "--to-destination", ip+":"+strconv.Itoa(port.To)) + addrule(iptablesChainIn(), "!", "-s", ip+"/32", "-p", "udp", + "--dport", strconv.Itoa(port.From), /* TODO:check from/to */ + "-j", "DNAT", "--to-destination", ip+":"+strconv.Itoa(port.To)) + } + return rr +} + +func setupNetDevice( + deviceName string, + unikernel string, + bridge string, + ports []NetDevicePortConfig, + logger hclog.Logger, +) (NetDeviceConfig, error) { + // list existing tap interfaces, and get a fresh tap interface number + tapDevice, err := getFreshTapDeviceName() + if err != nil { + return NetDeviceConfig{}, fmt.Errorf("could not get a fresh tap device name: %v", err) + } + logger.Debug("got tap device name", tapDevice) + + // create new tap device + if err := createTapDevice(tapDevice, bridge); err != nil { + return NetDeviceConfig{}, fmt.Errorf("could not create tap device: %v", err) + } + logger.Debug("successfully created tap device") + + // get a mac address for the unikernel + macAddr := getMacAddr(unikernel, bridge) + logger.Debug("mac address", macAddr) + + // find an unused ip address + ipAddr, err := getFreshIp() + if err != nil { + destroyTapDevice(tapDevice) + return NetDeviceConfig{}, fmt.Errorf("could not get a fresh ip address: %v", err) + } + logger.Debug("got ip address", ipAddr) + + // execute iptables rules + if err := execIptablesRules(netDeviceIptablesRules(ipAddr, ports)); err != nil { + destroyTapDevice(tapDevice) + return NetDeviceConfig{}, err + } + + return NetDeviceConfig{ + TapDevice: tapDevice, + Ip: ipAddr, + Mac: macAddr, + Ports: ports, + }, nil +} + +func destroyNetDevice(device NetDeviceConfig) { + rollbackIptablesRules(netDeviceIptablesRules(device.Ip, device.Ports)) + destroyTapDevice(device.TapDevice) +} + +func destroyNetDevices(devices map[string]NetDeviceConfig) { + for _, d := range devices { + destroyNetDevice(d) + } +} + +func setupNetDevices( + netDevicesPorts map[string][]NetDevicePortConfig, + unikernel string, + bridge string, + logger hclog.Logger, +) (map[string]NetDeviceConfig, error) { + netDevices := make(map[string]NetDeviceConfig) + for deviceName, devicePorts := range netDevicesPorts { + deviceCfg, err := + setupNetDevice(deviceName, unikernel, bridge, devicePorts, logger) + if err != nil { + destroyNetDevices(netDevices) + return nil, + fmt.Errorf("failed to setup net device %s: %v", + deviceName, err) + } + netDevices[deviceName] = deviceCfg + } + return netDevices, nil +} + +// TODO: also return the list of block device names, when we handle block +// devices +func readUnikernelManifest( + unikernel string, + logger hclog.Logger, +) ([]string /* net devices */, error) { + manifestCmd := exec.Command("solo5-elftool", "query-manifest", unikernel) + manifestStr, manifestErr := manifestCmd.Output() + if manifestErr != nil { + return nil, fmt.Errorf("failed to query unikernel manifest: %v", manifestErr) + } + + logger.Debug("unikernel manifest", manifestStr) + + var manifest unikernelManifest + if err := json.Unmarshal(manifestStr, &manifest); err != nil { + return nil, fmt.Errorf("could not parse manifest: %v", err) + } + + netDevicesNames := []string{} + for _, d := range manifest.Devices { + switch d.Type { + case "NET_BASIC": + netDevicesNames = append(netDevicesNames, d.Name) + // case "BLOCK_BASIC": + // TODO + default: + return nil, fmt.Errorf("unhandled device type: %s", d.Type) + } + } + return netDevicesNames, nil +} + // StartTask returns a task handle and a driver network if necessary. func (d *HelloDriverPlugin) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) { if _, ok := d.tasks.Get(cfg.ID); ok { @@ -375,28 +827,80 @@ func (d *HelloDriverPlugin) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHan LogLevel: "debug", } - exec, pluginClient, err := executor.CreateExecutor(d.logger, d.nomadConfig, executorConfig) + logger := d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID) + + netDevicesNames, err := readUnikernelManifest(driverConfig.Unikernel, logger) + if err != nil { + return nil, nil, err + } + netDevicesPorts := getNetDevicesPorts(netDevicesNames, driverConfig, cfg) + netDevicesArgs := getNetDevicesArgs(netDevicesNames, driverConfig) + netDevices, err := + setupNetDevices(netDevicesPorts, driverConfig.Unikernel, + d.config.Bridge, logger) + if err != nil { + return nil, nil, err + } + + solo5Args := []string{} + unikernelArgs := []string{} + // --net:= + // --net-mac:= + for name, device := range netDevices { + solo5Args = append(solo5Args, + fmt.Sprintf("--net:%s=%s", name, device.TapDevice)) + solo5Args = append(solo5Args, + fmt.Sprintf("--net-mac:%s=%s", name, device.Mac)) + } + // --ipv4=/16 --ipv4-gateway= (by default) + // + user-specified equivalent arguments for additional net devices + for name, arg := range netDevicesArgs { + if arg.Ip != "" { + unikernelArgs = append(unikernelArgs, + fmt.Sprintf("%s=%s/16", arg.Ip, netDevices[name].Ip)) + } + if arg.Gateway != "" { + unikernelArgs = append(unikernelArgs, + fmt.Sprintf("%s=%s", arg.Gateway, gatewayIp())) + } + } + + // TODO: collect additional arguments from the config file + + execu, pluginClient, err := executor.CreateExecutor(logger, d.nomadConfig, executorConfig) if err != nil { return nil, nil, fmt.Errorf("failed to create executor: %v", err) } - echoCmd := fmt.Sprintf(`echo "%s"`, driverConfig.Greeting) + cfgJson, _ := json.MarshalIndent(cfg, "", " ") + // cfgStr := base64.StdEncoding.EncodeToString(cfgJson) + logger.Debug("task config", string(cfgJson)) + + netCfgStr, _ := exec.Command("ip", "a").Output() + logger.Debug("host network config", string(netCfgStr)) + + args := append(solo5Args, []string{"--", driverConfig.Unikernel}...) + args = append(args, unikernelArgs...) + + logger.Debug("solo5 command", exec.Command("solo5-hvt", args...).String()) + execCmd := &executor.ExecCommand{ - Cmd: d.config.Shell, - Args: []string{"-c", echoCmd}, + Cmd: "solo5-hvt", + Args: args, StdoutPath: cfg.StdoutPath, StderrPath: cfg.StderrPath, } - ps, err := exec.Launch(execCmd) + ps, err := execu.Launch(execCmd) if err != nil { pluginClient.Kill() return nil, nil, fmt.Errorf("failed to launch command with executor: %v", err) } h := &taskHandle{ - exec: exec, + exec: execu, pid: ps.Pid, + netDevices: netDevices, pluginClient: pluginClient, taskConfig: cfg, procState: drivers.TaskStateRunning, @@ -407,11 +911,13 @@ func (d *HelloDriverPlugin) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHan driverState := TaskState{ ReattachConfig: structs.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), Pid: ps.Pid, + NetDevices: netDevices, TaskConfig: cfg, StartedAt: h.startedAt, } if err := handle.SetDriverState(&driverState); err != nil { + d.logger.Error("failed to start task, error setting driver state", "error", err) return nil, nil, fmt.Errorf("failed to set driver state: %v", err) } @@ -460,6 +966,7 @@ func (d *HelloDriverPlugin) RecoverTask(handle *drivers.TaskHandle) error { h := &taskHandle{ exec: execImpl, pid: taskState.Pid, + netDevices: taskState.NetDevices, pluginClient: pluginClient, taskConfig: taskState.TaskConfig, procState: drivers.TaskStateRunning, @@ -575,6 +1082,12 @@ func (d *HelloDriverPlugin) DestroyTask(taskID string, force bool) error { handle.pluginClient.Kill() } + // remove the relevant iptables rules and the tap device + destroyNetDevices(handle.netDevices) + + // TODO: destroy the bridge and the iptables chains if no one is using them + // anymore + d.tasks.Delete(taskID) return nil } diff --git a/hello/handle.go b/hello/handle.go index 08b798d..c5d71df 100644 --- a/hello/handle.go +++ b/hello/handle.go @@ -32,7 +32,8 @@ type taskHandle struct { exitResult *drivers.ExitResult // TODO: add any extra relevant information about the task. - pid int + pid int + netDevices map[string]NetDeviceConfig } func (h *taskHandle) TaskStatus() *drivers.TaskStatus { @@ -48,6 +49,7 @@ func (h *taskHandle) TaskStatus() *drivers.TaskStatus { ExitResult: h.exitResult, DriverAttributes: map[string]string{ "pid": strconv.Itoa(h.pid), + // TODO: emit more attributes? }, } }