initial networking setup
This commit is contained in:
Armaël Guéneau
parent
3008eec5b5
commit
14476852d4
|
|
@ -5,6 +5,16 @@ log_level = "TRACE"
|
|||
|
||||
plugin "hello-driver" {
|
||||
config {
|
||||
shell = "bash"
|
||||
bridge = "mybridge"
|
||||
}
|
||||
}
|
||||
|
||||
plugin "raw_exec" {
|
||||
config {
|
||||
enabled = true
|
||||
}
|
||||
}
|
||||
|
||||
consul {
|
||||
address = "127.0.0.1:8500"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,15 +3,33 @@
|
|||
|
||||
job "example" {
|
||||
datacenters = ["dc1"]
|
||||
type = "batch"
|
||||
type = "service"
|
||||
|
||||
group "example" {
|
||||
network {
|
||||
# mode = "driver"
|
||||
port "http" {
|
||||
to = 8080
|
||||
}
|
||||
}
|
||||
|
||||
task "hello-world" {
|
||||
driver = "hello-world-example"
|
||||
|
||||
config {
|
||||
greeting = "hello"
|
||||
unikernel = "/tmp/hello-key.hvt"
|
||||
# network "device1" {
|
||||
# ports = ["http", ...]
|
||||
# ip_arg = "--monitoring"
|
||||
# gateway_arg = "--xxx"
|
||||
# }
|
||||
# args = [...]
|
||||
}
|
||||
}
|
||||
|
||||
service {
|
||||
port = "http"
|
||||
name = "hello-service"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
585
hello/driver.go
585
hello/driver.go
|
|
@ -5,12 +5,17 @@ package hello
|
|||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
// "encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/consul-template/signals"
|
||||
|
|
@ -42,6 +47,15 @@ const (
|
|||
// this is used to allow modification and migration of the task schema
|
||||
// used by the plugin
|
||||
taskHandleVersion = 1
|
||||
|
||||
// TODO: should these be made configurable?
|
||||
tapDeviceBaseName = "miragetap"
|
||||
iptablesChainBase = "MIRAGE"
|
||||
|
||||
// TODO: make this configurable
|
||||
// addresses in the bridge are <bridgeRangeHi>.<bridgeRangeLo>.0.0/16
|
||||
bridgeRangeHi = 10
|
||||
bridgeRangeLo = 99
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -71,9 +85,9 @@ var (
|
|||
// shell = "fish"
|
||||
// }
|
||||
// }
|
||||
"shell": hclspec.NewDefault(
|
||||
hclspec.NewAttr("shell", "string", false),
|
||||
hclspec.NewLiteral(`"bash"`),
|
||||
"bridge": hclspec.NewDefault(
|
||||
hclspec.NewAttr("bridge", "string", false),
|
||||
hclspec.NewLiteral(`"mirage"`),
|
||||
),
|
||||
})
|
||||
|
||||
|
|
@ -99,9 +113,9 @@ var (
|
|||
// }
|
||||
// }
|
||||
// }
|
||||
"greeting": hclspec.NewDefault(
|
||||
hclspec.NewAttr("greeting", "string", false),
|
||||
hclspec.NewLiteral(`"Hello, World!"`),
|
||||
"unikernel": hclspec.NewDefault(
|
||||
hclspec.NewAttr("unikernel", "string", true),
|
||||
hclspec.NewLiteral(`""`),
|
||||
),
|
||||
})
|
||||
|
||||
|
|
@ -115,6 +129,11 @@ var (
|
|||
// https://godoc.org/github.com/hashicorp/nomad/plugins/drivers#Capabilities
|
||||
SendSignals: true,
|
||||
Exec: false,
|
||||
FSIsolation: drivers.FSIsolationNone,
|
||||
NetIsolationModes: []drivers.NetIsolationMode{
|
||||
drivers.NetIsolationModeTask,
|
||||
},
|
||||
MountConfigs: drivers.MountConfigSupportNone,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -125,7 +144,7 @@ type Config struct {
|
|||
// This struct is the decoded version of the schema defined in the
|
||||
// configSpec variable above. It's used to convert the HCL configuration
|
||||
// passed by the Nomad agent into Go contructs.
|
||||
Shell string `codec:"shell"`
|
||||
Bridge string `codec:"bridge"`
|
||||
}
|
||||
|
||||
// TaskConfig contains configuration information for a task that runs with
|
||||
|
|
@ -136,7 +155,7 @@ type TaskConfig struct {
|
|||
// This struct is the decoded version of the schema defined in the
|
||||
// taskConfigSpec variable above. It's used to convert the string
|
||||
// configuration for the task into Go contructs.
|
||||
Greeting string `codec:"greeting"`
|
||||
Unikernel string `codec:"unikernel"`
|
||||
}
|
||||
|
||||
// TaskState is the runtime state which is encoded in the handle returned to
|
||||
|
|
@ -156,7 +175,20 @@ type TaskState struct {
|
|||
// will respawn a new instance of the plugin and try to restore its
|
||||
// in-memory representation of the running tasks using the RecoverTask()
|
||||
// method below.
|
||||
Pid int
|
||||
Pid int
|
||||
NetDevices map[string]NetDeviceConfig
|
||||
}
|
||||
|
||||
type NetDeviceConfig struct {
|
||||
TapDevice string
|
||||
Ip string
|
||||
Mac string
|
||||
Ports []NetDevicePortConfig
|
||||
}
|
||||
|
||||
type NetDevicePortConfig struct {
|
||||
From int // port on the host
|
||||
To int // port inside the bridge
|
||||
}
|
||||
|
||||
// HelloDriverPlugin is an example driver plugin. When provisioned in a job,
|
||||
|
|
@ -187,6 +219,18 @@ type HelloDriverPlugin struct {
|
|||
logger hclog.Logger
|
||||
}
|
||||
|
||||
// A unikernel manifest. Obtained by deserializing JSON returned by solo5-elftool.
|
||||
type unikernelManifest struct {
|
||||
Type string `json:"type"`
|
||||
Version int `json:"version"`
|
||||
Devices []unikernelDevice `json:"devices"`
|
||||
}
|
||||
|
||||
type unikernelDevice struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"` // BLOCK_BASIC, NET_BASIC
|
||||
}
|
||||
|
||||
// NewPlugin returns a new example driver plugin
|
||||
func NewPlugin(logger hclog.Logger) drivers.DriverPlugin {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
|
@ -212,6 +256,22 @@ func (d *HelloDriverPlugin) ConfigSchema() (*hclspec.Spec, error) {
|
|||
return configSpec, nil
|
||||
}
|
||||
|
||||
func gatewayIp() string {
|
||||
return fmt.Sprintf("%d.%d.0.1", bridgeRangeHi, bridgeRangeLo)
|
||||
}
|
||||
|
||||
func broadcastIp() string {
|
||||
return fmt.Sprintf("%d.%d.0.255", bridgeRangeHi, bridgeRangeLo)
|
||||
}
|
||||
|
||||
func iptablesChainOut() string {
|
||||
return iptablesChainBase + "_OUT"
|
||||
}
|
||||
|
||||
func iptablesChainIn() string {
|
||||
return iptablesChainBase + "_IN"
|
||||
}
|
||||
|
||||
// SetConfig is called by the client to pass the configuration for the plugin.
|
||||
func (d *HelloDriverPlugin) SetConfig(cfg *base.Config) error {
|
||||
var config Config
|
||||
|
|
@ -233,10 +293,10 @@ func (d *HelloDriverPlugin) SetConfig(cfg *base.Config) error {
|
|||
//
|
||||
// In the example below we check if the shell specified by the user is
|
||||
// supported by the plugin.
|
||||
shell := d.config.Shell
|
||||
if shell != "bash" && shell != "fish" {
|
||||
return fmt.Errorf("invalid shell %s", d.config.Shell)
|
||||
}
|
||||
// shell := d.config.Shell
|
||||
// if shell != "bash" && shell != "fish" {
|
||||
// return fmt.Errorf("invalid shell %s", d.config.Shell)
|
||||
// }
|
||||
|
||||
// Save the Nomad agent configuration
|
||||
if cfg.AgentConfig != nil {
|
||||
|
|
@ -248,6 +308,40 @@ func (d *HelloDriverPlugin) SetConfig(cfg *base.Config) error {
|
|||
// Here you can use the config values to initialize any resources that are
|
||||
// shared by all tasks that use this driver, such as a daemon process.
|
||||
|
||||
// create the bridge and iptables chains if they do not exist
|
||||
// TODO: handle errors
|
||||
|
||||
// echo "1" > /proc/sys/net/ipv4/ip_forward
|
||||
// ip link add name mybridge type bridge
|
||||
// ip link set dev mybridge up
|
||||
// ip address add 10.0.0.1/24 broadcast 10.0.0.255 dev mybridge
|
||||
exec.Command("sh", "-c", `echo 1 > /proc/sys/net/ipv4/ip_forward`).Run()
|
||||
exec.Command("ip", "link", "add", "name", d.config.Bridge, "type", "bridge").Run()
|
||||
exec.Command("ip", "link", "set", "dev", d.config.Bridge, "up").Run()
|
||||
// TODO: remove broadcast?
|
||||
exec.Command("ip", "address", "add", gatewayIp()+"/16", "broadcast",
|
||||
broadcastIp(), "dev", d.config.Bridge).Run()
|
||||
|
||||
// iptables -t nat -N MIRAGE_OUT
|
||||
// iptables -t nat -A POSTROUTING -j MIRAGE_OUT
|
||||
// + a rule for each unikernel...
|
||||
exec.Command("iptables", "-t", "nat", "-N", iptablesChainOut()).Run()
|
||||
// FIXME: this one gets added unconditionally even if it already exists
|
||||
exec.Command("iptables", "-t", "nat", "-A", "POSTROUTING", "-j", iptablesChainOut()).Run()
|
||||
|
||||
// iptables -t nat -N MIRAGE_IN
|
||||
// iptables -t nat -A PREROUTING -m addrtype --dst-type LOCAL -j MIRAGE_IN
|
||||
// + a rule for each unikernel...
|
||||
exec.Command("iptables", "-t", "nat", "-N", iptablesChainIn()).Run()
|
||||
// FIXME: this one gets added unconditionally even if it already exists
|
||||
exec.Command("iptables", "-t", "nat", "-A", "PREROUTING",
|
||||
"-m", "addrtype", "--dst-type", "LOCAL",
|
||||
"-j", iptablesChainIn()).Run()
|
||||
|
||||
// TODO:
|
||||
// iptables -t nat -I OUTPUT -o lo -j MIRAGE_IN
|
||||
// sysctl -w net.ipv4.conf.all.route_localnet=1
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -311,34 +405,392 @@ func (d *HelloDriverPlugin) buildFingerprint() *drivers.Fingerprint {
|
|||
// the node in which the plugin is running (specific library availability,
|
||||
// installed versions of a software etc.). These attributes can then be
|
||||
// used by an operator to set job constrains.
|
||||
//
|
||||
// In the example below we check if the shell specified by the user exists
|
||||
// in the node.
|
||||
shell := d.config.Shell
|
||||
|
||||
cmd := exec.Command("which", shell)
|
||||
if err := cmd.Run(); err != nil {
|
||||
// FIXME: is this the right place to check this?
|
||||
// do we need to check for the binaries existence every N seconds?
|
||||
// should we do these checks in SetConfig() instead?
|
||||
|
||||
// check that solo5-hvt exists
|
||||
hvtCmd := exec.Command("which", "solo5-hvt")
|
||||
hvtPath, hvtErr := hvtCmd.Output()
|
||||
if hvtErr != nil {
|
||||
return &drivers.Fingerprint{
|
||||
Health: drivers.HealthStateUndetected,
|
||||
HealthDescription: fmt.Sprintf("shell %s not found", shell),
|
||||
HealthDescription: "solo5-hvt not found",
|
||||
}
|
||||
}
|
||||
|
||||
// We also set the shell and its version as attributes
|
||||
cmd = exec.Command(shell, "--version")
|
||||
if out, err := cmd.Output(); err != nil {
|
||||
d.logger.Warn("failed to find shell version: %v", err)
|
||||
} else {
|
||||
re := regexp.MustCompile("[0-9]\\.[0-9]\\.[0-9]")
|
||||
version := re.FindString(string(out))
|
||||
|
||||
fp.Attributes["driver.hello.shell_version"] = structs.NewStringAttribute(version)
|
||||
fp.Attributes["driver.hello.shell"] = structs.NewStringAttribute(shell)
|
||||
// check that solo5-elftool exists
|
||||
elftoolCmd := exec.Command("which", "solo5-elftool")
|
||||
elftoolPath, elftoolErr := elftoolCmd.Output()
|
||||
if elftoolErr != nil {
|
||||
return &drivers.Fingerprint{
|
||||
Health: drivers.HealthStateUndetected,
|
||||
HealthDescription: fmt.Sprintf("solo5-elftool not found"),
|
||||
}
|
||||
}
|
||||
|
||||
// We set their paths as attributes
|
||||
fp.Attributes["driver.hello.solo5-hvt"] = structs.NewStringAttribute(string(hvtPath))
|
||||
fp.Attributes["driver.hello.solo5-elftool"] = structs.NewStringAttribute(string(elftoolPath))
|
||||
|
||||
return fp
|
||||
}
|
||||
|
||||
func getFreshTapDeviceName() (string, error) {
|
||||
cmd := exec.Command("ip", "tuntap", "show")
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("running '%s' failed: %v", cmd.String(), err)
|
||||
}
|
||||
tuntapOut, _ := cmd.Output()
|
||||
|
||||
var tuntapLines []string
|
||||
if string(tuntapOut) == "" {
|
||||
tuntapLines = []string{}
|
||||
} else {
|
||||
tuntapLines = strings.Split(string(tuntapOut), "\n")
|
||||
}
|
||||
|
||||
maxId := 0
|
||||
for _, line := range(tuntapLines) {
|
||||
tapName, _, found := strings.Cut(line, ":")
|
||||
if !found {
|
||||
break
|
||||
}
|
||||
tapIdStr, found := strings.CutPrefix(tapName, tapDeviceBaseName)
|
||||
if !found {
|
||||
break
|
||||
}
|
||||
tapId, err := strconv.Atoi(tapIdStr)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if tapId > maxId {
|
||||
maxId = tapId
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s%d", tapDeviceBaseName, maxId+1), nil
|
||||
}
|
||||
|
||||
func destroyTapDevice(name string) {
|
||||
exec.Command("ip", "tuntap", "del", name, "mode", "tap").Run()
|
||||
}
|
||||
|
||||
func createTapDevice(name string, bridge string) error {
|
||||
// ip tuntap add <tap> mode tap
|
||||
cmd := exec.Command("ip", "tuntap", "add", name, "mode", "tap")
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("'%s' failed: %v", cmd.String(), err)
|
||||
}
|
||||
|
||||
// ip link set dev <tap> up
|
||||
cmd = exec.Command("ip", "link", "set", "dev", name, "up")
|
||||
if err := cmd.Run(); err != nil {
|
||||
destroyTapDevice(name)
|
||||
return fmt.Errorf("'%s' failed: %v", cmd.String(), err)
|
||||
}
|
||||
|
||||
// ip link set dev <tap> master <bridge>
|
||||
cmd = exec.Command("ip", "link", "set", "dev", name, "master", bridge)
|
||||
if err := cmd.Run(); err != nil {
|
||||
destroyTapDevice(name)
|
||||
return fmt.Errorf("'%s' failed: %v", cmd.String(), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getMacAddr(unikernel string, bridge string) string {
|
||||
// deterministic mac address computation
|
||||
// follow Albatross and use the VEB Kombinat Robotron prefix
|
||||
// (thanks again, friends!)
|
||||
ours := md5.Sum([]byte(bridge + unikernel))
|
||||
return fmt.Sprintf(
|
||||
"%x:%x:%x:%x:%x:%x",
|
||||
0x00, 0x80, 0x41,
|
||||
ours[0], ours[1], ours[2],
|
||||
)
|
||||
}
|
||||
|
||||
func nextIp(ip [2]int) ([2]int, error) {
|
||||
nip := [2]int{ip[0], ip[1]}
|
||||
// FIX: 255.255 is for broadcast
|
||||
// .255 is for broadcast
|
||||
if nip[1] < 255 {
|
||||
nip[1]++
|
||||
return nip, nil
|
||||
}
|
||||
nip[1] = 0
|
||||
nip[0]++
|
||||
|
||||
if nip[0] < 256 {
|
||||
return nip, nil
|
||||
}
|
||||
return [2]int{}, fmt.Errorf("nextIp")
|
||||
}
|
||||
|
||||
func getFreshIp() (string, error) {
|
||||
// iptables -t nat -L MIRAGE_OUT
|
||||
// then grep for: XX.XX.XX.XX to find existing ips
|
||||
// (where the first two bytes correspond to the range of our bridge)
|
||||
cmd := exec.Command("iptables", "-t", "nat", "-L", iptablesChainOut())
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("'%s' failed: %v", cmd.String(), err)
|
||||
}
|
||||
chainStr, _ := cmd.Output()
|
||||
chainLines := strings.Split(string(chainStr), "\n")
|
||||
|
||||
r, _ := regexp.Compile(
|
||||
strconv.Itoa(bridgeRangeHi) + "." +
|
||||
strconv.Itoa(bridgeRangeLo) + `.(\d+).(\d+)`)
|
||||
|
||||
usedIps := make(map[[2]int]bool)
|
||||
for i := 2; i < len(chainLines); i++ {
|
||||
line := chainLines[i]
|
||||
matches := r.FindStringSubmatch(line)
|
||||
b1, _ := strconv.Atoi(matches[1])
|
||||
b2, _ := strconv.Atoi(matches[2])
|
||||
usedIps[[2]int{b1, b2}] = true
|
||||
}
|
||||
|
||||
// XX.XX.0.1 is the gateway; start at XX.XX.0.2
|
||||
ip, err := [2]int{0, 2}, error(nil)
|
||||
for err == nil {
|
||||
if !usedIps[ip] {
|
||||
break
|
||||
}
|
||||
ip, err = nextIp(ip)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("no available ip found")
|
||||
}
|
||||
return fmt.Sprintf("%d.%d.%d.%d", bridgeRangeHi, bridgeRangeLo, ip[0], ip[1]), nil
|
||||
}
|
||||
|
||||
func getNetDevicesPorts(
|
||||
netDevicesNames []string,
|
||||
taskCfg TaskConfig,
|
||||
cfg *drivers.TaskConfig,
|
||||
) map[string][]NetDevicePortConfig {
|
||||
ports := make(map[string][]NetDevicePortConfig)
|
||||
|
||||
if len(netDevicesNames) == 0 {
|
||||
return ports
|
||||
}
|
||||
|
||||
// TODO: extract port<->device assignation from taskCfg
|
||||
// for now, assume that this config is empty, and implement the default
|
||||
// strategy
|
||||
|
||||
for _, name := range netDevicesNames {
|
||||
ports[name] = []NetDevicePortConfig{}
|
||||
}
|
||||
|
||||
var defaultDevice string
|
||||
if _, found := ports["service"]; found {
|
||||
// use 'service' as the default device name when it exists
|
||||
defaultDevice = "service"
|
||||
} else {
|
||||
// otherwise use the first one in the list
|
||||
defaultDevice = netDevicesNames[0]
|
||||
}
|
||||
|
||||
for _, portMapping := range *cfg.Resources.Ports {
|
||||
// only the default case for now
|
||||
ports[defaultDevice] = append(ports[defaultDevice],
|
||||
NetDevicePortConfig{
|
||||
From: portMapping.Value,
|
||||
To: portMapping.To,
|
||||
})
|
||||
}
|
||||
|
||||
return ports
|
||||
}
|
||||
|
||||
// fields are "" if not specified
|
||||
type NetDeviceArgs struct {
|
||||
Ip string
|
||||
Gateway string
|
||||
}
|
||||
|
||||
func getNetDevicesArgs(
|
||||
netDevicesNames []string,
|
||||
taskCfg TaskConfig,
|
||||
) map[string]NetDeviceArgs {
|
||||
args := make(map[string]NetDeviceArgs)
|
||||
// TODO: extract the flag<->device assignation from taskCfg
|
||||
// for now, always assign --ipv4 to service and fail if there is another device
|
||||
for _, name := range netDevicesNames {
|
||||
if name == "service" {
|
||||
args["service"] = NetDeviceArgs{
|
||||
Ip: "--ipv4",
|
||||
Gateway: "--ipv4-gateway",
|
||||
}
|
||||
}
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func rollbackIptablesRules(rules [][]string) {
|
||||
for j := len(rules) - 1; j >= 0; j-- {
|
||||
args := append([]string{"-t", "nat", "-D"}, rules[j]...)
|
||||
exec.Command("iptables", args...).Run()
|
||||
}
|
||||
}
|
||||
|
||||
// run a list of iptables commands of the form "iptables -t nat -A ..." as a
|
||||
// single transaction: if one rule fails, we roll back earlier rules before
|
||||
// returning an error
|
||||
func execIptablesRules(rules [][]string) error {
|
||||
for i, rule := range rules {
|
||||
args := append([]string{"-t", "nat", "-A"}, rule...)
|
||||
cmd := exec.Command("iptables", args...)
|
||||
if err := cmd.Run(); err != nil {
|
||||
rollbackIptablesRules(rules[0:i])
|
||||
return fmt.Errorf("'%s' failed: %v", cmd.String(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func netDeviceIptablesRules(ip string, ports []NetDevicePortConfig) [][]string {
|
||||
// iptables rules to talk to the outside and setup incoming port redirections
|
||||
rr := [][]string{}
|
||||
addrule := func(args ...string) { rr = append(rr, args) }
|
||||
|
||||
// allow the ip to talk to the outside
|
||||
// iptables -t nat -A MIRAGE_OUT -s <ip>/32 -j MASQUERADE
|
||||
addrule(iptablesChainOut(), "-s", ip+"/32", "-j", "MASQUERADE")
|
||||
|
||||
// NAT port redirections
|
||||
// iptables -t nat -A MIRAGE_IN ! -s <ip>/32 -p tcp -m tcp --dport 53
|
||||
// -j DNAT --to-destination <ip>:53
|
||||
// iptables -t nat -A MIRAGE_IN ! -s <ip>/32 -p udp -m udp --dport 53
|
||||
// -j DNAT --to-destination <ip>:53
|
||||
for _, port := range ports {
|
||||
addrule(iptablesChainIn(), "!", "-s", ip+"/32", "-p", "tcp",
|
||||
"--dport", strconv.Itoa(port.From), /* TODO:check */
|
||||
"-j", "DNAT", "--to-destination", ip+":"+strconv.Itoa(port.To))
|
||||
addrule(iptablesChainIn(), "!", "-s", ip+"/32", "-p", "udp",
|
||||
"--dport", strconv.Itoa(port.From), /* TODO:check from/to */
|
||||
"-j", "DNAT", "--to-destination", ip+":"+strconv.Itoa(port.To))
|
||||
}
|
||||
return rr
|
||||
}
|
||||
|
||||
func setupNetDevice(
|
||||
deviceName string,
|
||||
unikernel string,
|
||||
bridge string,
|
||||
ports []NetDevicePortConfig,
|
||||
logger hclog.Logger,
|
||||
) (NetDeviceConfig, error) {
|
||||
// list existing tap interfaces, and get a fresh tap interface number
|
||||
tapDevice, err := getFreshTapDeviceName()
|
||||
if err != nil {
|
||||
return NetDeviceConfig{}, fmt.Errorf("could not get a fresh tap device name: %v", err)
|
||||
}
|
||||
logger.Debug("got tap device name", tapDevice)
|
||||
|
||||
// create new tap device
|
||||
if err := createTapDevice(tapDevice, bridge); err != nil {
|
||||
return NetDeviceConfig{}, fmt.Errorf("could not create tap device: %v", err)
|
||||
}
|
||||
logger.Debug("successfully created tap device")
|
||||
|
||||
// get a mac address for the unikernel
|
||||
macAddr := getMacAddr(unikernel, bridge)
|
||||
logger.Debug("mac address", macAddr)
|
||||
|
||||
// find an unused ip address
|
||||
ipAddr, err := getFreshIp()
|
||||
if err != nil {
|
||||
destroyTapDevice(tapDevice)
|
||||
return NetDeviceConfig{}, fmt.Errorf("could not get a fresh ip address: %v", err)
|
||||
}
|
||||
logger.Debug("got ip address", ipAddr)
|
||||
|
||||
// execute iptables rules
|
||||
if err := execIptablesRules(netDeviceIptablesRules(ipAddr, ports)); err != nil {
|
||||
destroyTapDevice(tapDevice)
|
||||
return NetDeviceConfig{}, err
|
||||
}
|
||||
|
||||
return NetDeviceConfig{
|
||||
TapDevice: tapDevice,
|
||||
Ip: ipAddr,
|
||||
Mac: macAddr,
|
||||
Ports: ports,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func destroyNetDevice(device NetDeviceConfig) {
|
||||
rollbackIptablesRules(netDeviceIptablesRules(device.Ip, device.Ports))
|
||||
destroyTapDevice(device.TapDevice)
|
||||
}
|
||||
|
||||
func destroyNetDevices(devices map[string]NetDeviceConfig) {
|
||||
for _, d := range devices {
|
||||
destroyNetDevice(d)
|
||||
}
|
||||
}
|
||||
|
||||
func setupNetDevices(
|
||||
netDevicesPorts map[string][]NetDevicePortConfig,
|
||||
unikernel string,
|
||||
bridge string,
|
||||
logger hclog.Logger,
|
||||
) (map[string]NetDeviceConfig, error) {
|
||||
netDevices := make(map[string]NetDeviceConfig)
|
||||
for deviceName, devicePorts := range netDevicesPorts {
|
||||
deviceCfg, err :=
|
||||
setupNetDevice(deviceName, unikernel, bridge, devicePorts, logger)
|
||||
if err != nil {
|
||||
destroyNetDevices(netDevices)
|
||||
return nil,
|
||||
fmt.Errorf("failed to setup net device %s: %v",
|
||||
deviceName, err)
|
||||
}
|
||||
netDevices[deviceName] = deviceCfg
|
||||
}
|
||||
return netDevices, nil
|
||||
}
|
||||
|
||||
// TODO: also return the list of block device names, when we handle block
|
||||
// devices
|
||||
func readUnikernelManifest(
|
||||
unikernel string,
|
||||
logger hclog.Logger,
|
||||
) ([]string /* net devices */, error) {
|
||||
manifestCmd := exec.Command("solo5-elftool", "query-manifest", unikernel)
|
||||
manifestStr, manifestErr := manifestCmd.Output()
|
||||
if manifestErr != nil {
|
||||
return nil, fmt.Errorf("failed to query unikernel manifest: %v", manifestErr)
|
||||
}
|
||||
|
||||
logger.Debug("unikernel manifest", manifestStr)
|
||||
|
||||
var manifest unikernelManifest
|
||||
if err := json.Unmarshal(manifestStr, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("could not parse manifest: %v", err)
|
||||
}
|
||||
|
||||
netDevicesNames := []string{}
|
||||
for _, d := range manifest.Devices {
|
||||
switch d.Type {
|
||||
case "NET_BASIC":
|
||||
netDevicesNames = append(netDevicesNames, d.Name)
|
||||
// case "BLOCK_BASIC":
|
||||
// TODO
|
||||
default:
|
||||
return nil, fmt.Errorf("unhandled device type: %s", d.Type)
|
||||
}
|
||||
}
|
||||
return netDevicesNames, nil
|
||||
}
|
||||
|
||||
// StartTask returns a task handle and a driver network if necessary.
|
||||
func (d *HelloDriverPlugin) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) {
|
||||
if _, ok := d.tasks.Get(cfg.ID); ok {
|
||||
|
|
@ -375,28 +827,80 @@ func (d *HelloDriverPlugin) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHan
|
|||
LogLevel: "debug",
|
||||
}
|
||||
|
||||
exec, pluginClient, err := executor.CreateExecutor(d.logger, d.nomadConfig, executorConfig)
|
||||
logger := d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID)
|
||||
|
||||
netDevicesNames, err := readUnikernelManifest(driverConfig.Unikernel, logger)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
netDevicesPorts := getNetDevicesPorts(netDevicesNames, driverConfig, cfg)
|
||||
netDevicesArgs := getNetDevicesArgs(netDevicesNames, driverConfig)
|
||||
netDevices, err :=
|
||||
setupNetDevices(netDevicesPorts, driverConfig.Unikernel,
|
||||
d.config.Bridge, logger)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
solo5Args := []string{}
|
||||
unikernelArgs := []string{}
|
||||
// --net:<service from manifest>=<tapDevice>
|
||||
// --net-mac:<service from manifest>=<macAddr>
|
||||
for name, device := range netDevices {
|
||||
solo5Args = append(solo5Args,
|
||||
fmt.Sprintf("--net:%s=%s", name, device.TapDevice))
|
||||
solo5Args = append(solo5Args,
|
||||
fmt.Sprintf("--net-mac:%s=%s", name, device.Mac))
|
||||
}
|
||||
// --ipv4=<ipAddr>/16 --ipv4-gateway=<gatewayIp()> (by default)
|
||||
// + user-specified equivalent arguments for additional net devices
|
||||
for name, arg := range netDevicesArgs {
|
||||
if arg.Ip != "" {
|
||||
unikernelArgs = append(unikernelArgs,
|
||||
fmt.Sprintf("%s=%s/16", arg.Ip, netDevices[name].Ip))
|
||||
}
|
||||
if arg.Gateway != "" {
|
||||
unikernelArgs = append(unikernelArgs,
|
||||
fmt.Sprintf("%s=%s", arg.Gateway, gatewayIp()))
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: collect additional arguments from the config file
|
||||
|
||||
execu, pluginClient, err := executor.CreateExecutor(logger, d.nomadConfig, executorConfig)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create executor: %v", err)
|
||||
}
|
||||
|
||||
echoCmd := fmt.Sprintf(`echo "%s"`, driverConfig.Greeting)
|
||||
cfgJson, _ := json.MarshalIndent(cfg, "", " ")
|
||||
// cfgStr := base64.StdEncoding.EncodeToString(cfgJson)
|
||||
logger.Debug("task config", string(cfgJson))
|
||||
|
||||
netCfgStr, _ := exec.Command("ip", "a").Output()
|
||||
logger.Debug("host network config", string(netCfgStr))
|
||||
|
||||
args := append(solo5Args, []string{"--", driverConfig.Unikernel}...)
|
||||
args = append(args, unikernelArgs...)
|
||||
|
||||
logger.Debug("solo5 command", exec.Command("solo5-hvt", args...).String())
|
||||
|
||||
execCmd := &executor.ExecCommand{
|
||||
Cmd: d.config.Shell,
|
||||
Args: []string{"-c", echoCmd},
|
||||
Cmd: "solo5-hvt",
|
||||
Args: args,
|
||||
StdoutPath: cfg.StdoutPath,
|
||||
StderrPath: cfg.StderrPath,
|
||||
}
|
||||
|
||||
ps, err := exec.Launch(execCmd)
|
||||
ps, err := execu.Launch(execCmd)
|
||||
if err != nil {
|
||||
pluginClient.Kill()
|
||||
return nil, nil, fmt.Errorf("failed to launch command with executor: %v", err)
|
||||
}
|
||||
|
||||
h := &taskHandle{
|
||||
exec: exec,
|
||||
exec: execu,
|
||||
pid: ps.Pid,
|
||||
netDevices: netDevices,
|
||||
pluginClient: pluginClient,
|
||||
taskConfig: cfg,
|
||||
procState: drivers.TaskStateRunning,
|
||||
|
|
@ -407,11 +911,13 @@ func (d *HelloDriverPlugin) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHan
|
|||
driverState := TaskState{
|
||||
ReattachConfig: structs.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()),
|
||||
Pid: ps.Pid,
|
||||
NetDevices: netDevices,
|
||||
TaskConfig: cfg,
|
||||
StartedAt: h.startedAt,
|
||||
}
|
||||
|
||||
if err := handle.SetDriverState(&driverState); err != nil {
|
||||
d.logger.Error("failed to start task, error setting driver state", "error", err)
|
||||
return nil, nil, fmt.Errorf("failed to set driver state: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -460,6 +966,7 @@ func (d *HelloDriverPlugin) RecoverTask(handle *drivers.TaskHandle) error {
|
|||
h := &taskHandle{
|
||||
exec: execImpl,
|
||||
pid: taskState.Pid,
|
||||
netDevices: taskState.NetDevices,
|
||||
pluginClient: pluginClient,
|
||||
taskConfig: taskState.TaskConfig,
|
||||
procState: drivers.TaskStateRunning,
|
||||
|
|
@ -575,6 +1082,12 @@ func (d *HelloDriverPlugin) DestroyTask(taskID string, force bool) error {
|
|||
handle.pluginClient.Kill()
|
||||
}
|
||||
|
||||
// remove the relevant iptables rules and the tap device
|
||||
destroyNetDevices(handle.netDevices)
|
||||
|
||||
// TODO: destroy the bridge and the iptables chains if no one is using them
|
||||
// anymore
|
||||
|
||||
d.tasks.Delete(taskID)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,8 @@ type taskHandle struct {
|
|||
exitResult *drivers.ExitResult
|
||||
|
||||
// TODO: add any extra relevant information about the task.
|
||||
pid int
|
||||
pid int
|
||||
netDevices map[string]NetDeviceConfig
|
||||
}
|
||||
|
||||
func (h *taskHandle) TaskStatus() *drivers.TaskStatus {
|
||||
|
|
@ -48,6 +49,7 @@ func (h *taskHandle) TaskStatus() *drivers.TaskStatus {
|
|||
ExitResult: h.exitResult,
|
||||
DriverAttributes: map[string]string{
|
||||
"pid": strconv.Itoa(h.pid),
|
||||
// TODO: emit more attributes?
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue