Not work ; must fix executor for symlink

This commit is contained in:
Alex 2022-11-28 17:06:00 +01:00
parent cf4285e812
commit 63e31b9ed9
Signed by: lx
GPG key ID: 0E496D15096376BE
5 changed files with 48 additions and 80 deletions

View file

@ -1,7 +1,7 @@
log_level = "TRACE" #log_level = "TRACE"
plugin "hello-driver" { client {
config { }
shell = "bash"
} plugin "exec2" {
} }

19
example/example.hcl Normal file
View file

@ -0,0 +1,19 @@
job "example" {
datacenters = ["dc1"]
type = "batch"
group "example" {
task "hello-world" {
driver = "exec2"
config {
command = "/nix/store/y41s1vcn0irn9ahn9wh62yx2cygs7qjj-coreutils-8.32/bin/cat"
args = ["/host-etc/nscd.conf"]
bind_read_only = {
"/nix" = "/nix",
"/etc" = "/host-etc",
}
}
}
}
}

View file

@ -1,14 +0,0 @@
job "example" {
datacenters = ["dc1"]
type = "batch"
group "example" {
task "hello-world" {
driver = "hello-world-example"
config {
greeting = "hello"
}
}
}
}

View file

@ -4,7 +4,6 @@ import (
"context" "context"
"fmt" "fmt"
"os" "os"
"path/filepath"
"runtime" "runtime"
"sync" "sync"
"time" "time"
@ -92,7 +91,7 @@ var (
driverCapabilities = &drivers.Capabilities{ driverCapabilities = &drivers.Capabilities{
SendSignals: true, SendSignals: true,
Exec: true, Exec: true,
FSIsolation: drivers.FSIsolationChroot, FSIsolation: drivers.FSIsolationNone,
NetIsolationModes: []drivers.NetIsolationMode{ NetIsolationModes: []drivers.NetIsolationMode{
drivers.NetIsolationModeHost, drivers.NetIsolationModeHost,
drivers.NetIsolationModeGroup, drivers.NetIsolationModeGroup,
@ -111,9 +110,6 @@ type Driver struct {
// config is the driver configuration set by the SetConfig RPC // config is the driver configuration set by the SetConfig RPC
config Config config Config
// nomadConfig is the client config from nomad
nomadConfig *base.ClientDriverConfig
// tasks is the in memory datastore mapping taskIDs to driverHandles // tasks is the in memory datastore mapping taskIDs to driverHandles
tasks *taskStore tasks *taskStore
@ -233,7 +229,6 @@ func (tc *TaskConfig) validate() error {
// StartTask. This information is needed to rebuild the task state and handler // StartTask. This information is needed to rebuild the task state and handler
// during recovery. // during recovery.
type TaskState struct { type TaskState struct {
ReattachConfig *pstructs.ReattachConfig
TaskConfig *drivers.TaskConfig TaskConfig *drivers.TaskConfig
Pid int Pid int
StartedAt time.Time StartedAt time.Time
@ -295,9 +290,6 @@ func (d *Driver) SetConfig(cfg *base.Config) error {
} }
d.config = config d.config = config
if cfg != nil && cfg.AgentConfig != nil {
d.nomadConfig = cfg.AgentConfig.Driver
}
return nil return nil
} }
@ -399,24 +391,13 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error {
return fmt.Errorf("failed to decode task state from handle: %v", err) return fmt.Errorf("failed to decode task state from handle: %v", err)
} }
// Create client for reattached executor // Create new executor
plugRC, err := pstructs.ReattachConfigToGoPlugin(taskState.ReattachConfig) exec := executor.NewExecutorWithIsolation(
if err != nil { d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID),)
d.logger.Error("failed to build ReattachConfig from task state", "error", err, "task_id", handle.Config.ID)
return fmt.Errorf("failed to build ReattachConfig from task state: %v", err)
}
exec, pluginClient, err := executor.ReattachToExecutor(plugRC,
d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID))
if err != nil {
d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID)
return fmt.Errorf("failed to reattach to executor: %v", err)
}
h := &taskHandle{ h := &taskHandle{
exec: exec, exec: exec,
pid: taskState.Pid, pid: taskState.Pid,
pluginClient: pluginClient,
taskConfig: taskState.TaskConfig, taskConfig: taskState.TaskConfig,
procState: drivers.TaskStateRunning, procState: drivers.TaskStateRunning,
startedAt: taskState.StartedAt, startedAt: taskState.StartedAt,
@ -448,19 +429,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
handle := drivers.NewTaskHandle(taskHandleVersion) handle := drivers.NewTaskHandle(taskHandleVersion)
handle.Config = cfg handle.Config = cfg
pluginLogFile := filepath.Join(cfg.TaskDir().Dir, "executor.out") exec := executor.NewExecutorWithIsolation(
executorConfig := &executor.ExecutorConfig{ d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID),)
LogFile: pluginLogFile,
LogLevel: "debug",
FSIsolation: true,
}
exec, pluginClient, err := executor.CreateExecutor(
d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID),
d.nomadConfig, executorConfig)
if err != nil {
return nil, nil, fmt.Errorf("failed to create executor: %v", err)
}
user := cfg.User user := cfg.User
if user == "" { if user == "" {
@ -476,23 +446,27 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
} }
if driverConfig.Bind != nil { if driverConfig.Bind != nil {
for k, v := range driverConfig.Bind { for host, task := range driverConfig.Bind {
cfg.Mounts = append(cfg.Mounts, &drivers.MountConfig{ mount_config := drivers.MountConfig{
TaskPath: v, TaskPath: task,
HostPath: k, HostPath: host,
Readonly: false, Readonly: false,
PropagationMode: "private", PropagationMode: "private",
}) }
d.logger.Info("got mount (RW)", "mount_config", hclog.Fmt("%+v", mount_config))
cfg.Mounts = append(cfg.Mounts, &mount_config)
} }
} }
if driverConfig.BindReadOnly != nil { if driverConfig.BindReadOnly != nil {
for k, v := range driverConfig.Bind { for host, task := range driverConfig.BindReadOnly {
cfg.Mounts = append(cfg.Mounts, &drivers.MountConfig{ mount_config := drivers.MountConfig{
TaskPath: v, TaskPath: task,
HostPath: k, HostPath: host,
Readonly: true, Readonly: true,
PropagationMode: "private", PropagationMode: "private",
}) }
d.logger.Info("got mount (RO)", "mount_config", hclog.Fmt("%+v", mount_config))
cfg.Mounts = append(cfg.Mounts, &mount_config)
} }
} }
@ -523,16 +497,16 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
Capabilities: caps, Capabilities: caps,
} }
d.logger.Info("launching with", "exec_cmd", hclog.Fmt("%+v", execCmd))
ps, err := exec.Launch(execCmd) ps, err := exec.Launch(execCmd)
if err != nil { if err != nil {
pluginClient.Kill()
return nil, nil, fmt.Errorf("failed to launch command with executor: %v", err) return nil, nil, fmt.Errorf("failed to launch command with executor: %v", err)
} }
h := &taskHandle{ h := &taskHandle{
exec: exec, exec: exec,
pid: ps.Pid, pid: ps.Pid,
pluginClient: pluginClient,
taskConfig: cfg, taskConfig: cfg,
procState: drivers.TaskStateRunning, procState: drivers.TaskStateRunning,
startedAt: time.Now().Round(time.Millisecond), startedAt: time.Now().Round(time.Millisecond),
@ -540,7 +514,6 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
} }
driverState := TaskState{ driverState := TaskState{
ReattachConfig: pstructs.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()),
Pid: ps.Pid, Pid: ps.Pid,
TaskConfig: cfg, TaskConfig: cfg,
StartedAt: h.startedAt, StartedAt: h.startedAt,
@ -549,7 +522,6 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
if err := handle.SetDriverState(&driverState); err != nil { if err := handle.SetDriverState(&driverState); err != nil {
d.logger.Error("failed to start task, error setting driver state", "error", err) d.logger.Error("failed to start task, error setting driver state", "error", err)
_ = exec.Shutdown("", 0) _ = exec.Shutdown("", 0)
pluginClient.Kill()
return nil, nil, fmt.Errorf("failed to set driver state: %v", err) return nil, nil, fmt.Errorf("failed to set driver state: %v", err)
} }
@ -601,9 +573,6 @@ func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) e
} }
if err := handle.exec.Shutdown(signal, timeout); err != nil { if err := handle.exec.Shutdown(signal, timeout); err != nil {
if handle.pluginClient.Exited() {
return nil
}
return fmt.Errorf("executor Shutdown failed: %v", err) return fmt.Errorf("executor Shutdown failed: %v", err)
} }
@ -639,12 +608,8 @@ func (d *Driver) DestroyTask(taskID string, force bool) error {
return fmt.Errorf("cannot destroy running task") return fmt.Errorf("cannot destroy running task")
} }
if !handle.pluginClient.Exited() { if err := handle.exec.Shutdown("", 0); err != nil {
if err := handle.exec.Shutdown("", 0); err != nil { handle.logger.Error("destroying executor failed", "error", err)
handle.logger.Error("destroying executor failed", "error", err)
}
handle.pluginClient.Kill()
} }
// workaround for the case where DestroyTask was issued on task restart // workaround for the case where DestroyTask was issued on task restart

View file

@ -7,7 +7,6 @@ import (
"time" "time"
hclog "github.com/hashicorp/go-hclog" hclog "github.com/hashicorp/go-hclog"
plugin "github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/plugins/drivers"
) )
@ -15,7 +14,6 @@ import (
type taskHandle struct { type taskHandle struct {
exec executor.Executor exec executor.Executor
pid int pid int
pluginClient *plugin.Client
logger hclog.Logger logger hclog.Logger
// stateLock syncs access to all fields below // stateLock syncs access to all fields below