Vendor executor module so that we can patch it
This commit is contained in:
parent
63e31b9ed9
commit
bf3165a706
GPG key ID:
0E496D15096376BE
19 changed files with 3213 additions and 2 deletions
|
|
@ -13,7 +13,7 @@ import (
|
|||
"github.com/hashicorp/nomad/client/lib/cgutil"
|
||||
"github.com/hashicorp/nomad/drivers/shared/capabilities"
|
||||
"github.com/hashicorp/nomad/drivers/shared/eventer"
|
||||
"github.com/hashicorp/nomad/drivers/shared/executor"
|
||||
"github.com/Alexis211/nomad-driver-exec2/executor"
|
||||
"github.com/hashicorp/nomad/drivers/shared/resolvconf"
|
||||
"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
|
||||
"github.com/hashicorp/nomad/helper/pluginutils/loader"
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import (
|
|||
"time"
|
||||
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/nomad/drivers/shared/executor"
|
||||
"github.com/Alexis211/nomad-driver-exec2/executor"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
)
|
||||
|
||||
|
|
|
|||
285
executor/exec_utils.go
Normal file
285
executor/exec_utils.go
Normal file
|
|
@ -0,0 +1,285 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
dproto "github.com/hashicorp/nomad/plugins/drivers/proto"
|
||||
)
|
||||
|
||||
// execHelper is a convenient wrapper for starting and executing commands, and handling their output
|
||||
type execHelper struct {
|
||||
logger hclog.Logger
|
||||
|
||||
// newTerminal function creates a tty appropriate for the command
|
||||
// The returned pty end of tty function is to be called after process start.
|
||||
newTerminal func() (pty func() (*os.File, error), tty *os.File, err error)
|
||||
|
||||
// setTTY is a callback to configure the command with slave end of the tty of the terminal, when tty is enabled
|
||||
setTTY func(tty *os.File) error
|
||||
|
||||
// setTTY is a callback to configure the command with std{in|out|err}, when tty is disabled
|
||||
setIO func(stdin io.Reader, stdout, stderr io.Writer) error
|
||||
|
||||
// processStart starts the process, like `exec.Cmd.Start()`
|
||||
processStart func() error
|
||||
|
||||
// processWait blocks until command terminates and returns its final state
|
||||
processWait func() (*os.ProcessState, error)
|
||||
}
|
||||
|
||||
func (e *execHelper) run(ctx context.Context, tty bool, stream drivers.ExecTaskStream) error {
|
||||
if tty {
|
||||
return e.runTTY(ctx, stream)
|
||||
}
|
||||
return e.runNoTTY(ctx, stream)
|
||||
}
|
||||
|
||||
func (e *execHelper) runTTY(ctx context.Context, stream drivers.ExecTaskStream) error {
|
||||
ptyF, tty, err := e.newTerminal()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open a tty: %v", err)
|
||||
}
|
||||
defer tty.Close()
|
||||
|
||||
if err := e.setTTY(tty); err != nil {
|
||||
return fmt.Errorf("failed to set command tty: %v", err)
|
||||
}
|
||||
if err := e.processStart(); err != nil {
|
||||
return fmt.Errorf("failed to start command: %v", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errCh := make(chan error, 3)
|
||||
|
||||
pty, err := ptyF()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get pty: %v", err)
|
||||
}
|
||||
|
||||
defer pty.Close()
|
||||
wg.Add(1)
|
||||
go handleStdin(e.logger, pty, stream, errCh)
|
||||
// when tty is on, stdout and stderr point to the same pty so only read once
|
||||
go handleStdout(e.logger, pty, &wg, stream.Send, errCh)
|
||||
|
||||
ps, err := e.processWait()
|
||||
|
||||
// force close streams to close out the stream copying goroutines
|
||||
tty.Close()
|
||||
|
||||
// wait until we get all process output
|
||||
wg.Wait()
|
||||
|
||||
// wait to flush out output
|
||||
stream.Send(cmdExitResult(ps, err))
|
||||
|
||||
select {
|
||||
case cerr := <-errCh:
|
||||
return cerr
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (e *execHelper) runNoTTY(ctx context.Context, stream drivers.ExecTaskStream) error {
|
||||
var sendLock sync.Mutex
|
||||
send := func(v *drivers.ExecTaskStreamingResponseMsg) error {
|
||||
sendLock.Lock()
|
||||
defer sendLock.Unlock()
|
||||
|
||||
return stream.Send(v)
|
||||
}
|
||||
|
||||
stdinPr, stdinPw := io.Pipe()
|
||||
stdoutPr, stdoutPw := io.Pipe()
|
||||
stderrPr, stderrPw := io.Pipe()
|
||||
|
||||
defer stdoutPw.Close()
|
||||
defer stderrPw.Close()
|
||||
|
||||
if err := e.setIO(stdinPr, stdoutPw, stderrPw); err != nil {
|
||||
return fmt.Errorf("failed to set command io: %v", err)
|
||||
}
|
||||
|
||||
if err := e.processStart(); err != nil {
|
||||
return fmt.Errorf("failed to start command: %v", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errCh := make(chan error, 3)
|
||||
|
||||
wg.Add(2)
|
||||
go handleStdin(e.logger, stdinPw, stream, errCh)
|
||||
go handleStdout(e.logger, stdoutPr, &wg, send, errCh)
|
||||
go handleStderr(e.logger, stderrPr, &wg, send, errCh)
|
||||
|
||||
ps, err := e.processWait()
|
||||
|
||||
// force close streams to close out the stream copying goroutines
|
||||
stdinPr.Close()
|
||||
stdoutPw.Close()
|
||||
stderrPw.Close()
|
||||
|
||||
// wait until we get all process output
|
||||
wg.Wait()
|
||||
|
||||
// wait to flush out output
|
||||
stream.Send(cmdExitResult(ps, err))
|
||||
|
||||
select {
|
||||
case cerr := <-errCh:
|
||||
return cerr
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
func cmdExitResult(ps *os.ProcessState, err error) *drivers.ExecTaskStreamingResponseMsg {
|
||||
exitCode := -1
|
||||
|
||||
if ps == nil {
|
||||
if ee, ok := err.(*exec.ExitError); ok {
|
||||
ps = ee.ProcessState
|
||||
}
|
||||
}
|
||||
|
||||
if ps == nil {
|
||||
exitCode = -2
|
||||
} else if status, ok := ps.Sys().(syscall.WaitStatus); ok {
|
||||
exitCode = status.ExitStatus()
|
||||
if status.Signaled() {
|
||||
const exitSignalBase = 128
|
||||
signal := int(status.Signal())
|
||||
exitCode = exitSignalBase + signal
|
||||
}
|
||||
}
|
||||
|
||||
return &drivers.ExecTaskStreamingResponseMsg{
|
||||
Exited: true,
|
||||
Result: &dproto.ExitResult{
|
||||
ExitCode: int32(exitCode),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func handleStdin(logger hclog.Logger, stdin io.WriteCloser, stream drivers.ExecTaskStream, errCh chan<- error) {
|
||||
for {
|
||||
m, err := stream.Recv()
|
||||
if isClosedError(err) {
|
||||
return
|
||||
} else if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
if m.Stdin != nil {
|
||||
if len(m.Stdin.Data) != 0 {
|
||||
_, err := stdin.Write(m.Stdin.Data)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
if m.Stdin.Close {
|
||||
stdin.Close()
|
||||
}
|
||||
} else if m.TtySize != nil {
|
||||
err := setTTYSize(stdin, m.TtySize.Height, m.TtySize.Width)
|
||||
if err != nil {
|
||||
errCh <- fmt.Errorf("failed to resize tty: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleStdout(logger hclog.Logger, reader io.Reader, wg *sync.WaitGroup, send func(*drivers.ExecTaskStreamingResponseMsg) error, errCh chan<- error) {
|
||||
defer wg.Done()
|
||||
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := reader.Read(buf)
|
||||
// always send output first if we read something
|
||||
if n > 0 {
|
||||
if err := send(&drivers.ExecTaskStreamingResponseMsg{
|
||||
Stdout: &dproto.ExecTaskStreamingIOOperation{
|
||||
Data: buf[:n],
|
||||
},
|
||||
}); err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// then process error
|
||||
if isClosedError(err) {
|
||||
if err := send(&drivers.ExecTaskStreamingResponseMsg{
|
||||
Stdout: &dproto.ExecTaskStreamingIOOperation{
|
||||
Close: true,
|
||||
},
|
||||
}); err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
return
|
||||
} else if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func handleStderr(logger hclog.Logger, reader io.Reader, wg *sync.WaitGroup, send func(*drivers.ExecTaskStreamingResponseMsg) error, errCh chan<- error) {
|
||||
defer wg.Done()
|
||||
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := reader.Read(buf)
|
||||
// always send output first if we read something
|
||||
if n > 0 {
|
||||
if err := send(&drivers.ExecTaskStreamingResponseMsg{
|
||||
Stderr: &dproto.ExecTaskStreamingIOOperation{
|
||||
Data: buf[:n],
|
||||
},
|
||||
}); err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// then process error
|
||||
if isClosedError(err) {
|
||||
if err := send(&drivers.ExecTaskStreamingResponseMsg{
|
||||
Stderr: &dproto.ExecTaskStreamingIOOperation{
|
||||
Close: true,
|
||||
},
|
||||
}); err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
return
|
||||
} else if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func isClosedError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return err == io.EOF ||
|
||||
err == io.ErrClosedPipe ||
|
||||
isUnixEIOErr(err)
|
||||
}
|
||||
722
executor/executor.go
Normal file
722
executor/executor.go
Normal file
|
|
@ -0,0 +1,722 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/armon/circbuf"
|
||||
"github.com/creack/pty"
|
||||
"github.com/hashicorp/consul-template/signals"
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/hashicorp/nomad/client/allocdir"
|
||||
"github.com/hashicorp/nomad/client/lib/fifo"
|
||||
"github.com/hashicorp/nomad/client/lib/resources"
|
||||
"github.com/hashicorp/nomad/client/stats"
|
||||
cstructs "github.com/hashicorp/nomad/client/structs"
|
||||
shelpers "github.com/hashicorp/nomad/helper/stats"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
"github.com/syndtr/gocapability/capability"
|
||||
)
|
||||
|
||||
const (
|
||||
// ExecutorVersionLatest is the current and latest version of the executor
|
||||
ExecutorVersionLatest = "2.0.0"
|
||||
|
||||
// ExecutorVersionPre0_9 is the version of executor use prior to the release
|
||||
// of 0.9.x
|
||||
ExecutorVersionPre0_9 = "1.1.0"
|
||||
|
||||
// IsolationModePrivate represents the private isolation mode for a namespace
|
||||
IsolationModePrivate = "private"
|
||||
|
||||
// IsolationModeHost represents the host isolation mode for a namespace
|
||||
IsolationModeHost = "host"
|
||||
)
|
||||
|
||||
var (
|
||||
// The statistics the basic executor exposes
|
||||
ExecutorBasicMeasuredMemStats = []string{"RSS", "Swap"}
|
||||
ExecutorBasicMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
|
||||
)
|
||||
|
||||
// Executor is the interface which allows a driver to launch and supervise
|
||||
// a process
|
||||
type Executor interface {
|
||||
// Launch a user process configured by the given ExecCommand
|
||||
Launch(launchCmd *ExecCommand) (*ProcessState, error)
|
||||
|
||||
// Wait blocks until the process exits or an error occures
|
||||
Wait(ctx context.Context) (*ProcessState, error)
|
||||
|
||||
// Shutdown will shutdown the executor by stopping the user process,
|
||||
// cleaning up and resources created by the executor. The shutdown sequence
|
||||
// will first send the given signal to the process. This defaults to "SIGINT"
|
||||
// if not specified. The executor will then wait for the process to exit
|
||||
// before cleaning up other resources. If the executor waits longer than the
|
||||
// given grace period, the process is forcefully killed.
|
||||
//
|
||||
// To force kill the user process, gracePeriod can be set to 0.
|
||||
Shutdown(signal string, gracePeriod time.Duration) error
|
||||
|
||||
// UpdateResources updates any resource isolation enforcement with new
|
||||
// constraints if supported.
|
||||
UpdateResources(*drivers.Resources) error
|
||||
|
||||
// Version returns the executor API version
|
||||
Version() (*ExecutorVersion, error)
|
||||
|
||||
// Returns a channel of stats. Stats are collected and
|
||||
// pushed to the channel on the given interval
|
||||
Stats(context.Context, time.Duration) (<-chan *cstructs.TaskResourceUsage, error)
|
||||
|
||||
// Signal sends the given signal to the user process
|
||||
Signal(os.Signal) error
|
||||
|
||||
// Exec executes the given command and args inside the executor context
|
||||
// and returns the output and exit code.
|
||||
Exec(deadline time.Time, cmd string, args []string) ([]byte, int, error)
|
||||
|
||||
ExecStreaming(ctx context.Context, cmd []string, tty bool,
|
||||
stream drivers.ExecTaskStream) error
|
||||
}
|
||||
|
||||
// ExecCommand holds the user command, args, and other isolation related
|
||||
// settings.
|
||||
//
|
||||
// Important (!): when adding fields, make sure to update the RPC methods in
|
||||
// grpcExecutorClient.Launch and grpcExecutorServer.Launch. Number of hours
|
||||
// spent tracking this down: too many.
|
||||
type ExecCommand struct {
|
||||
// Cmd is the command that the user wants to run.
|
||||
Cmd string
|
||||
|
||||
// Args is the args of the command that the user wants to run.
|
||||
Args []string
|
||||
|
||||
// Resources defined by the task
|
||||
Resources *drivers.Resources
|
||||
|
||||
// StdoutPath is the path the process stdout should be written to
|
||||
StdoutPath string
|
||||
stdout io.WriteCloser
|
||||
|
||||
// StderrPath is the path the process stderr should be written to
|
||||
StderrPath string
|
||||
stderr io.WriteCloser
|
||||
|
||||
// Env is the list of KEY=val pairs of environment variables to be set
|
||||
Env []string
|
||||
|
||||
// User is the user which the executor uses to run the command.
|
||||
User string
|
||||
|
||||
// TaskDir is the directory path on the host where for the task
|
||||
TaskDir string
|
||||
|
||||
// ResourceLimits determines whether resource limits are enforced by the
|
||||
// executor.
|
||||
ResourceLimits bool
|
||||
|
||||
// Cgroup marks whether we put the process in a cgroup. Setting this field
|
||||
// doesn't enforce resource limits. To enforce limits, set ResourceLimits.
|
||||
// Using the cgroup does allow more precise cleanup of processes.
|
||||
BasicProcessCgroup bool
|
||||
|
||||
// NoPivotRoot disables using pivot_root for isolation, useful when the root
|
||||
// partition is on a ramdisk which does not support pivot_root,
|
||||
// see man 2 pivot_root
|
||||
NoPivotRoot bool
|
||||
|
||||
// Mounts are the host paths to be be made available inside rootfs
|
||||
Mounts []*drivers.MountConfig
|
||||
|
||||
// Devices are the the device nodes to be created in isolation environment
|
||||
Devices []*drivers.DeviceConfig
|
||||
|
||||
// NetworkIsolation is the network isolation configuration.
|
||||
NetworkIsolation *drivers.NetworkIsolationSpec
|
||||
|
||||
// ModePID is the PID isolation mode (private or host).
|
||||
ModePID string
|
||||
|
||||
// ModeIPC is the IPC isolation mode (private or host).
|
||||
ModeIPC string
|
||||
|
||||
// Capabilities are the linux capabilities to be enabled by the task driver.
|
||||
Capabilities []string
|
||||
}
|
||||
|
||||
// SetWriters sets the writer for the process stdout and stderr. This should
|
||||
// not be used if writing to a file path such as a fifo file. SetStdoutWriter
|
||||
// is mainly used for unit testing purposes.
|
||||
func (c *ExecCommand) SetWriters(out io.WriteCloser, err io.WriteCloser) {
|
||||
c.stdout = out
|
||||
c.stderr = err
|
||||
}
|
||||
|
||||
// GetWriters returns the unexported io.WriteCloser for the stdout and stderr
|
||||
// handles. This is mainly used for unit testing purposes.
|
||||
func (c *ExecCommand) GetWriters() (stdout io.WriteCloser, stderr io.WriteCloser) {
|
||||
return c.stdout, c.stderr
|
||||
}
|
||||
|
||||
type nopCloser struct {
|
||||
io.Writer
|
||||
}
|
||||
|
||||
func (nopCloser) Close() error { return nil }
|
||||
|
||||
// Stdout returns a writer for the configured file descriptor
|
||||
func (c *ExecCommand) Stdout() (io.WriteCloser, error) {
|
||||
if c.stdout == nil {
|
||||
if c.StdoutPath != "" {
|
||||
f, err := fifo.OpenWriter(c.StdoutPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create stdout: %v", err)
|
||||
}
|
||||
c.stdout = f
|
||||
} else {
|
||||
c.stdout = nopCloser{ioutil.Discard}
|
||||
}
|
||||
}
|
||||
return c.stdout, nil
|
||||
}
|
||||
|
||||
// Stderr returns a writer for the configured file descriptor
|
||||
func (c *ExecCommand) Stderr() (io.WriteCloser, error) {
|
||||
if c.stderr == nil {
|
||||
if c.StderrPath != "" {
|
||||
f, err := fifo.OpenWriter(c.StderrPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create stderr: %v", err)
|
||||
}
|
||||
c.stderr = f
|
||||
} else {
|
||||
c.stderr = nopCloser{ioutil.Discard}
|
||||
}
|
||||
}
|
||||
return c.stderr, nil
|
||||
}
|
||||
|
||||
func (c *ExecCommand) Close() {
|
||||
if c.stdout != nil {
|
||||
c.stdout.Close()
|
||||
}
|
||||
if c.stderr != nil {
|
||||
c.stderr.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessState holds information about the state of a user process.
|
||||
type ProcessState struct {
|
||||
Pid int
|
||||
ExitCode int
|
||||
Signal int
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
// ExecutorVersion is the version of the executor
|
||||
type ExecutorVersion struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
func (v *ExecutorVersion) GoString() string {
|
||||
return v.Version
|
||||
}
|
||||
|
||||
// UniversalExecutor is an implementation of the Executor which launches and
|
||||
// supervises processes. In addition to process supervision it provides resource
|
||||
// and file system isolation
|
||||
type UniversalExecutor struct {
|
||||
childCmd exec.Cmd
|
||||
commandCfg *ExecCommand
|
||||
|
||||
exitState *ProcessState
|
||||
processExited chan interface{}
|
||||
|
||||
// containment is used to cleanup resources created by the executor
|
||||
// currently only used for killing pids via freezer cgroup on linux
|
||||
containment resources.Containment
|
||||
|
||||
totalCpuStats *stats.CpuStats
|
||||
userCpuStats *stats.CpuStats
|
||||
systemCpuStats *stats.CpuStats
|
||||
pidCollector *pidCollector
|
||||
|
||||
logger hclog.Logger
|
||||
}
|
||||
|
||||
// NewExecutor returns an Executor
|
||||
func NewExecutor(logger hclog.Logger) Executor {
|
||||
logger = logger.Named("executor")
|
||||
if err := shelpers.Init(); err != nil {
|
||||
logger.Error("unable to initialize stats", "error", err)
|
||||
}
|
||||
|
||||
return &UniversalExecutor{
|
||||
logger: logger,
|
||||
processExited: make(chan interface{}),
|
||||
totalCpuStats: stats.NewCpuStats(),
|
||||
userCpuStats: stats.NewCpuStats(),
|
||||
systemCpuStats: stats.NewCpuStats(),
|
||||
pidCollector: newPidCollector(logger),
|
||||
}
|
||||
}
|
||||
|
||||
// Version returns the api version of the executor
|
||||
func (e *UniversalExecutor) Version() (*ExecutorVersion, error) {
|
||||
return &ExecutorVersion{Version: ExecutorVersionLatest}, nil
|
||||
}
|
||||
|
||||
// Launch launches the main process and returns its state. It also
|
||||
// configures an applies isolation on certain platforms.
|
||||
func (e *UniversalExecutor) Launch(command *ExecCommand) (*ProcessState, error) {
|
||||
e.logger.Trace("preparing to launch command", "command", command.Cmd, "args", strings.Join(command.Args, " "))
|
||||
|
||||
e.commandCfg = command
|
||||
|
||||
// setting the user of the process
|
||||
if command.User != "" {
|
||||
e.logger.Debug("running command as user", "user", command.User)
|
||||
if err := setCmdUser(&e.childCmd, command.User); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// set the task dir as the working directory for the command
|
||||
e.childCmd.Dir = e.commandCfg.TaskDir
|
||||
|
||||
// start command in separate process group
|
||||
if err := e.setNewProcessGroup(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Maybe setup containment (for now, cgroups only only on linux)
|
||||
if e.commandCfg.ResourceLimits || e.commandCfg.BasicProcessCgroup {
|
||||
pid := os.Getpid()
|
||||
if err := e.configureResourceContainer(pid); err != nil {
|
||||
e.logger.Error("failed to configure resource container", "pid", pid, "error", err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
stdout, err := e.commandCfg.Stdout()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stderr, err := e.commandCfg.Stderr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
e.childCmd.Stdout = stdout
|
||||
e.childCmd.Stderr = stderr
|
||||
|
||||
// Look up the binary path and make it executable
|
||||
absPath, err := lookupBin(command.TaskDir, command.Cmd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := makeExecutable(absPath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
path := absPath
|
||||
|
||||
// Set the commands arguments
|
||||
e.childCmd.Path = path
|
||||
e.childCmd.Args = append([]string{e.childCmd.Path}, command.Args...)
|
||||
e.childCmd.Env = e.commandCfg.Env
|
||||
|
||||
// Start the process
|
||||
if err = withNetworkIsolation(e.childCmd.Start, command.NetworkIsolation); err != nil {
|
||||
return nil, fmt.Errorf("failed to start command path=%q --- args=%q: %v", path, e.childCmd.Args, err)
|
||||
}
|
||||
|
||||
go e.pidCollector.collectPids(e.processExited, e.getAllPids)
|
||||
go e.wait()
|
||||
return &ProcessState{Pid: e.childCmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
|
||||
}
|
||||
|
||||
// Exec a command inside a container for exec and java drivers.
|
||||
func (e *UniversalExecutor) Exec(deadline time.Time, name string, args []string) ([]byte, int, error) {
|
||||
ctx, cancel := context.WithDeadline(context.Background(), deadline)
|
||||
defer cancel()
|
||||
return ExecScript(ctx, e.childCmd.Dir, e.commandCfg.Env, e.childCmd.SysProcAttr, e.commandCfg.NetworkIsolation, name, args)
|
||||
}
|
||||
|
||||
// ExecScript executes cmd with args and returns the output, exit code, and
|
||||
// error. Output is truncated to drivers/shared/structs.CheckBufSize
|
||||
func ExecScript(ctx context.Context, dir string, env []string, attrs *syscall.SysProcAttr,
|
||||
netSpec *drivers.NetworkIsolationSpec, name string, args []string) ([]byte, int, error) {
|
||||
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
|
||||
// Copy runtime environment from the main command
|
||||
cmd.SysProcAttr = attrs
|
||||
cmd.Dir = dir
|
||||
cmd.Env = env
|
||||
|
||||
// Capture output
|
||||
buf, _ := circbuf.NewBuffer(int64(drivers.CheckBufSize))
|
||||
cmd.Stdout = buf
|
||||
cmd.Stderr = buf
|
||||
|
||||
if err := withNetworkIsolation(cmd.Run, netSpec); err != nil {
|
||||
exitErr, ok := err.(*exec.ExitError)
|
||||
if !ok {
|
||||
// Non-exit error, return it and let the caller treat
|
||||
// it as a critical failure
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
// Some kind of error happened; default to critical
|
||||
exitCode := 2
|
||||
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
|
||||
exitCode = status.ExitStatus()
|
||||
}
|
||||
|
||||
// Don't return the exitError as the caller only needs the
|
||||
// output and code.
|
||||
return buf.Bytes(), exitCode, nil
|
||||
}
|
||||
return buf.Bytes(), 0, nil
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) ExecStreaming(ctx context.Context, command []string, tty bool,
|
||||
stream drivers.ExecTaskStream) error {
|
||||
|
||||
if len(command) == 0 {
|
||||
return fmt.Errorf("command is required")
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, command[0], command[1:]...)
|
||||
|
||||
cmd.Dir = "/"
|
||||
cmd.Env = e.childCmd.Env
|
||||
|
||||
execHelper := &execHelper{
|
||||
logger: e.logger,
|
||||
|
||||
newTerminal: func() (func() (*os.File, error), *os.File, error) {
|
||||
pty, tty, err := pty.Open()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return func() (*os.File, error) { return pty, nil }, tty, err
|
||||
},
|
||||
setTTY: func(tty *os.File) error {
|
||||
cmd.SysProcAttr = sessionCmdAttr(tty)
|
||||
|
||||
cmd.Stdin = tty
|
||||
cmd.Stdout = tty
|
||||
cmd.Stderr = tty
|
||||
return nil
|
||||
},
|
||||
setIO: func(stdin io.Reader, stdout, stderr io.Writer) error {
|
||||
cmd.Stdin = stdin
|
||||
cmd.Stdout = stdout
|
||||
cmd.Stderr = stderr
|
||||
return nil
|
||||
},
|
||||
processStart: func() error {
|
||||
if u := e.commandCfg.User; u != "" {
|
||||
if err := setCmdUser(cmd, u); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return withNetworkIsolation(cmd.Start, e.commandCfg.NetworkIsolation)
|
||||
},
|
||||
processWait: func() (*os.ProcessState, error) {
|
||||
err := cmd.Wait()
|
||||
return cmd.ProcessState, err
|
||||
},
|
||||
}
|
||||
|
||||
return execHelper.run(ctx, tty, stream)
|
||||
}
|
||||
|
||||
// Wait waits until a process has exited and returns it's exitcode and errors
|
||||
func (e *UniversalExecutor) Wait(ctx context.Context) (*ProcessState, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-e.processExited:
|
||||
return e.exitState, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) UpdateResources(resources *drivers.Resources) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) wait() {
|
||||
defer close(e.processExited)
|
||||
defer e.commandCfg.Close()
|
||||
pid := e.childCmd.Process.Pid
|
||||
err := e.childCmd.Wait()
|
||||
if err == nil {
|
||||
e.exitState = &ProcessState{Pid: pid, ExitCode: 0, Time: time.Now()}
|
||||
return
|
||||
}
|
||||
|
||||
exitCode := 1
|
||||
var signal int
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
|
||||
exitCode = status.ExitStatus()
|
||||
if status.Signaled() {
|
||||
// bash(1) uses the lower 7 bits of a uint8
|
||||
// to indicate normal program failure (see
|
||||
// <sysexits.h>). If a process terminates due
|
||||
// to a signal, encode the signal number to
|
||||
// indicate which signal caused the process
|
||||
// to terminate. Mirror this exit code
|
||||
// encoding scheme.
|
||||
const exitSignalBase = 128
|
||||
signal = int(status.Signal())
|
||||
exitCode = exitSignalBase + signal
|
||||
}
|
||||
}
|
||||
} else {
|
||||
e.logger.Warn("unexpected Cmd.Wait() error type", "error", err)
|
||||
}
|
||||
|
||||
e.exitState = &ProcessState{Pid: pid, ExitCode: exitCode, Signal: signal, Time: time.Now()}
|
||||
}
|
||||
|
||||
var (
|
||||
// finishedErr is the error message received when trying to kill and already
|
||||
// exited process.
|
||||
finishedErr = "os: process already finished"
|
||||
|
||||
// noSuchProcessErr is the error message received when trying to kill a non
|
||||
// existing process (e.g. when killing a process group).
|
||||
noSuchProcessErr = "no such process"
|
||||
)
|
||||
|
||||
// Shutdown cleans up the alloc directory, destroys resource container and
|
||||
// kills the user process.
|
||||
func (e *UniversalExecutor) Shutdown(signal string, grace time.Duration) error {
|
||||
e.logger.Debug("shutdown requested", "signal", signal, "grace_period_ms", grace.Round(time.Millisecond))
|
||||
var merr multierror.Error
|
||||
|
||||
// If the executor did not launch a process, return.
|
||||
if e.commandCfg == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If there is no process we can't shutdown
|
||||
if e.childCmd.Process == nil {
|
||||
e.logger.Warn("failed to shutdown due to missing process", "error", "no process found")
|
||||
return fmt.Errorf("executor failed to shutdown error: no process found")
|
||||
}
|
||||
|
||||
proc, err := os.FindProcess(e.childCmd.Process.Pid)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("executor failed to find process: %v", err)
|
||||
e.logger.Warn("failed to shutdown due to inability to find process", "pid", e.childCmd.Process.Pid, "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// If grace is 0 then skip shutdown logic
|
||||
if grace > 0 {
|
||||
// Default signal to SIGINT if not set
|
||||
if signal == "" {
|
||||
signal = "SIGINT"
|
||||
}
|
||||
|
||||
sig, ok := signals.SignalLookup[signal]
|
||||
if !ok {
|
||||
err = fmt.Errorf("error unknown signal given for shutdown: %s", signal)
|
||||
e.logger.Warn("failed to shutdown", "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.shutdownProcess(sig, proc); err != nil {
|
||||
e.logger.Warn("failed to shutdown process", "pid", proc.Pid, "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
select {
|
||||
case <-e.processExited:
|
||||
case <-time.After(grace):
|
||||
proc.Kill()
|
||||
}
|
||||
} else {
|
||||
proc.Kill()
|
||||
}
|
||||
|
||||
// Wait for process to exit
|
||||
select {
|
||||
case <-e.processExited:
|
||||
case <-time.After(time.Second * 15):
|
||||
e.logger.Warn("process did not exit after 15 seconds")
|
||||
merr.Errors = append(merr.Errors, fmt.Errorf("process did not exit after 15 seconds"))
|
||||
}
|
||||
|
||||
// prefer killing the process via platform-dependent resource containment
|
||||
killByContainment := e.commandCfg.ResourceLimits || e.commandCfg.BasicProcessCgroup
|
||||
|
||||
if !killByContainment {
|
||||
// there is no containment, so kill the group the old fashioned way by sending
|
||||
// SIGKILL to the negative pid
|
||||
if cleanupChildrenErr := e.killProcessTree(proc); cleanupChildrenErr != nil && cleanupChildrenErr.Error() != finishedErr {
|
||||
merr.Errors = append(merr.Errors,
|
||||
fmt.Errorf("can't kill process with pid %d: %v", e.childCmd.Process.Pid, cleanupChildrenErr))
|
||||
}
|
||||
} else {
|
||||
// there is containment available (e.g. cgroups) so defer to that implementation
|
||||
// for killing the processes
|
||||
if cleanupErr := e.containment.Cleanup(); cleanupErr != nil {
|
||||
e.logger.Warn("containment cleanup failed", "error", cleanupErr)
|
||||
merr.Errors = append(merr.Errors, cleanupErr)
|
||||
}
|
||||
}
|
||||
|
||||
if err = merr.ErrorOrNil(); err != nil {
|
||||
e.logger.Warn("failed to shutdown due to some error", "error", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Signal sends the passed signal to the task
|
||||
func (e *UniversalExecutor) Signal(s os.Signal) error {
|
||||
if e.childCmd.Process == nil {
|
||||
return fmt.Errorf("Task not yet run")
|
||||
}
|
||||
|
||||
e.logger.Debug("sending signal to PID", "signal", s, "pid", e.childCmd.Process.Pid)
|
||||
err := e.childCmd.Process.Signal(s)
|
||||
if err != nil {
|
||||
e.logger.Error("sending signal failed", "signal", s, "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) Stats(ctx context.Context, interval time.Duration) (<-chan *cstructs.TaskResourceUsage, error) {
|
||||
ch := make(chan *cstructs.TaskResourceUsage)
|
||||
go e.handleStats(ch, ctx, interval)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) handleStats(ch chan *cstructs.TaskResourceUsage, ctx context.Context, interval time.Duration) {
|
||||
defer close(ch)
|
||||
timer := time.NewTimer(0)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
||||
case <-timer.C:
|
||||
timer.Reset(interval)
|
||||
}
|
||||
|
||||
pidStats, err := e.pidCollector.pidStats()
|
||||
if err != nil {
|
||||
e.logger.Warn("error collecting stats", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case ch <- aggregatedResourceUsage(e.systemCpuStats, pidStats):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lookupBin looks for path to the binary to run by looking for the binary in
|
||||
// the following locations, in-order:
|
||||
// task/local/, task/, on the host file system, in host $PATH
|
||||
// The return path is absolute.
|
||||
func lookupBin(taskDir string, bin string) (string, error) {
|
||||
// Check in the local directory
|
||||
local := filepath.Join(taskDir, allocdir.TaskLocal, bin)
|
||||
if _, err := os.Stat(local); err == nil {
|
||||
return local, nil
|
||||
}
|
||||
|
||||
// Check at the root of the task's directory
|
||||
root := filepath.Join(taskDir, bin)
|
||||
if _, err := os.Stat(root); err == nil {
|
||||
return root, nil
|
||||
}
|
||||
|
||||
// when checking host paths, check with Stat first if path is absolute
|
||||
// as exec.LookPath only considers files already marked as executable
|
||||
// and only consider this for absolute paths to avoid depending on
|
||||
// current directory of nomad which may cause unexpected behavior
|
||||
if _, err := os.Stat(bin); err == nil && filepath.IsAbs(bin) {
|
||||
return bin, nil
|
||||
}
|
||||
|
||||
// Check the $PATH
|
||||
if host, err := exec.LookPath(bin); err == nil {
|
||||
return host, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("binary %q could not be found", bin)
|
||||
}
|
||||
|
||||
// makeExecutable makes the given file executable for root,group,others.
|
||||
func makeExecutable(binPath string) error {
|
||||
if runtime.GOOS == "windows" {
|
||||
return nil
|
||||
}
|
||||
|
||||
fi, err := os.Stat(binPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("binary %q does not exist", binPath)
|
||||
}
|
||||
return fmt.Errorf("specified binary is invalid: %v", err)
|
||||
}
|
||||
|
||||
// If it is not executable, make it so.
|
||||
perm := fi.Mode().Perm()
|
||||
req := os.FileMode(0555)
|
||||
if perm&req != req {
|
||||
if err := os.Chmod(binPath, perm|req); err != nil {
|
||||
return fmt.Errorf("error making %q executable: %s", binPath, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SupportedCaps returns a list of all supported capabilities in kernel.
|
||||
func SupportedCaps(allowNetRaw bool) []string {
|
||||
var allCaps []string
|
||||
last := capability.CAP_LAST_CAP
|
||||
// workaround for RHEL6 which has no /proc/sys/kernel/cap_last_cap
|
||||
if last == capability.Cap(63) {
|
||||
last = capability.CAP_BLOCK_SUSPEND
|
||||
}
|
||||
for _, cap := range capability.List() {
|
||||
if cap > last {
|
||||
continue
|
||||
}
|
||||
if !allowNetRaw && cap == capability.CAP_NET_RAW {
|
||||
continue
|
||||
}
|
||||
allCaps = append(allCaps, fmt.Sprintf("CAP_%s", strings.ToUpper(cap.String())))
|
||||
}
|
||||
return allCaps
|
||||
}
|
||||
33
executor/executor_basic.go
Normal file
33
executor/executor_basic.go
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
//go:build !linux
|
||||
|
||||
package executor
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/nomad/client/lib/resources"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
)
|
||||
|
||||
func NewExecutorWithIsolation(logger hclog.Logger) Executor {
|
||||
logger = logger.Named("executor")
|
||||
logger.Error("isolation executor is not supported on this platform, using default")
|
||||
return NewExecutor(logger)
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) configureResourceContainer(_ int) error { return nil }
|
||||
|
||||
func (e *UniversalExecutor) getAllPids() (resources.PIDs, error) {
|
||||
return getAllPidsByScanning()
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) start(command *ExecCommand) error {
|
||||
return e.childCmd.Start()
|
||||
}
|
||||
|
||||
func withNetworkIsolation(f func() error, _ *drivers.NetworkIsolationSpec) error {
|
||||
return f()
|
||||
}
|
||||
|
||||
func setCmdUser(*exec.Cmd, string) error { return nil }
|
||||
926
executor/executor_linux.go
Normal file
926
executor/executor_linux.go
Normal file
|
|
@ -0,0 +1,926 @@
|
|||
//go:build linux
|
||||
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/armon/circbuf"
|
||||
"github.com/hashicorp/consul-template/signals"
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/nomad/client/allocdir"
|
||||
"github.com/hashicorp/nomad/client/lib/cgutil"
|
||||
"github.com/hashicorp/nomad/client/lib/resources"
|
||||
"github.com/hashicorp/nomad/client/stats"
|
||||
cstructs "github.com/hashicorp/nomad/client/structs"
|
||||
"github.com/hashicorp/nomad/drivers/shared/capabilities"
|
||||
shelpers "github.com/hashicorp/nomad/helper/stats"
|
||||
"github.com/hashicorp/nomad/helper/uuid"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
"github.com/opencontainers/runc/libcontainer"
|
||||
"github.com/opencontainers/runc/libcontainer/cgroups"
|
||||
lconfigs "github.com/opencontainers/runc/libcontainer/configs"
|
||||
"github.com/opencontainers/runc/libcontainer/devices"
|
||||
ldevices "github.com/opencontainers/runc/libcontainer/devices"
|
||||
"github.com/opencontainers/runc/libcontainer/specconv"
|
||||
lutils "github.com/opencontainers/runc/libcontainer/utils"
|
||||
"github.com/opencontainers/runtime-spec/specs-go"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
var (
|
||||
// ExecutorCgroupV1MeasuredMemStats is the list of memory stats captured by the executor with cgroup-v1
|
||||
ExecutorCgroupV1MeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
|
||||
|
||||
// ExecutorCgroupV2MeasuredMemStats is the list of memory stats captured by the executor with cgroup-v2. cgroup-v2 exposes different memory stats and no longer reports rss or max usage.
|
||||
ExecutorCgroupV2MeasuredMemStats = []string{"Cache", "Swap", "Usage"}
|
||||
|
||||
// ExecutorCgroupMeasuredCpuStats is the list of CPU stats captures by the executor
|
||||
ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"}
|
||||
)
|
||||
|
||||
// LibcontainerExecutor implements an Executor with the runc/libcontainer api
|
||||
type LibcontainerExecutor struct {
|
||||
id string
|
||||
command *ExecCommand
|
||||
|
||||
logger hclog.Logger
|
||||
|
||||
totalCpuStats *stats.CpuStats
|
||||
userCpuStats *stats.CpuStats
|
||||
systemCpuStats *stats.CpuStats
|
||||
pidCollector *pidCollector
|
||||
|
||||
container libcontainer.Container
|
||||
userProc *libcontainer.Process
|
||||
userProcExited chan interface{}
|
||||
exitState *ProcessState
|
||||
}
|
||||
|
||||
func NewExecutorWithIsolation(logger hclog.Logger) Executor {
|
||||
logger = logger.Named("isolated_executor")
|
||||
if err := shelpers.Init(); err != nil {
|
||||
logger.Error("unable to initialize stats", "error", err)
|
||||
}
|
||||
return &LibcontainerExecutor{
|
||||
id: strings.ReplaceAll(uuid.Generate(), "-", "_"),
|
||||
logger: logger,
|
||||
totalCpuStats: stats.NewCpuStats(),
|
||||
userCpuStats: stats.NewCpuStats(),
|
||||
systemCpuStats: stats.NewCpuStats(),
|
||||
pidCollector: newPidCollector(logger),
|
||||
}
|
||||
}
|
||||
|
||||
// Launch creates a new container in libcontainer and starts a new process with it
|
||||
func (l *LibcontainerExecutor) Launch(command *ExecCommand) (*ProcessState, error) {
|
||||
l.logger.Trace("preparing to launch command", "command", command.Cmd, "args", strings.Join(command.Args, " "))
|
||||
|
||||
if command.Resources == nil {
|
||||
command.Resources = &drivers.Resources{
|
||||
NomadResources: &structs.AllocatedTaskResources{},
|
||||
}
|
||||
}
|
||||
|
||||
l.command = command
|
||||
|
||||
// create a new factory which will store the container state in the allocDir
|
||||
factory, err := libcontainer.New(
|
||||
path.Join(command.TaskDir, "../alloc/container"),
|
||||
// note that os.Args[0] refers to the executor shim typically
|
||||
// and first args arguments is ignored now due
|
||||
// until https://github.com/opencontainers/runc/pull/1888 is merged
|
||||
libcontainer.InitArgs(os.Args[0], "libcontainer-shim"),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create factory: %v", err)
|
||||
}
|
||||
|
||||
// A container groups processes under the same isolation enforcement
|
||||
containerCfg, err := newLibcontainerConfig(command)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to configure container(%s): %v", l.id, err)
|
||||
}
|
||||
|
||||
container, err := factory.Create(l.id, containerCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create container(%s): %v", l.id, err)
|
||||
}
|
||||
l.container = container
|
||||
|
||||
// Look up the binary path and make it executable
|
||||
taskPath, hostPath, err := lookupTaskBin(command)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := makeExecutable(hostPath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
combined := append([]string{taskPath}, command.Args...)
|
||||
stdout, err := command.Stdout()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stderr, err := command.Stderr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
l.logger.Debug("launching", "command", command.Cmd, "args", strings.Join(command.Args, " "))
|
||||
|
||||
// the task process will be started by the container
|
||||
process := &libcontainer.Process{
|
||||
Args: combined,
|
||||
Env: command.Env,
|
||||
Stdout: stdout,
|
||||
Stderr: stderr,
|
||||
Init: true,
|
||||
}
|
||||
|
||||
if command.User != "" {
|
||||
process.User = command.User
|
||||
}
|
||||
l.userProc = process
|
||||
|
||||
l.totalCpuStats = stats.NewCpuStats()
|
||||
l.userCpuStats = stats.NewCpuStats()
|
||||
l.systemCpuStats = stats.NewCpuStats()
|
||||
|
||||
// Starts the task
|
||||
if err := container.Run(process); err != nil {
|
||||
container.Destroy()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pid, err := process.Pid()
|
||||
if err != nil {
|
||||
container.Destroy()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// start a goroutine to wait on the process to complete, so Wait calls can
|
||||
// be multiplexed
|
||||
l.userProcExited = make(chan interface{})
|
||||
go l.pidCollector.collectPids(l.userProcExited, l.getAllPids)
|
||||
go l.wait()
|
||||
|
||||
return &ProcessState{
|
||||
Pid: pid,
|
||||
ExitCode: -1,
|
||||
Time: time.Now(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (l *LibcontainerExecutor) getAllPids() (resources.PIDs, error) {
|
||||
pids, err := l.container.Processes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := make(resources.PIDs, 1)
|
||||
for _, pid := range pids {
|
||||
m[pid] = resources.NewPID(pid)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Wait waits until a process has exited and returns it's exitcode and errors
|
||||
func (l *LibcontainerExecutor) Wait(ctx context.Context) (*ProcessState, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-l.userProcExited:
|
||||
return l.exitState, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (l *LibcontainerExecutor) wait() {
|
||||
defer close(l.userProcExited)
|
||||
|
||||
ps, err := l.userProc.Wait()
|
||||
if err != nil {
|
||||
// If the process has exited before we called wait an error is returned
|
||||
// the process state is embedded in the error
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
ps = exitErr.ProcessState
|
||||
} else {
|
||||
l.logger.Error("failed to call wait on user process", "error", err)
|
||||
l.exitState = &ProcessState{Pid: 0, ExitCode: 1, Time: time.Now()}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
l.command.Close()
|
||||
|
||||
exitCode := 1
|
||||
var signal int
|
||||
if status, ok := ps.Sys().(syscall.WaitStatus); ok {
|
||||
exitCode = status.ExitStatus()
|
||||
if status.Signaled() {
|
||||
const exitSignalBase = 128
|
||||
signal = int(status.Signal())
|
||||
exitCode = exitSignalBase + signal
|
||||
}
|
||||
}
|
||||
|
||||
l.exitState = &ProcessState{
|
||||
Pid: ps.Pid(),
|
||||
ExitCode: exitCode,
|
||||
Signal: signal,
|
||||
Time: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown stops all processes started and cleans up any resources
|
||||
// created (such as mountpoints, devices, etc).
|
||||
func (l *LibcontainerExecutor) Shutdown(signal string, grace time.Duration) error {
|
||||
if l.container == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
status, err := l.container.Status()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer l.container.Destroy()
|
||||
|
||||
if status == libcontainer.Stopped {
|
||||
return nil
|
||||
}
|
||||
|
||||
if grace > 0 {
|
||||
if signal == "" {
|
||||
signal = "SIGINT"
|
||||
}
|
||||
|
||||
sig, ok := signals.SignalLookup[signal]
|
||||
if !ok {
|
||||
return fmt.Errorf("error unknown signal given for shutdown: %s", signal)
|
||||
}
|
||||
|
||||
// Signal initial container processes only during graceful
|
||||
// shutdown; hence `false` arg.
|
||||
err = l.container.Signal(sig, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
select {
|
||||
case <-l.userProcExited:
|
||||
return nil
|
||||
case <-time.After(grace):
|
||||
// Force kill all container processes after grace period,
|
||||
// hence `true` argument.
|
||||
if err := l.container.Signal(os.Kill, true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err := l.container.Signal(os.Kill, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-l.userProcExited:
|
||||
return nil
|
||||
case <-time.After(time.Second * 15):
|
||||
return fmt.Errorf("process failed to exit after 15 seconds")
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateResources updates the resource isolation with new values to be enforced
|
||||
func (l *LibcontainerExecutor) UpdateResources(resources *drivers.Resources) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Version returns the api version of the executor
|
||||
func (l *LibcontainerExecutor) Version() (*ExecutorVersion, error) {
|
||||
return &ExecutorVersion{Version: ExecutorVersionLatest}, nil
|
||||
}
|
||||
|
||||
// Stats returns the resource statistics for processes managed by the executor
|
||||
func (l *LibcontainerExecutor) Stats(ctx context.Context, interval time.Duration) (<-chan *cstructs.TaskResourceUsage, error) {
|
||||
ch := make(chan *cstructs.TaskResourceUsage)
|
||||
go l.handleStats(ch, ctx, interval)
|
||||
return ch, nil
|
||||
|
||||
}
|
||||
|
||||
func (l *LibcontainerExecutor) handleStats(ch chan *cstructs.TaskResourceUsage, ctx context.Context, interval time.Duration) {
|
||||
defer close(ch)
|
||||
timer := time.NewTimer(0)
|
||||
|
||||
measuredMemStats := ExecutorCgroupV1MeasuredMemStats
|
||||
if cgroups.IsCgroup2UnifiedMode() {
|
||||
measuredMemStats = ExecutorCgroupV2MeasuredMemStats
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
||||
case <-timer.C:
|
||||
timer.Reset(interval)
|
||||
}
|
||||
|
||||
lstats, err := l.container.Stats()
|
||||
if err != nil {
|
||||
l.logger.Warn("error collecting stats", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
pidStats, err := l.pidCollector.pidStats()
|
||||
if err != nil {
|
||||
l.logger.Warn("error collecting stats", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
ts := time.Now()
|
||||
stats := lstats.CgroupStats
|
||||
|
||||
// Memory Related Stats
|
||||
swap := stats.MemoryStats.SwapUsage
|
||||
maxUsage := stats.MemoryStats.Usage.MaxUsage
|
||||
rss := stats.MemoryStats.Stats["rss"]
|
||||
cache := stats.MemoryStats.Stats["cache"]
|
||||
mapped_file := stats.MemoryStats.Stats["mapped_file"]
|
||||
ms := &cstructs.MemoryStats{
|
||||
RSS: rss,
|
||||
Cache: cache,
|
||||
Swap: swap.Usage,
|
||||
MappedFile: mapped_file,
|
||||
Usage: stats.MemoryStats.Usage.Usage,
|
||||
MaxUsage: maxUsage,
|
||||
KernelUsage: stats.MemoryStats.KernelUsage.Usage,
|
||||
KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage,
|
||||
Measured: measuredMemStats,
|
||||
}
|
||||
|
||||
// CPU Related Stats
|
||||
totalProcessCPUUsage := float64(stats.CpuStats.CpuUsage.TotalUsage)
|
||||
userModeTime := float64(stats.CpuStats.CpuUsage.UsageInUsermode)
|
||||
kernelModeTime := float64(stats.CpuStats.CpuUsage.UsageInKernelmode)
|
||||
|
||||
totalPercent := l.totalCpuStats.Percent(totalProcessCPUUsage)
|
||||
cs := &cstructs.CpuStats{
|
||||
SystemMode: l.systemCpuStats.Percent(kernelModeTime),
|
||||
UserMode: l.userCpuStats.Percent(userModeTime),
|
||||
Percent: totalPercent,
|
||||
ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods,
|
||||
ThrottledTime: stats.CpuStats.ThrottlingData.ThrottledTime,
|
||||
TotalTicks: l.systemCpuStats.TicksConsumed(totalPercent),
|
||||
Measured: ExecutorCgroupMeasuredCpuStats,
|
||||
}
|
||||
taskResUsage := cstructs.TaskResourceUsage{
|
||||
ResourceUsage: &cstructs.ResourceUsage{
|
||||
MemoryStats: ms,
|
||||
CpuStats: cs,
|
||||
},
|
||||
Timestamp: ts.UTC().UnixNano(),
|
||||
Pids: pidStats,
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case ch <- &taskResUsage:
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Signal sends a signal to the process managed by the executor
|
||||
func (l *LibcontainerExecutor) Signal(s os.Signal) error {
|
||||
return l.userProc.Signal(s)
|
||||
}
|
||||
|
||||
// Exec starts an additional process inside the container
|
||||
func (l *LibcontainerExecutor) Exec(deadline time.Time, cmd string, args []string) ([]byte, int, error) {
|
||||
combined := append([]string{cmd}, args...)
|
||||
// Capture output
|
||||
buf, _ := circbuf.NewBuffer(int64(drivers.CheckBufSize))
|
||||
|
||||
process := &libcontainer.Process{
|
||||
Args: combined,
|
||||
Env: l.command.Env,
|
||||
Stdout: buf,
|
||||
Stderr: buf,
|
||||
}
|
||||
|
||||
err := l.container.Run(process)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
waitCh := make(chan *waitResult)
|
||||
defer close(waitCh)
|
||||
go l.handleExecWait(waitCh, process)
|
||||
|
||||
select {
|
||||
case result := <-waitCh:
|
||||
ps := result.ps
|
||||
if result.err != nil {
|
||||
if exitErr, ok := result.err.(*exec.ExitError); ok {
|
||||
ps = exitErr.ProcessState
|
||||
} else {
|
||||
return nil, 0, result.err
|
||||
}
|
||||
}
|
||||
var exitCode int
|
||||
if status, ok := ps.Sys().(syscall.WaitStatus); ok {
|
||||
exitCode = status.ExitStatus()
|
||||
}
|
||||
return buf.Bytes(), exitCode, nil
|
||||
|
||||
case <-time.After(time.Until(deadline)):
|
||||
process.Signal(os.Kill)
|
||||
return nil, 0, context.DeadlineExceeded
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (l *LibcontainerExecutor) newTerminalSocket() (pty func() (*os.File, error), tty *os.File, err error) {
|
||||
parent, child, err := lutils.NewSockPair("socket")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create terminal: %v", err)
|
||||
}
|
||||
|
||||
return func() (*os.File, error) { return lutils.RecvFd(parent) }, child, err
|
||||
|
||||
}
|
||||
|
||||
func (l *LibcontainerExecutor) ExecStreaming(ctx context.Context, cmd []string, tty bool,
|
||||
stream drivers.ExecTaskStream) error {
|
||||
|
||||
// the task process will be started by the container
|
||||
process := &libcontainer.Process{
|
||||
Args: cmd,
|
||||
Env: l.userProc.Env,
|
||||
User: l.userProc.User,
|
||||
Init: false,
|
||||
Cwd: "/",
|
||||
}
|
||||
|
||||
execHelper := &execHelper{
|
||||
logger: l.logger,
|
||||
|
||||
newTerminal: l.newTerminalSocket,
|
||||
setTTY: func(tty *os.File) error {
|
||||
process.ConsoleSocket = tty
|
||||
return nil
|
||||
},
|
||||
setIO: func(stdin io.Reader, stdout, stderr io.Writer) error {
|
||||
process.Stdin = stdin
|
||||
process.Stdout = stdout
|
||||
process.Stderr = stderr
|
||||
return nil
|
||||
},
|
||||
|
||||
processStart: func() error { return l.container.Run(process) },
|
||||
processWait: func() (*os.ProcessState, error) {
|
||||
return process.Wait()
|
||||
},
|
||||
}
|
||||
|
||||
return execHelper.run(ctx, tty, stream)
|
||||
|
||||
}
|
||||
|
||||
type waitResult struct {
|
||||
ps *os.ProcessState
|
||||
err error
|
||||
}
|
||||
|
||||
func (l *LibcontainerExecutor) handleExecWait(ch chan *waitResult, process *libcontainer.Process) {
|
||||
ps, err := process.Wait()
|
||||
ch <- &waitResult{ps, err}
|
||||
}
|
||||
|
||||
func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) {
|
||||
switch command.User {
|
||||
case "root":
|
||||
// when running as root, use the legacy set of system capabilities, so
|
||||
// that we do not break existing nomad clusters using this "feature"
|
||||
legacyCaps := capabilities.LegacySupported().Slice(true)
|
||||
cfg.Capabilities = &lconfigs.Capabilities{
|
||||
Bounding: legacyCaps,
|
||||
Permitted: legacyCaps,
|
||||
Effective: legacyCaps,
|
||||
Ambient: nil,
|
||||
Inheritable: nil,
|
||||
}
|
||||
default:
|
||||
// otherwise apply the plugin + task capability configuration
|
||||
cfg.Capabilities = &lconfigs.Capabilities{
|
||||
Bounding: command.Capabilities,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func configureNamespaces(pidMode, ipcMode string) lconfigs.Namespaces {
|
||||
namespaces := lconfigs.Namespaces{{Type: lconfigs.NEWNS}}
|
||||
if pidMode == IsolationModePrivate {
|
||||
namespaces = append(namespaces, lconfigs.Namespace{Type: lconfigs.NEWPID})
|
||||
}
|
||||
if ipcMode == IsolationModePrivate {
|
||||
namespaces = append(namespaces, lconfigs.Namespace{Type: lconfigs.NEWIPC})
|
||||
}
|
||||
return namespaces
|
||||
}
|
||||
|
||||
// configureIsolation prepares the isolation primitives of the container.
|
||||
// The process runs in a container configured with the following:
|
||||
//
|
||||
// * the task directory as the chroot
|
||||
// * dedicated mount points namespace, but shares the PID, User, domain, network namespaces with host
|
||||
// * small subset of devices (e.g. stdout/stderr/stdin, tty, shm, pts); default to using the same set of devices as Docker
|
||||
// * some special filesystems: `/proc`, `/sys`. Some case is given to avoid exec escaping or setting malicious values through them.
|
||||
func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) error {
|
||||
defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV
|
||||
|
||||
// set the new root directory for the container
|
||||
cfg.Rootfs = command.TaskDir
|
||||
|
||||
// disable pivot_root if set in the driver's configuration
|
||||
cfg.NoPivotRoot = command.NoPivotRoot
|
||||
|
||||
// set up default namespaces as configured
|
||||
cfg.Namespaces = configureNamespaces(command.ModePID, command.ModeIPC)
|
||||
|
||||
if command.NetworkIsolation != nil {
|
||||
cfg.Namespaces = append(cfg.Namespaces, lconfigs.Namespace{
|
||||
Type: lconfigs.NEWNET,
|
||||
Path: command.NetworkIsolation.Path,
|
||||
})
|
||||
}
|
||||
|
||||
// paths to mask using a bind mount to /dev/null to prevent reading
|
||||
cfg.MaskPaths = []string{
|
||||
"/proc/kcore",
|
||||
"/sys/firmware",
|
||||
}
|
||||
|
||||
// paths that should be remounted as readonly inside the container
|
||||
cfg.ReadonlyPaths = []string{
|
||||
"/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus",
|
||||
}
|
||||
|
||||
cfg.Devices = specconv.AllowedDevices
|
||||
if len(command.Devices) > 0 {
|
||||
devs, err := cmdDevices(command.Devices)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Devices = append(cfg.Devices, devs...)
|
||||
}
|
||||
|
||||
cfg.Mounts = []*lconfigs.Mount{
|
||||
{
|
||||
Source: "tmpfs",
|
||||
Destination: "/dev",
|
||||
Device: "tmpfs",
|
||||
Flags: syscall.MS_NOSUID | syscall.MS_STRICTATIME,
|
||||
Data: "mode=755",
|
||||
},
|
||||
{
|
||||
Source: "proc",
|
||||
Destination: "/proc",
|
||||
Device: "proc",
|
||||
Flags: defaultMountFlags,
|
||||
},
|
||||
{
|
||||
Source: "devpts",
|
||||
Destination: "/dev/pts",
|
||||
Device: "devpts",
|
||||
Flags: syscall.MS_NOSUID | syscall.MS_NOEXEC,
|
||||
Data: "newinstance,ptmxmode=0666,mode=0620,gid=5",
|
||||
},
|
||||
{
|
||||
Device: "tmpfs",
|
||||
Source: "shm",
|
||||
Destination: "/dev/shm",
|
||||
Data: "mode=1777,size=65536k",
|
||||
Flags: defaultMountFlags,
|
||||
},
|
||||
{
|
||||
Source: "mqueue",
|
||||
Destination: "/dev/mqueue",
|
||||
Device: "mqueue",
|
||||
Flags: defaultMountFlags,
|
||||
},
|
||||
{
|
||||
Source: "sysfs",
|
||||
Destination: "/sys",
|
||||
Device: "sysfs",
|
||||
Flags: defaultMountFlags | syscall.MS_RDONLY,
|
||||
},
|
||||
}
|
||||
|
||||
if len(command.Mounts) > 0 {
|
||||
cfg.Mounts = append(cfg.Mounts, cmdMounts(command.Mounts)...)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func configureCgroups(cfg *lconfigs.Config, command *ExecCommand) error {
|
||||
// If resources are not limited then manually create cgroups needed
|
||||
if !command.ResourceLimits {
|
||||
return cgutil.ConfigureBasicCgroups(cfg)
|
||||
}
|
||||
|
||||
// set cgroups path
|
||||
if cgutil.UseV2 {
|
||||
// in v2, the cgroup must have been created by the client already,
|
||||
// which breaks a lot of existing tests that run drivers without a client
|
||||
if command.Resources == nil || command.Resources.LinuxResources == nil || command.Resources.LinuxResources.CpusetCgroupPath == "" {
|
||||
return errors.New("cgroup path must be set")
|
||||
}
|
||||
parent, cgroup := cgutil.SplitPath(command.Resources.LinuxResources.CpusetCgroupPath)
|
||||
cfg.Cgroups.Path = filepath.Join("/", parent, cgroup)
|
||||
} else {
|
||||
// in v1, the cgroup is created using /nomad, which is a bug because it
|
||||
// does not respect the cgroup_parent client configuration
|
||||
// (but makes testing easy)
|
||||
id := uuid.Generate()
|
||||
cfg.Cgroups.Path = filepath.Join("/", cgutil.DefaultCgroupV1Parent, id)
|
||||
}
|
||||
|
||||
if command.Resources == nil || command.Resources.NomadResources == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Total amount of memory allowed to consume
|
||||
res := command.Resources.NomadResources
|
||||
memHard, memSoft := res.Memory.MemoryMaxMB, res.Memory.MemoryMB
|
||||
if memHard <= 0 {
|
||||
memHard = res.Memory.MemoryMB
|
||||
memSoft = 0
|
||||
}
|
||||
|
||||
if memHard > 0 {
|
||||
cfg.Cgroups.Resources.Memory = memHard * 1024 * 1024
|
||||
cfg.Cgroups.Resources.MemoryReservation = memSoft * 1024 * 1024
|
||||
|
||||
// Disable swap to avoid issues on the machine
|
||||
var memSwappiness uint64
|
||||
cfg.Cgroups.Resources.MemorySwappiness = &memSwappiness
|
||||
}
|
||||
|
||||
cpuShares := res.Cpu.CpuShares
|
||||
if cpuShares < 2 {
|
||||
return fmt.Errorf("resources.Cpu.CpuShares must be equal to or greater than 2: %v", cpuShares)
|
||||
}
|
||||
|
||||
// Set the relative CPU shares for this cgroup, and convert for cgroupv2
|
||||
cfg.Cgroups.Resources.CpuShares = uint64(cpuShares)
|
||||
cfg.Cgroups.Resources.CpuWeight = cgroups.ConvertCPUSharesToCgroupV2Value(uint64(cpuShares))
|
||||
|
||||
if command.Resources.LinuxResources != nil && command.Resources.LinuxResources.CpusetCgroupPath != "" {
|
||||
cfg.Hooks = lconfigs.Hooks{
|
||||
lconfigs.CreateRuntime: lconfigs.HookList{
|
||||
newSetCPUSetCgroupHook(command.Resources.LinuxResources.CpusetCgroupPath),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newLibcontainerConfig(command *ExecCommand) (*lconfigs.Config, error) {
|
||||
cfg := &lconfigs.Config{
|
||||
Cgroups: &lconfigs.Cgroup{
|
||||
Resources: &lconfigs.Resources{
|
||||
MemorySwappiness: nil,
|
||||
},
|
||||
},
|
||||
Version: "1.0.0",
|
||||
}
|
||||
|
||||
for _, device := range specconv.AllowedDevices {
|
||||
cfg.Cgroups.Resources.Devices = append(cfg.Cgroups.Resources.Devices, &device.Rule)
|
||||
}
|
||||
|
||||
configureCapabilities(cfg, command)
|
||||
|
||||
// children should not inherit Nomad agent oom_score_adj value
|
||||
oomScoreAdj := 0
|
||||
cfg.OomScoreAdj = &oomScoreAdj
|
||||
|
||||
if err := configureIsolation(cfg, command); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := configureCgroups(cfg, command); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// cmdDevices converts a list of driver.DeviceConfigs into excutor.Devices.
|
||||
func cmdDevices(driverDevices []*drivers.DeviceConfig) ([]*devices.Device, error) {
|
||||
if len(driverDevices) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
r := make([]*devices.Device, len(driverDevices))
|
||||
|
||||
for i, d := range driverDevices {
|
||||
ed, err := ldevices.DeviceFromPath(d.HostPath, d.Permissions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to make device out for %s: %v", d.HostPath, err)
|
||||
}
|
||||
ed.Path = d.TaskPath
|
||||
r[i] = ed
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
var userMountToUnixMount = map[string]int{
|
||||
// Empty string maps to `rprivate` for backwards compatibility in restored
|
||||
// older tasks, where mount propagation will not be present.
|
||||
"": unix.MS_PRIVATE | unix.MS_REC, // rprivate
|
||||
structs.VolumeMountPropagationPrivate: unix.MS_PRIVATE | unix.MS_REC, // rprivate
|
||||
structs.VolumeMountPropagationHostToTask: unix.MS_SLAVE | unix.MS_REC, // rslave
|
||||
structs.VolumeMountPropagationBidirectional: unix.MS_SHARED | unix.MS_REC, // rshared
|
||||
}
|
||||
|
||||
// cmdMounts converts a list of driver.MountConfigs into excutor.Mounts.
|
||||
func cmdMounts(mounts []*drivers.MountConfig) []*lconfigs.Mount {
|
||||
if len(mounts) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
r := make([]*lconfigs.Mount, len(mounts))
|
||||
|
||||
for i, m := range mounts {
|
||||
flags := unix.MS_BIND
|
||||
if m.Readonly {
|
||||
flags |= unix.MS_RDONLY
|
||||
}
|
||||
|
||||
r[i] = &lconfigs.Mount{
|
||||
Source: m.HostPath,
|
||||
Destination: m.TaskPath,
|
||||
Device: "bind",
|
||||
Flags: flags,
|
||||
PropagationFlags: []int{userMountToUnixMount[m.PropagationMode]},
|
||||
}
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// lookupTaskBin finds the file `bin`, searching in order:
|
||||
// - taskDir/local
|
||||
// - taskDir
|
||||
// - each mount, in order listed in the jobspec
|
||||
// - a PATH-like search of usr/local/bin/, usr/bin/, and bin/ inside the taskDir
|
||||
//
|
||||
// Returns an absolute path inside the container that will get passed as arg[0]
|
||||
// to the launched process, and the absolute path to that binary as seen by the
|
||||
// host (these will be identical for binaries that don't come from mounts).
|
||||
//
|
||||
// See also executor.lookupBin for a version used by non-isolated drivers.
|
||||
func lookupTaskBin(command *ExecCommand) (string, string, error) {
|
||||
taskDir := command.TaskDir
|
||||
bin := command.Cmd
|
||||
|
||||
// Check in the local directory
|
||||
localDir := filepath.Join(taskDir, allocdir.TaskLocal)
|
||||
taskPath, hostPath, err := getPathInTaskDir(command.TaskDir, localDir, bin)
|
||||
if err == nil {
|
||||
return taskPath, hostPath, nil
|
||||
}
|
||||
|
||||
// Check at the root of the task's directory
|
||||
taskPath, hostPath, err = getPathInTaskDir(command.TaskDir, command.TaskDir, bin)
|
||||
if err == nil {
|
||||
return taskPath, hostPath, nil
|
||||
}
|
||||
|
||||
// Check in our mounts
|
||||
for _, mount := range command.Mounts {
|
||||
taskPath, hostPath, err = getPathInMount(mount.HostPath, mount.TaskPath, bin)
|
||||
if err == nil {
|
||||
return taskPath, hostPath, nil
|
||||
}
|
||||
}
|
||||
|
||||
// If there's a / in the binary's path, we can't fallback to a PATH search
|
||||
if strings.Contains(bin, "/") {
|
||||
return "", "", fmt.Errorf("file %s not found under path %s", bin, taskDir)
|
||||
}
|
||||
|
||||
// look for a file using a PATH-style lookup inside the directory
|
||||
// root. Similar to the stdlib's exec.LookPath except:
|
||||
// - uses a restricted lookup PATH rather than the agent process's PATH env var.
|
||||
// - does not require that the file is already executable (this will be ensured
|
||||
// by the caller)
|
||||
// - does not prevent using relative path as added to exec.LookPath in go1.19
|
||||
// (this gets fixed-up in the caller)
|
||||
|
||||
// This is a fake PATH so that we're not using the agent's PATH
|
||||
restrictedPaths := []string{"/usr/local/bin", "/usr/bin", "/bin"}
|
||||
|
||||
for _, dir := range restrictedPaths {
|
||||
pathDir := filepath.Join(command.TaskDir, dir)
|
||||
taskPath, hostPath, err = getPathInTaskDir(command.TaskDir, pathDir, bin)
|
||||
if err == nil {
|
||||
return taskPath, hostPath, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", "", fmt.Errorf("file %s not found under path", bin)
|
||||
}
|
||||
|
||||
// getPathInTaskDir searches for the binary in the task directory and nested
|
||||
// search directory. It returns the absolute path rooted inside the container
|
||||
// and the absolute path on the host.
|
||||
func getPathInTaskDir(taskDir, searchDir, bin string) (string, string, error) {
|
||||
|
||||
hostPath := filepath.Join(searchDir, bin)
|
||||
err := filepathIsRegular(hostPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// Find the path relative to the task directory
|
||||
rel, err := filepath.Rel(taskDir, hostPath)
|
||||
if rel == "" || err != nil {
|
||||
return "", "", fmt.Errorf(
|
||||
"failed to determine relative path base=%q target=%q: %v",
|
||||
taskDir, hostPath, err)
|
||||
}
|
||||
|
||||
// Turn relative-to-taskdir path into re-rooted absolute path to avoid
|
||||
// libcontainer trying to resolve the binary using $PATH.
|
||||
// Do *not* use filepath.Join as it will translate ".."s returned by
|
||||
// filepath.Rel. Prepending "/" will cause the path to be rooted in the
|
||||
// chroot which is the desired behavior.
|
||||
return filepath.Clean("/" + rel), hostPath, nil
|
||||
}
|
||||
|
||||
// getPathInMount for the binary in the mount's host path, constructing the path
|
||||
// considering that the bin path is rooted in the mount's task path and not its
|
||||
// host path. It returns the absolute path rooted inside the container and the
|
||||
// absolute path on the host.
|
||||
func getPathInMount(mountHostPath, mountTaskPath, bin string) (string, string, error) {
|
||||
|
||||
// Find the path relative to the mount point in the task so that we can
|
||||
// trim off any shared prefix when we search on the host path
|
||||
mountRel, err := filepath.Rel(mountTaskPath, bin)
|
||||
if mountRel == "" || err != nil {
|
||||
return "", "", fmt.Errorf("path was not relative to the mount task path")
|
||||
}
|
||||
|
||||
hostPath := filepath.Join(mountHostPath, mountRel)
|
||||
|
||||
err = filepathIsRegular(hostPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// Turn relative-to-taskdir path into re-rooted absolute path to avoid
|
||||
// libcontainer trying to resolve the binary using $PATH.
|
||||
// Do *not* use filepath.Join as it will translate ".."s returned by
|
||||
// filepath.Rel. Prepending "/" will cause the path to be rooted in the
|
||||
// chroot which is the desired behavior.
|
||||
return filepath.Clean("/" + bin), hostPath, nil
|
||||
}
|
||||
|
||||
// filepathIsRegular verifies that a filepath is a regular file (i.e. not a
|
||||
// directory, socket, device, etc.)
|
||||
func filepathIsRegular(path string) error {
|
||||
f, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !f.Mode().Type().IsRegular() {
|
||||
return fmt.Errorf("path was not a regular file")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newSetCPUSetCgroupHook(cgroupPath string) lconfigs.Hook {
|
||||
return lconfigs.NewFunctionHook(func(state *specs.State) error {
|
||||
return cgroups.WriteCgroupProc(cgroupPath, state.Pid)
|
||||
})
|
||||
}
|
||||
34
executor/executor_plugin.go
Normal file
34
executor/executor_plugin.go
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
"github.com/hashicorp/nomad/drivers/shared/executor/proto"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type ExecutorPlugin struct {
|
||||
// TODO: support backwards compatibility with pre 0.9 NetRPC plugin
|
||||
plugin.NetRPCUnsupportedPlugin
|
||||
logger hclog.Logger
|
||||
fsIsolation bool
|
||||
}
|
||||
|
||||
func (p *ExecutorPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error {
|
||||
if p.fsIsolation {
|
||||
proto.RegisterExecutorServer(s, &grpcExecutorServer{impl: NewExecutorWithIsolation(p.logger)})
|
||||
} else {
|
||||
proto.RegisterExecutorServer(s, &grpcExecutorServer{impl: NewExecutor(p.logger)})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *ExecutorPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) {
|
||||
return &grpcExecutorClient{
|
||||
client: proto.NewExecutorClient(c),
|
||||
doneCtx: ctx,
|
||||
logger: p.logger,
|
||||
}, nil
|
||||
}
|
||||
154
executor/executor_universal_linux.go
Normal file
154
executor/executor_universal_linux.go
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
"github.com/hashicorp/nomad/client/lib/cgutil"
|
||||
"github.com/hashicorp/nomad/client/lib/resources"
|
||||
"github.com/hashicorp/nomad/client/taskenv"
|
||||
"github.com/hashicorp/nomad/helper/users"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
"github.com/opencontainers/runc/libcontainer/configs"
|
||||
"github.com/opencontainers/runc/libcontainer/specconv"
|
||||
)
|
||||
|
||||
// setCmdUser takes a user id as a string and looks up the user, and sets the command
|
||||
// to execute as that user.
|
||||
func setCmdUser(cmd *exec.Cmd, userid string) error {
|
||||
u, err := users.Lookup(userid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to identify user %v: %v", userid, err)
|
||||
}
|
||||
|
||||
// Get the groups the user is a part of
|
||||
gidStrings, err := u.GroupIds()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to lookup user's group membership: %v", err)
|
||||
}
|
||||
|
||||
gids := make([]uint32, len(gidStrings))
|
||||
for _, gidString := range gidStrings {
|
||||
u, err := strconv.ParseUint(gidString, 10, 32)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert user's group to uint32 %s: %v", gidString, err)
|
||||
}
|
||||
|
||||
gids = append(gids, uint32(u))
|
||||
}
|
||||
|
||||
// Convert the uid and gid
|
||||
uid, err := strconv.ParseUint(u.Uid, 10, 32)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert userid to uint32: %s", err)
|
||||
}
|
||||
gid, err := strconv.ParseUint(u.Gid, 10, 32)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert groupid to uint32: %s", err)
|
||||
}
|
||||
|
||||
// Set the command to run as that user and group.
|
||||
if cmd.SysProcAttr == nil {
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{}
|
||||
}
|
||||
if cmd.SysProcAttr.Credential == nil {
|
||||
cmd.SysProcAttr.Credential = &syscall.Credential{}
|
||||
}
|
||||
cmd.SysProcAttr.Credential.Uid = uint32(uid)
|
||||
cmd.SysProcAttr.Credential.Gid = uint32(gid)
|
||||
cmd.SysProcAttr.Credential.Groups = gids
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// configureResourceContainer configured the cgroups to be used to track pids
|
||||
// created by the executor
|
||||
func (e *UniversalExecutor) configureResourceContainer(pid int) error {
|
||||
cfg := &configs.Config{
|
||||
Cgroups: &configs.Cgroup{
|
||||
Resources: &configs.Resources{},
|
||||
},
|
||||
}
|
||||
|
||||
// note: this was always here, but not used until cgroups v2 support
|
||||
for _, device := range specconv.AllowedDevices {
|
||||
cfg.Cgroups.Resources.Devices = append(cfg.Cgroups.Resources.Devices, &device.Rule)
|
||||
}
|
||||
|
||||
lookup := func(env []string, name string) (result string) {
|
||||
for _, s := range env {
|
||||
if strings.HasPrefix(s, name+"=") {
|
||||
result = strings.TrimLeft(s, name+"=")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if cgutil.UseV2 {
|
||||
// in v2 we have the definitive cgroup; create and enter it
|
||||
|
||||
// use the task environment variables for determining the cgroup path -
|
||||
// not ideal but plumbing the values directly requires grpc protobuf changes
|
||||
parent := lookup(e.commandCfg.Env, taskenv.CgroupParent)
|
||||
allocID := lookup(e.commandCfg.Env, taskenv.AllocID)
|
||||
task := lookup(e.commandCfg.Env, taskenv.TaskName)
|
||||
if parent == "" || allocID == "" || task == "" {
|
||||
return fmt.Errorf(
|
||||
"environment variables %s must be set",
|
||||
strings.Join([]string{taskenv.CgroupParent, taskenv.AllocID, taskenv.TaskName}, ","),
|
||||
)
|
||||
}
|
||||
scope := cgutil.CgroupScope(allocID, task)
|
||||
path := filepath.Join("/", cgutil.GetCgroupParent(parent), scope)
|
||||
cfg.Cgroups.Path = path
|
||||
e.containment = resources.Contain(e.logger, cfg.Cgroups)
|
||||
return e.containment.Apply(pid)
|
||||
|
||||
} else {
|
||||
// in v1 create a freezer cgroup for use by containment
|
||||
|
||||
if err := cgutil.ConfigureBasicCgroups(cfg); err != nil {
|
||||
// Log this error to help diagnose cases where nomad is run with too few
|
||||
// permissions, but don't return an error. There is no separate check for
|
||||
// cgroup creation permissions, so this may be the happy path.
|
||||
e.logger.Warn("failed to create cgroup",
|
||||
"docs", "https://www.nomadproject.io/docs/drivers/raw_exec.html#no_cgroups",
|
||||
"error", err)
|
||||
return nil
|
||||
}
|
||||
path := cfg.Cgroups.Path
|
||||
e.logger.Trace("cgroup created, now need to apply", "path", path)
|
||||
e.containment = resources.Contain(e.logger, cfg.Cgroups)
|
||||
return e.containment.Apply(pid)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) getAllPids() (resources.PIDs, error) {
|
||||
if e.containment == nil {
|
||||
return getAllPidsByScanning()
|
||||
}
|
||||
return e.containment.GetPIDs(), nil
|
||||
}
|
||||
|
||||
// withNetworkIsolation calls the passed function the network namespace `spec`
|
||||
func withNetworkIsolation(f func() error, spec *drivers.NetworkIsolationSpec) error {
|
||||
if spec != nil && spec.Path != "" {
|
||||
// Get a handle to the target network namespace
|
||||
netNS, err := ns.GetNS(spec.Path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Start the container in the network namespace
|
||||
return netNS.Do(func(ns.NetNS) error {
|
||||
return f()
|
||||
})
|
||||
}
|
||||
return f()
|
||||
}
|
||||
50
executor/executor_unix.go
Normal file
50
executor/executor_unix.go
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
|
||||
|
||||
package executor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// configure new process group for child process
|
||||
func (e *UniversalExecutor) setNewProcessGroup() error {
|
||||
if e.childCmd.SysProcAttr == nil {
|
||||
e.childCmd.SysProcAttr = &syscall.SysProcAttr{}
|
||||
}
|
||||
e.childCmd.SysProcAttr.Setpgid = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SIGKILL the process group starting at process.Pid
|
||||
func (e *UniversalExecutor) killProcessTree(process *os.Process) error {
|
||||
pid := process.Pid
|
||||
negative := -pid // tells unix to kill entire process group
|
||||
signal := syscall.SIGKILL
|
||||
|
||||
// If new process group was created upon command execution
|
||||
// we can kill the whole process group now to cleanup any leftovers.
|
||||
if e.childCmd.SysProcAttr != nil && e.childCmd.SysProcAttr.Setpgid {
|
||||
e.logger.Trace("sending sigkill to process group", "pid", pid, "negative", negative, "signal", signal)
|
||||
if err := syscall.Kill(negative, signal); err != nil && err.Error() != noSuchProcessErr {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return process.Kill()
|
||||
}
|
||||
|
||||
// Only send the process a shutdown signal (default INT), doesn't
|
||||
// necessarily kill it.
|
||||
func (e *UniversalExecutor) shutdownProcess(sig os.Signal, proc *os.Process) error {
|
||||
if sig == nil {
|
||||
sig = os.Interrupt
|
||||
}
|
||||
|
||||
if err := proc.Signal(sig); err != nil && err.Error() != finishedErr {
|
||||
return fmt.Errorf("executor shutdown error: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
267
executor/grpc_client.go
Normal file
267
executor/grpc_client.go
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/LK4D4/joincontext"
|
||||
"github.com/golang/protobuf/ptypes"
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
cstructs "github.com/hashicorp/nomad/client/structs"
|
||||
"github.com/hashicorp/nomad/drivers/shared/executor/proto"
|
||||
"github.com/hashicorp/nomad/helper/pluginutils/grpcutils"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
dproto "github.com/hashicorp/nomad/plugins/drivers/proto"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
var _ Executor = (*grpcExecutorClient)(nil)
|
||||
|
||||
type grpcExecutorClient struct {
|
||||
client proto.ExecutorClient
|
||||
logger hclog.Logger
|
||||
|
||||
// doneCtx is close when the plugin exits
|
||||
doneCtx context.Context
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) Launch(cmd *ExecCommand) (*ProcessState, error) {
|
||||
ctx := context.Background()
|
||||
req := &proto.LaunchRequest{
|
||||
Cmd: cmd.Cmd,
|
||||
Args: cmd.Args,
|
||||
Resources: drivers.ResourcesToProto(cmd.Resources),
|
||||
StdoutPath: cmd.StdoutPath,
|
||||
StderrPath: cmd.StderrPath,
|
||||
Env: cmd.Env,
|
||||
User: cmd.User,
|
||||
TaskDir: cmd.TaskDir,
|
||||
ResourceLimits: cmd.ResourceLimits,
|
||||
BasicProcessCgroup: cmd.BasicProcessCgroup,
|
||||
NoPivotRoot: cmd.NoPivotRoot,
|
||||
Mounts: drivers.MountsToProto(cmd.Mounts),
|
||||
Devices: drivers.DevicesToProto(cmd.Devices),
|
||||
NetworkIsolation: drivers.NetworkIsolationSpecToProto(cmd.NetworkIsolation),
|
||||
DefaultPidMode: cmd.ModePID,
|
||||
DefaultIpcMode: cmd.ModeIPC,
|
||||
Capabilities: cmd.Capabilities,
|
||||
}
|
||||
resp, err := c.client.Launch(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ps, err := processStateFromProto(resp.Process)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ps, nil
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) Wait(ctx context.Context) (*ProcessState, error) {
|
||||
// Join the passed context and the shutdown context
|
||||
ctx, _ = joincontext.Join(ctx, c.doneCtx)
|
||||
|
||||
resp, err := c.client.Wait(ctx, &proto.WaitRequest{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ps, err := processStateFromProto(resp.Process)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ps, nil
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) Shutdown(signal string, gracePeriod time.Duration) error {
|
||||
ctx := context.Background()
|
||||
req := &proto.ShutdownRequest{
|
||||
Signal: signal,
|
||||
GracePeriod: gracePeriod.Nanoseconds(),
|
||||
}
|
||||
if _, err := c.client.Shutdown(ctx, req); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) UpdateResources(r *drivers.Resources) error {
|
||||
ctx := context.Background()
|
||||
req := &proto.UpdateResourcesRequest{Resources: drivers.ResourcesToProto(r)}
|
||||
if _, err := c.client.UpdateResources(ctx, req); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) Version() (*ExecutorVersion, error) {
|
||||
ctx := context.Background()
|
||||
resp, err := c.client.Version(ctx, &proto.VersionRequest{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ExecutorVersion{Version: resp.Version}, nil
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) Stats(ctx context.Context, interval time.Duration) (<-chan *cstructs.TaskResourceUsage, error) {
|
||||
stream, err := c.client.Stats(ctx, &proto.StatsRequest{
|
||||
Interval: int64(interval),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ch := make(chan *cstructs.TaskResourceUsage)
|
||||
go c.handleStats(ctx, stream, ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) handleStats(ctx context.Context, stream proto.Executor_StatsClient, ch chan<- *cstructs.TaskResourceUsage) {
|
||||
defer close(ch)
|
||||
for {
|
||||
resp, err := stream.Recv()
|
||||
if ctx.Err() != nil {
|
||||
// Context canceled; exit gracefully
|
||||
return
|
||||
}
|
||||
|
||||
if err == io.EOF ||
|
||||
status.Code(err) == codes.Unavailable ||
|
||||
status.Code(err) == codes.Canceled ||
|
||||
err == context.Canceled {
|
||||
c.logger.Trace("executor Stats stream closed", "msg", err)
|
||||
return
|
||||
} else if err != nil {
|
||||
c.logger.Warn("failed to receive Stats executor RPC stream, closing stream", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
stats, err := drivers.TaskStatsFromProto(resp.Stats)
|
||||
if err != nil {
|
||||
c.logger.Error("failed to decode stats from RPC", "error", err, "stats", resp.Stats)
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- stats:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) Signal(s os.Signal) error {
|
||||
ctx := context.Background()
|
||||
sig, ok := s.(syscall.Signal)
|
||||
if !ok {
|
||||
return fmt.Errorf("unsupported signal type: %q", s.String())
|
||||
}
|
||||
req := &proto.SignalRequest{
|
||||
Signal: int32(sig),
|
||||
}
|
||||
if _, err := c.client.Signal(ctx, req); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) Exec(deadline time.Time, cmd string, args []string) ([]byte, int, error) {
|
||||
ctx := context.Background()
|
||||
pbDeadline, err := ptypes.TimestampProto(deadline)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
req := &proto.ExecRequest{
|
||||
Deadline: pbDeadline,
|
||||
Cmd: cmd,
|
||||
Args: args,
|
||||
}
|
||||
|
||||
resp, err := c.client.Exec(ctx, req)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return resp.Output, int(resp.ExitCode), nil
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) ExecStreaming(ctx context.Context,
|
||||
command []string,
|
||||
tty bool,
|
||||
execStream drivers.ExecTaskStream) error {
|
||||
|
||||
err := c.execStreaming(ctx, command, tty, execStream)
|
||||
if err != nil {
|
||||
return grpcutils.HandleGrpcErr(err, c.doneCtx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *grpcExecutorClient) execStreaming(ctx context.Context,
|
||||
command []string,
|
||||
tty bool,
|
||||
execStream drivers.ExecTaskStream) error {
|
||||
|
||||
stream, err := c.client.ExecStreaming(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = stream.Send(&dproto.ExecTaskStreamingRequest{
|
||||
Setup: &dproto.ExecTaskStreamingRequest_Setup{
|
||||
Command: command,
|
||||
Tty: tty,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
for {
|
||||
m, err := execStream.Recv()
|
||||
if err == io.EOF {
|
||||
return
|
||||
} else if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
if err := stream.Send(m); err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case err := <-errCh:
|
||||
return err
|
||||
default:
|
||||
}
|
||||
|
||||
m, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := execStream.Send(m); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
178
executor/grpc_server.go
Normal file
178
executor/grpc_server.go
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/golang/protobuf/ptypes"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/hashicorp/nomad/drivers/shared/executor/proto"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
sproto "github.com/hashicorp/nomad/plugins/shared/structs/proto"
|
||||
)
|
||||
|
||||
type grpcExecutorServer struct {
|
||||
impl Executor
|
||||
}
|
||||
|
||||
func (s *grpcExecutorServer) Launch(ctx context.Context, req *proto.LaunchRequest) (*proto.LaunchResponse, error) {
|
||||
ps, err := s.impl.Launch(&ExecCommand{
|
||||
Cmd: req.Cmd,
|
||||
Args: req.Args,
|
||||
Resources: drivers.ResourcesFromProto(req.Resources),
|
||||
StdoutPath: req.StdoutPath,
|
||||
StderrPath: req.StderrPath,
|
||||
Env: req.Env,
|
||||
User: req.User,
|
||||
TaskDir: req.TaskDir,
|
||||
ResourceLimits: req.ResourceLimits,
|
||||
BasicProcessCgroup: req.BasicProcessCgroup,
|
||||
NoPivotRoot: req.NoPivotRoot,
|
||||
Mounts: drivers.MountsFromProto(req.Mounts),
|
||||
Devices: drivers.DevicesFromProto(req.Devices),
|
||||
NetworkIsolation: drivers.NetworkIsolationSpecFromProto(req.NetworkIsolation),
|
||||
ModePID: req.DefaultPidMode,
|
||||
ModeIPC: req.DefaultIpcMode,
|
||||
Capabilities: req.Capabilities,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
process, err := processStateToProto(ps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &proto.LaunchResponse{
|
||||
Process: process,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *grpcExecutorServer) Wait(ctx context.Context, req *proto.WaitRequest) (*proto.WaitResponse, error) {
|
||||
ps, err := s.impl.Wait(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
process, err := processStateToProto(ps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &proto.WaitResponse{
|
||||
Process: process,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *grpcExecutorServer) Shutdown(ctx context.Context, req *proto.ShutdownRequest) (*proto.ShutdownResponse, error) {
|
||||
if err := s.impl.Shutdown(req.Signal, time.Duration(req.GracePeriod)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &proto.ShutdownResponse{}, nil
|
||||
}
|
||||
|
||||
func (s *grpcExecutorServer) UpdateResources(ctx context.Context, req *proto.UpdateResourcesRequest) (*proto.UpdateResourcesResponse, error) {
|
||||
if err := s.impl.UpdateResources(drivers.ResourcesFromProto(req.Resources)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &proto.UpdateResourcesResponse{}, nil
|
||||
}
|
||||
|
||||
func (s *grpcExecutorServer) Version(context.Context, *proto.VersionRequest) (*proto.VersionResponse, error) {
|
||||
v, err := s.impl.Version()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &proto.VersionResponse{
|
||||
Version: v.Version,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *grpcExecutorServer) Stats(req *proto.StatsRequest, stream proto.Executor_StatsServer) error {
|
||||
interval := time.Duration(req.Interval)
|
||||
if interval == 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
|
||||
outCh, err := s.impl.Stats(stream.Context(), interval)
|
||||
if err != nil {
|
||||
if rec, ok := err.(structs.Recoverable); ok {
|
||||
st := status.New(codes.FailedPrecondition, rec.Error())
|
||||
st, err := st.WithDetails(&sproto.RecoverableError{Recoverable: rec.IsRecoverable()})
|
||||
if err != nil {
|
||||
// If this error, it will always error
|
||||
panic(err)
|
||||
}
|
||||
return st.Err()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
for resp := range outCh {
|
||||
pbStats, err := drivers.TaskStatsToProto(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
presp := &proto.StatsResponse{
|
||||
Stats: pbStats,
|
||||
}
|
||||
|
||||
// Send the stats
|
||||
if err := stream.Send(presp); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *grpcExecutorServer) Signal(ctx context.Context, req *proto.SignalRequest) (*proto.SignalResponse, error) {
|
||||
sig := syscall.Signal(req.Signal)
|
||||
if err := s.impl.Signal(sig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &proto.SignalResponse{}, nil
|
||||
}
|
||||
|
||||
func (s *grpcExecutorServer) Exec(ctx context.Context, req *proto.ExecRequest) (*proto.ExecResponse, error) {
|
||||
deadline, err := ptypes.Timestamp(req.Deadline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out, exit, err := s.impl.Exec(deadline, req.Cmd, req.Args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &proto.ExecResponse{
|
||||
Output: out,
|
||||
ExitCode: int32(exit),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *grpcExecutorServer) ExecStreaming(server proto.Executor_ExecStreamingServer) error {
|
||||
msg, err := server.Recv()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to receive initial message: %v", err)
|
||||
}
|
||||
|
||||
if msg.Setup == nil {
|
||||
return fmt.Errorf("first message should always be setup")
|
||||
}
|
||||
|
||||
return s.impl.ExecStreaming(server.Context(),
|
||||
msg.Setup.Command, msg.Setup.Tty,
|
||||
server)
|
||||
}
|
||||
29
executor/libcontainer_nsenter_linux.go
Normal file
29
executor/libcontainer_nsenter_linux.go
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"os"
|
||||
"runtime"
|
||||
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
"github.com/opencontainers/runc/libcontainer"
|
||||
_ "github.com/opencontainers/runc/libcontainer/nsenter"
|
||||
)
|
||||
|
||||
// init is only run on linux and is used when the LibcontainerExecutor starts
|
||||
// a new process. The libcontainer shim takes over the process, setting up the
|
||||
// configured isolation and limitions before execve into the user process
|
||||
//
|
||||
// This subcommand handler is implemented as an `init`, libcontainer shim is handled anywhere
|
||||
// this package is used (including tests) without needing to write special command handler.
|
||||
func init() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "libcontainer-shim" {
|
||||
runtime.GOMAXPROCS(1)
|
||||
runtime.LockOSThread()
|
||||
factory, _ := libcontainer.New("")
|
||||
if err := factory.StartInitialization(); err != nil {
|
||||
hclog.L().Error("failed to initialize libcontainer-shim", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
panic("--this line should have never been executed, congratulations--")
|
||||
}
|
||||
}
|
||||
211
executor/pid_collector.go
Normal file
211
executor/pid_collector.go
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/nomad/client/lib/resources"
|
||||
"github.com/hashicorp/nomad/client/stats"
|
||||
"github.com/hashicorp/nomad/plugins/drivers"
|
||||
ps "github.com/mitchellh/go-ps"
|
||||
"github.com/shirou/gopsutil/v3/process"
|
||||
)
|
||||
|
||||
var (
|
||||
// pidScanInterval is the interval at which the executor scans the process
|
||||
// tree for finding out the pids that the executor and it's child processes
|
||||
// have forked
|
||||
pidScanInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
// pidCollector is a utility that can be embedded in an executor to collect pid
|
||||
// stats
|
||||
type pidCollector struct {
|
||||
pids map[int]*resources.PID
|
||||
pidLock sync.RWMutex
|
||||
logger hclog.Logger
|
||||
}
|
||||
|
||||
// allPidGetter is a func which is used by the pid collector to gather
|
||||
// stats on
|
||||
type allPidGetter func() (resources.PIDs, error)
|
||||
|
||||
func newPidCollector(logger hclog.Logger) *pidCollector {
|
||||
return &pidCollector{
|
||||
pids: make(map[int]*resources.PID),
|
||||
logger: logger.Named("pid_collector"),
|
||||
}
|
||||
}
|
||||
|
||||
// collectPids collects the pids of the child processes that the executor is
|
||||
// running every 5 seconds
|
||||
func (c *pidCollector) collectPids(stopCh chan interface{}, pidGetter allPidGetter) {
|
||||
// Fire the timer right away when the executor starts from there on the pids
|
||||
// are collected every scan interval
|
||||
timer := time.NewTimer(0)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-timer.C:
|
||||
pids, err := pidGetter()
|
||||
if err != nil {
|
||||
c.logger.Debug("error collecting pids", "error", err)
|
||||
}
|
||||
c.pidLock.Lock()
|
||||
|
||||
// Adding pids which are not being tracked
|
||||
for pid, np := range pids {
|
||||
if _, ok := c.pids[pid]; !ok {
|
||||
c.pids[pid] = np
|
||||
}
|
||||
}
|
||||
// Removing pids which are no longer present
|
||||
for pid := range c.pids {
|
||||
if _, ok := pids[pid]; !ok {
|
||||
delete(c.pids, pid)
|
||||
}
|
||||
}
|
||||
c.pidLock.Unlock()
|
||||
timer.Reset(pidScanInterval)
|
||||
case <-stopCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// scanPids scans all the pids on the machine running the current executor and
|
||||
// returns the child processes of the executor.
|
||||
func scanPids(parentPid int, allPids []ps.Process) (map[int]*resources.PID, error) {
|
||||
processFamily := make(map[int]struct{})
|
||||
processFamily[parentPid] = struct{}{}
|
||||
|
||||
// A mapping of pids to their parent pids. It is used to build the process
|
||||
// tree of the executing task
|
||||
pidsRemaining := make(map[int]int, len(allPids))
|
||||
for _, pid := range allPids {
|
||||
pidsRemaining[pid.Pid()] = pid.PPid()
|
||||
}
|
||||
|
||||
for {
|
||||
// flag to indicate if we have found a match
|
||||
foundNewPid := false
|
||||
|
||||
for pid, ppid := range pidsRemaining {
|
||||
_, childPid := processFamily[ppid]
|
||||
|
||||
// checking if the pid is a child of any of the parents
|
||||
if childPid {
|
||||
processFamily[pid] = struct{}{}
|
||||
delete(pidsRemaining, pid)
|
||||
foundNewPid = true
|
||||
}
|
||||
}
|
||||
|
||||
// not scanning anymore if we couldn't find a single match
|
||||
if !foundNewPid {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
res := make(map[int]*resources.PID)
|
||||
for pid := range processFamily {
|
||||
res[pid] = &resources.PID{
|
||||
PID: pid,
|
||||
StatsTotalCPU: stats.NewCpuStats(),
|
||||
StatsUserCPU: stats.NewCpuStats(),
|
||||
StatsSysCPU: stats.NewCpuStats(),
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// pidStats returns the resource usage stats per pid
|
||||
func (c *pidCollector) pidStats() (map[string]*drivers.ResourceUsage, error) {
|
||||
stats := make(map[string]*drivers.ResourceUsage)
|
||||
c.pidLock.RLock()
|
||||
pids := make(map[int]*resources.PID, len(c.pids))
|
||||
for k, v := range c.pids {
|
||||
pids[k] = v
|
||||
}
|
||||
c.pidLock.RUnlock()
|
||||
for pid, np := range pids {
|
||||
p, err := process.NewProcess(int32(pid))
|
||||
if err != nil {
|
||||
c.logger.Trace("unable to create new process", "pid", pid, "error", err)
|
||||
continue
|
||||
}
|
||||
ms := &drivers.MemoryStats{}
|
||||
if memInfo, err := p.MemoryInfo(); err == nil {
|
||||
ms.RSS = memInfo.RSS
|
||||
ms.Swap = memInfo.Swap
|
||||
ms.Measured = ExecutorBasicMeasuredMemStats
|
||||
}
|
||||
|
||||
cs := &drivers.CpuStats{}
|
||||
if cpuStats, err := p.Times(); err == nil {
|
||||
cs.SystemMode = np.StatsSysCPU.Percent(cpuStats.System * float64(time.Second))
|
||||
cs.UserMode = np.StatsUserCPU.Percent(cpuStats.User * float64(time.Second))
|
||||
cs.Measured = ExecutorBasicMeasuredCpuStats
|
||||
|
||||
// calculate cpu usage percent
|
||||
cs.Percent = np.StatsTotalCPU.Percent(cpuStats.Total() * float64(time.Second))
|
||||
}
|
||||
stats[strconv.Itoa(pid)] = &drivers.ResourceUsage{MemoryStats: ms, CpuStats: cs}
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// aggregatedResourceUsage aggregates the resource usage of all the pids and
|
||||
// returns a TaskResourceUsage data point
|
||||
func aggregatedResourceUsage(systemCpuStats *stats.CpuStats, pidStats map[string]*drivers.ResourceUsage) *drivers.TaskResourceUsage {
|
||||
ts := time.Now().UTC().UnixNano()
|
||||
var (
|
||||
systemModeCPU, userModeCPU, percent float64
|
||||
totalRSS, totalSwap uint64
|
||||
)
|
||||
|
||||
for _, pidStat := range pidStats {
|
||||
systemModeCPU += pidStat.CpuStats.SystemMode
|
||||
userModeCPU += pidStat.CpuStats.UserMode
|
||||
percent += pidStat.CpuStats.Percent
|
||||
|
||||
totalRSS += pidStat.MemoryStats.RSS
|
||||
totalSwap += pidStat.MemoryStats.Swap
|
||||
}
|
||||
|
||||
totalCPU := &drivers.CpuStats{
|
||||
SystemMode: systemModeCPU,
|
||||
UserMode: userModeCPU,
|
||||
Percent: percent,
|
||||
Measured: ExecutorBasicMeasuredCpuStats,
|
||||
TotalTicks: systemCpuStats.TicksConsumed(percent),
|
||||
}
|
||||
|
||||
totalMemory := &drivers.MemoryStats{
|
||||
RSS: totalRSS,
|
||||
Swap: totalSwap,
|
||||
Measured: ExecutorBasicMeasuredMemStats,
|
||||
}
|
||||
|
||||
resourceUsage := drivers.ResourceUsage{
|
||||
MemoryStats: totalMemory,
|
||||
CpuStats: totalCPU,
|
||||
}
|
||||
return &drivers.TaskResourceUsage{
|
||||
ResourceUsage: &resourceUsage,
|
||||
Timestamp: ts,
|
||||
Pids: pidStats,
|
||||
}
|
||||
}
|
||||
|
||||
func getAllPidsByScanning() (resources.PIDs, error) {
|
||||
allProcesses, err := ps.Processes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanPids(os.Getpid(), allProcesses)
|
||||
}
|
||||
55
executor/plugins.go
Normal file
55
executor/plugins.go
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"net"
|
||||
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
)
|
||||
|
||||
// ExecutorConfig is the config that Nomad passes to the executor
|
||||
type ExecutorConfig struct {
|
||||
|
||||
// LogFile is the file to which Executor logs
|
||||
LogFile string
|
||||
|
||||
// LogLevel is the level of the logs to putout
|
||||
LogLevel string
|
||||
|
||||
// FSIsolation if set will use an executor implementation that support
|
||||
// filesystem isolation
|
||||
FSIsolation bool
|
||||
}
|
||||
|
||||
func GetPluginMap(logger hclog.Logger, fsIsolation bool) map[string]plugin.Plugin {
|
||||
return map[string]plugin.Plugin{
|
||||
"executor": &ExecutorPlugin{
|
||||
logger: logger,
|
||||
fsIsolation: fsIsolation,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ExecutorReattachConfig is the config that we serialize and de-serialize and
|
||||
// store in disk
|
||||
type PluginReattachConfig struct {
|
||||
Pid int
|
||||
AddrNet string
|
||||
AddrName string
|
||||
}
|
||||
|
||||
// PluginConfig returns a config from an ExecutorReattachConfig
|
||||
func (c *PluginReattachConfig) PluginConfig() *plugin.ReattachConfig {
|
||||
var addr net.Addr
|
||||
switch c.AddrNet {
|
||||
case "unix", "unixgram", "unixpacket":
|
||||
addr, _ = net.ResolveUnixAddr(c.AddrNet, c.AddrName)
|
||||
case "tcp", "tcp4", "tcp6":
|
||||
addr, _ = net.ResolveTCPAddr(c.AddrNet, c.AddrName)
|
||||
}
|
||||
return &plugin.ReattachConfig{Pid: c.Pid, Addr: addr}
|
||||
}
|
||||
|
||||
func NewPluginReattachConfig(c *plugin.ReattachConfig) *PluginReattachConfig {
|
||||
return &PluginReattachConfig{Pid: c.Pid, AddrNet: c.Addr.Network(), AddrName: c.Addr.String()}
|
||||
}
|
||||
43
executor/pty_unix.go
Normal file
43
executor/pty_unix.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
|
||||
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
|
||||
|
||||
package executor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/creack/pty"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func sessionCmdAttr(tty *os.File) *syscall.SysProcAttr {
|
||||
return &syscall.SysProcAttr{
|
||||
Setsid: true,
|
||||
Setctty: true,
|
||||
}
|
||||
}
|
||||
|
||||
func setTTYSize(w io.Writer, height, width int32) error {
|
||||
f, ok := w.(*os.File)
|
||||
if !ok {
|
||||
return fmt.Errorf("attempted to resize a non-tty session")
|
||||
}
|
||||
|
||||
return pty.Setsize(f, &pty.Winsize{
|
||||
Rows: uint16(height),
|
||||
Cols: uint16(width),
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func isUnixEIOErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.Contains(err.Error(), unix.EIO.Error())
|
||||
}
|
||||
12
executor/resource_container_default.go
Normal file
12
executor/resource_container_default.go
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
//go:build !linux
|
||||
|
||||
package executor
|
||||
|
||||
// resourceContainerContext is a platform-specific struct for managing a
|
||||
// resource container.
|
||||
type resourceContainerContext struct {
|
||||
}
|
||||
|
||||
func (rc *resourceContainerContext) executorCleanup() error {
|
||||
return nil
|
||||
}
|
||||
138
executor/utils.go
Normal file
138
executor/utils.go
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"github.com/golang/protobuf/ptypes"
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
"github.com/hashicorp/nomad/drivers/shared/executor/proto"
|
||||
"github.com/hashicorp/nomad/plugins/base"
|
||||
)
|
||||
|
||||
const (
|
||||
// ExecutorDefaultMaxPort is the default max port used by the executor for
|
||||
// searching for an available port
|
||||
ExecutorDefaultMaxPort = 14512
|
||||
|
||||
// ExecutorDefaultMinPort is the default min port used by the executor for
|
||||
// searching for an available port
|
||||
ExecutorDefaultMinPort = 14000
|
||||
)
|
||||
|
||||
// CreateExecutor launches an executor plugin and returns an instance of the
|
||||
// Executor interface
|
||||
func CreateExecutor(logger hclog.Logger, driverConfig *base.ClientDriverConfig,
|
||||
executorConfig *ExecutorConfig) (Executor, *plugin.Client, error) {
|
||||
|
||||
c, err := json.Marshal(executorConfig)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to create executor config: %v", err)
|
||||
}
|
||||
bin, err := os.Executable()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to find the nomad binary: %v", err)
|
||||
}
|
||||
|
||||
p := &ExecutorPlugin{
|
||||
logger: logger,
|
||||
fsIsolation: executorConfig.FSIsolation,
|
||||
}
|
||||
|
||||
config := &plugin.ClientConfig{
|
||||
HandshakeConfig: base.Handshake,
|
||||
Plugins: map[string]plugin.Plugin{"executor": p},
|
||||
Cmd: exec.Command(bin, "executor", string(c)),
|
||||
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
|
||||
Logger: logger.Named("executor"),
|
||||
}
|
||||
|
||||
if driverConfig != nil {
|
||||
config.MaxPort = driverConfig.ClientMaxPort
|
||||
config.MinPort = driverConfig.ClientMinPort
|
||||
} else {
|
||||
config.MaxPort = ExecutorDefaultMaxPort
|
||||
config.MinPort = ExecutorDefaultMinPort
|
||||
}
|
||||
|
||||
// setting the setsid of the plugin process so that it doesn't get signals sent to
|
||||
// the nomad client.
|
||||
if config.Cmd != nil {
|
||||
isolateCommand(config.Cmd)
|
||||
}
|
||||
|
||||
return newExecutorClient(config, logger)
|
||||
}
|
||||
|
||||
// ReattachToExecutor launches a plugin with a given plugin config
|
||||
func ReattachToExecutor(reattachConfig *plugin.ReattachConfig, logger hclog.Logger) (Executor, *plugin.Client, error) {
|
||||
config := &plugin.ClientConfig{
|
||||
HandshakeConfig: base.Handshake,
|
||||
Reattach: reattachConfig,
|
||||
Plugins: GetPluginMap(logger, false),
|
||||
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
|
||||
Logger: logger.Named("executor"),
|
||||
}
|
||||
|
||||
return newExecutorClient(config, logger)
|
||||
}
|
||||
|
||||
func newExecutorClient(config *plugin.ClientConfig, logger hclog.Logger) (Executor, *plugin.Client, error) {
|
||||
executorClient := plugin.NewClient(config)
|
||||
rpcClient, err := executorClient.Client()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
|
||||
}
|
||||
|
||||
raw, err := rpcClient.Dispense("executor")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
|
||||
}
|
||||
executorPlugin, ok := raw.(Executor)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("unexpected executor rpc type: %T", raw)
|
||||
}
|
||||
return executorPlugin, executorClient, nil
|
||||
}
|
||||
|
||||
func processStateToProto(ps *ProcessState) (*proto.ProcessState, error) {
|
||||
timestamp, err := ptypes.TimestampProto(ps.Time)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pb := &proto.ProcessState{
|
||||
Pid: int32(ps.Pid),
|
||||
ExitCode: int32(ps.ExitCode),
|
||||
Signal: int32(ps.Signal),
|
||||
Time: timestamp,
|
||||
}
|
||||
|
||||
return pb, nil
|
||||
}
|
||||
|
||||
func processStateFromProto(pb *proto.ProcessState) (*ProcessState, error) {
|
||||
timestamp, err := ptypes.Timestamp(pb.Time)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ProcessState{
|
||||
Pid: int(pb.Pid),
|
||||
ExitCode: int(pb.ExitCode),
|
||||
Signal: int(pb.Signal),
|
||||
Time: timestamp,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// IsolationMode returns the namespace isolation mode as determined from agent
|
||||
// plugin configuration and task driver configuration. The task configuration
|
||||
// takes precedence, if it is configured.
|
||||
func IsolationMode(plugin, task string) string {
|
||||
if task != "" {
|
||||
return task
|
||||
}
|
||||
return plugin
|
||||
}
|
||||
19
executor/utils_unix.go
Normal file
19
executor/utils_unix.go
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
|
||||
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
|
||||
|
||||
package executor
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// isolateCommand sets the setsid flag in exec.Cmd to true so that the process
|
||||
// becomes the process leader in a new session and doesn't receive signals that
|
||||
// are sent to the parent process.
|
||||
func isolateCommand(cmd *exec.Cmd) {
|
||||
if cmd.SysProcAttr == nil {
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{}
|
||||
}
|
||||
cmd.SysProcAttr.Setsid = true
|
||||
}
|
||||
55
executor/z_executor_cmd.go
Normal file
55
executor/z_executor_cmd.go
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
|
||||
hclog "github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
|
||||
"github.com/hashicorp/nomad/plugins/base"
|
||||
)
|
||||
|
||||
// Install a plugin cli handler to ease working with tests
|
||||
// and external plugins.
|
||||
// This init() must be initialized last in package required by the child plugin
|
||||
// process. It's recommended to avoid any other `init()` or inline any necessary calls
|
||||
// here. See eeaa95d commit message for more details.
|
||||
func init() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "executor" {
|
||||
if len(os.Args) != 3 {
|
||||
hclog.L().Error("json configuration not provided")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
config := os.Args[2]
|
||||
var executorConfig ExecutorConfig
|
||||
if err := json.Unmarshal([]byte(config), &executorConfig); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(executorConfig.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
|
||||
if err != nil {
|
||||
hclog.L().Error(err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Create the logger
|
||||
logger := hclog.New(&hclog.LoggerOptions{
|
||||
Level: hclog.LevelFromString(executorConfig.LogLevel),
|
||||
JSONFormat: true,
|
||||
Output: f,
|
||||
})
|
||||
|
||||
plugin.Serve(&plugin.ServeConfig{
|
||||
HandshakeConfig: base.Handshake,
|
||||
Plugins: GetPluginMap(
|
||||
logger,
|
||||
executorConfig.FSIsolation,
|
||||
),
|
||||
GRPCServer: plugin.DefaultGRPCServer,
|
||||
Logger: logger,
|
||||
})
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue