gvisor/pkg/shim/runsc/runsc.go

// Copyright 2018 The containerd Authors.
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package runsc provides an API to interact with runsc command line.
package runsc

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"

	"github.com/containerd/containerd/log"
	runc "github.com/containerd/go-runc"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"golang.org/x/sys/unix"
)

// DefaultCommand is the default command for Runsc.
const DefaultCommand = "runsc"

// Monitor is the default process monitor to be used by runsc.
var Monitor runc.ProcessMonitor = &LogMonitor{Next: runc.Monitor}

// LogMonitor implements the runc.ProcessMonitor interface, logging the command
// that is getting executed, and then forwarding the call to another
// implementation.
type LogMonitor struct {
	Next runc.ProcessMonitor
}

// Start implements runc.ProcessMonitor.
func (l *LogMonitor) Start(cmd *exec.Cmd) (chan runc.Exit, error) {
	log.L.Debugf("Executing: %s", cmd.Args)
	return l.Next.Start(cmd)
}

// Wait implements runc.ProcessMonitor.
func (l *LogMonitor) Wait(cmd *exec.Cmd, ch chan runc.Exit) (int, error) {
	status, err := l.Next.Wait(cmd, ch)
	log.L.Debugf("Command exit code: %d, err: %v", status, err)
	return status, err
}

// Runsc is the client to the runsc cli.
type Runsc struct {
	Command      string
	PdeathSignal unix.Signal
	Setpgid      bool
	Root         string
	Log          string
	LogFormat    runc.Format
	Config       map[string]string
}

// List returns all containers created inside the provided runsc root directory.
func (r *Runsc) List(context context.Context) ([]*runc.Container, error) {
	data, stderr, err := cmdOutput(r.command(context, "list", "--format=json"), false)
	if err != nil {
		return nil, fmt.Errorf("%w: %s", err, stderr)
	}
	var out []*runc.Container
	if err := json.Unmarshal(data, &out); err != nil {
		return nil, err
	}
	return out, nil
}

// State returns the state for the container provided by id.
func (r *Runsc) State(context context.Context, id string) (*runc.Container, error) {
	data, stderr, err := cmdOutput(r.command(context, "state", id), false)
	if err != nil {
		return nil, fmt.Errorf("%w: %s", err, stderr)
	}
	var c runc.Container
	if err := json.Unmarshal(data, &c); err != nil {
		return nil, err
	}
	return &c, nil
}

// CreateOpts is a set of options to Runsc.Create().
type CreateOpts struct {
	runc.IO
	ConsoleSocket runc.ConsoleSocket

	// PidFile is a path to where a pid file should be created.
	PidFile string

	// UserLog is a path to where runsc user log should be generated.
	UserLog string
}

func (o *CreateOpts) args() (out []string, err error) {
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.UserLog != "" {
		out = append(out, "--user-log", o.UserLog)
	}
	return out, nil
}

// Create creates a new container and returns its pid if it was created successfully.
func (r *Runsc) Create(context context.Context, id, bundle string, opts *CreateOpts) error {
	args := []string{"create", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}

	if cmd.Stdout == nil && cmd.Stderr == nil {
		out, _, err := cmdOutput(cmd, true)
		if err != nil {
			return fmt.Errorf("%w: %s", err, out)
		}
		return nil
	}
	ec, err := Monitor.Start(cmd)
	if err != nil {
		return err
	}
	if opts != nil && opts.IO != nil {
		if c, ok := opts.IO.(runc.StartCloser); ok {
			if err := c.CloseAfterStart(); err != nil {
				return err
			}
		}
	}
	status, err := Monitor.Wait(cmd, ec)
	if err == nil && status != 0 {
		err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
	}

	return err
}

func (r *Runsc) Pause(context context.Context, id string) error {
	if out, _, err := cmdOutput(r.command(context, "pause", id), true); err != nil {
		return fmt.Errorf("unable to pause: %w: %s", err, out)
	}
	return nil
}

func (r *Runsc) Resume(context context.Context, id string) error {
	if out, _, err := cmdOutput(r.command(context, "resume", id), true); err != nil {
		return fmt.Errorf("unable to resume: %w: %s", err, out)
	}
	return nil
}

// Start will start an already created container.
func (r *Runsc) Start(context context.Context, id string, cio runc.IO) error {
	cmd := r.command(context, "start", id)
	if cio != nil {
		cio.Set(cmd)
	}

	if cmd.Stdout == nil && cmd.Stderr == nil {
		out, _, err := cmdOutput(cmd, true)
		if err != nil {
			return fmt.Errorf("%w: %s", err, out)
		}
		return nil
	}

	ec, err := Monitor.Start(cmd)
	if err != nil {
		return err
	}
	if cio != nil {
		if c, ok := cio.(runc.StartCloser); ok {
			if err := c.CloseAfterStart(); err != nil {
				return err
			}
		}
	}
	status, err := Monitor.Wait(cmd, ec)
	if err == nil && status != 0 {
		err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
	}

	return err
}

type waitResult struct {
	ID         string `json:"id"`
	ExitStatus int    `json:"exitStatus"`
}

// Wait will wait for a running container, and return its exit status.
func (r *Runsc) Wait(context context.Context, id string) (int, error) {
	data, stderr, err := cmdOutput(r.command(context, "wait", id), false)
	if err != nil {
		return 0, fmt.Errorf("%w: %s", err, stderr)
	}
	var res waitResult
	if err := json.Unmarshal(data, &res); err != nil {
		return 0, err
	}
	return res.ExitStatus, nil
}

// ExecOpts is a set of options to runsc.Exec().
type ExecOpts struct {
	runc.IO
	PidFile         string
	InternalPidFile string
	ConsoleSocket   runc.ConsoleSocket
	Detach          bool
}

func (o *ExecOpts) args() (out []string, err error) {
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.Detach {
		out = append(out, "--detach")
	}
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	if o.InternalPidFile != "" {
		abs, err := filepath.Abs(o.InternalPidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--internal-pid-file", abs)
	}
	return out, nil
}

// Exec executes an additional process inside the container based on a full OCI
// Process specification.
func (r *Runsc) Exec(context context.Context, id string, spec specs.Process, opts *ExecOpts) error {
	f, err := ioutil.TempFile(os.Getenv("XDG_RUNTIME_DIR"), "runsc-process")
	if err != nil {
		return err
	}
	defer os.Remove(f.Name())
	err = json.NewEncoder(f).Encode(spec)
	f.Close()
	if err != nil {
		return err
	}
	args := []string{"exec", "--process", f.Name()}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}
	if cmd.Stdout == nil && cmd.Stderr == nil {
		out, _, err := cmdOutput(cmd, true)
		if err != nil {
			return fmt.Errorf("%w: %s", err, out)
		}
		return nil
	}
	ec, err := Monitor.Start(cmd)
	if err != nil {
		return err
	}
	if opts != nil && opts.IO != nil {
		if c, ok := opts.IO.(runc.StartCloser); ok {
			if err := c.CloseAfterStart(); err != nil {
				return err
			}
		}
	}
	status, err := Monitor.Wait(cmd, ec)
	if err == nil && status != 0 {
		err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
	}
	return err
}

// Run runs the create, start, delete lifecycle of the container and returns
// its exit status after it has exited.
func (r *Runsc) Run(context context.Context, id, bundle string, opts *CreateOpts) (int, error) {
	args := []string{"run", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return -1, err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}
	ec, err := Monitor.Start(cmd)
	if err != nil {
		return -1, err
	}
	return Monitor.Wait(cmd, ec)
}

// DeleteOpts is a set of options to runsc.Delete().
type DeleteOpts struct {
	Force bool
}

func (o *DeleteOpts) args() (out []string) {
	if o.Force {
		out = append(out, "--force")
	}
	return out
}

// Delete deletes the container.
func (r *Runsc) Delete(context context.Context, id string, opts *DeleteOpts) error {
	args := []string{"delete"}
	if opts != nil {
		args = append(args, opts.args()...)
	}
	return r.runOrError(r.command(context, append(args, id)...))
}

// KillOpts specifies options for killing a container and its processes.
type KillOpts struct {
	All bool
	Pid int
}

func (o *KillOpts) args() (out []string) {
	if o.All {
		out = append(out, "--all")
	}
	if o.Pid != 0 {
		out = append(out, "--pid", strconv.Itoa(o.Pid))
	}
	return out
}

// Kill sends the specified signal to the container.
func (r *Runsc) Kill(context context.Context, id string, sig int, opts *KillOpts) error {
	args := []string{
		"kill",
	}
	if opts != nil {
		args = append(args, opts.args()...)
	}
	return r.runOrError(r.command(context, append(args, id, strconv.Itoa(sig))...))
}

// Stats return the stats for a container like cpu, memory, and I/O.
func (r *Runsc) Stats(context context.Context, id string) (*runc.Stats, error) {
	cmd := r.command(context, "events", "--stats", id)
	data, stderr, err := cmdOutput(cmd, false)
	if err != nil {
		return nil, fmt.Errorf("%w: %s", err, stderr)
	}
	var e runc.Event
	if err := json.Unmarshal(data, &e); err != nil {
		log.L.Debugf("Parsing events error: %v", err)
		return nil, err
	}
	log.L.Debugf("Stats returned, type: %s, stats: %+v", e.Type, e.Stats)
	if e.Type != "stats" {
		return nil, fmt.Errorf(`unexpected event type %q, wanted "stats"`, e.Type)
	}
	if e.Stats == nil {
		return nil, fmt.Errorf(`"runsc events -stat" succeeded but no stat was provided`)
	}
	return e.Stats, nil
}

// Events returns an event stream from runsc for a container with stats and OOM notifications.
func (r *Runsc) Events(context context.Context, id string, interval time.Duration) (chan *runc.Event, error) {
	cmd := r.command(context, "events", fmt.Sprintf("--interval=%ds", int(interval.Seconds())), id)
	rd, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	ec, err := Monitor.Start(cmd)
	if err != nil {
		rd.Close()
		return nil, err
	}
	var (
		dec = json.NewDecoder(rd)
		c   = make(chan *runc.Event, 128)
	)
	go func() {
		defer func() {
			close(c)
			rd.Close()
			Monitor.Wait(cmd, ec)
		}()
		for {
			var e runc.Event
			if err := dec.Decode(&e); err != nil {
				if err == io.EOF {
					return
				}
				e = runc.Event{
					Type: "error",
					Err:  err,
				}
			}
			c <- &e
		}
	}()
	return c, nil
}

// Ps lists all the processes inside the container returning their pids.
func (r *Runsc) Ps(context context.Context, id string) ([]int, error) {
	data, stderr, err := cmdOutput(r.command(context, "ps", "--format", "json", id), false)
	if err != nil {
		return nil, fmt.Errorf("%w: %s", err, stderr)
	}
	var pids []int
	if err := json.Unmarshal(data, &pids); err != nil {
		return nil, err
	}
	return pids, nil
}

// Top lists all the processes inside the container returning the full ps data.
func (r *Runsc) Top(context context.Context, id string) (*runc.TopResults, error) {
	data, stderr, err := cmdOutput(r.command(context, "ps", "--format", "table", id), false)
	if err != nil {
		return nil, fmt.Errorf("%w: %s", err, stderr)
	}

	topResults, err := runc.ParsePSOutput(data)
	if err != nil {
		return nil, fmt.Errorf("%s: ", err)
	}
	return topResults, nil
}

func (r *Runsc) args() []string {
	var args []string
	if r.Root != "" {
		args = append(args, fmt.Sprintf("--root=%s", r.Root))
	}
	if r.Log != "" {
		args = append(args, fmt.Sprintf("--log=%s", r.Log))
	}
	if r.LogFormat != "" {
		args = append(args, fmt.Sprintf("--log-format=%s", r.LogFormat))
	}
	for k, v := range r.Config {
		args = append(args, fmt.Sprintf("--%s=%s", k, v))
	}
	return args
}

// runOrError will run the provided command.
//
// If an error is encountered and neither Stdout or Stderr was set the error
// will be returned in the format of <error>: <stderr>.
func (r *Runsc) runOrError(cmd *exec.Cmd) error {
	if cmd.Stdout != nil || cmd.Stderr != nil {
		ec, err := Monitor.Start(cmd)
		if err != nil {
			return err
		}
		status, err := Monitor.Wait(cmd, ec)
		if err == nil && status != 0 {
			err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
		}
		return err
	}
	out, _, err := cmdOutput(cmd, true)
	if err != nil {
		return fmt.Errorf("%w: %s", err, out)
	}
	return nil
}

func (r *Runsc) command(context context.Context, args ...string) *exec.Cmd {
	command := r.Command
	if command == "" {
		command = DefaultCommand
	}
	cmd := exec.CommandContext(context, command, append(r.args(), args...)...)
	cmd.SysProcAttr = &unix.SysProcAttr{
		Setpgid: r.Setpgid,
	}
	if r.PdeathSignal != 0 {
		cmd.SysProcAttr.Pdeathsig = r.PdeathSignal
	}

	return cmd
}

func cmdOutput(cmd *exec.Cmd, combined bool) ([]byte, []byte, error) {
	stdout := getBuf()
	defer putBuf(stdout)
	cmd.Stdout = stdout
	cmd.Stderr = stdout

	var stderr *bytes.Buffer
	if !combined {
		stderr = getBuf()
		defer putBuf(stderr)
		cmd.Stderr = stderr
	}
	ec, err := Monitor.Start(cmd)
	if err != nil {
		return nil, nil, err
	}

	status, err := Monitor.Wait(cmd, ec)
	if err == nil && status != 0 {
		err = fmt.Errorf("%q did not terminate sucessfully", cmd.Args[0])
	}
	if stderr == nil {
		return stdout.Bytes(), nil, err
	}
	return stdout.Bytes(), stderr.Bytes(), err
}