2019-04-29 21:25:05 +00:00
|
|
|
// Copyright 2018 The gVisor Authors.
|
2018-11-02 01:28:12 +00:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2019-09-04 05:01:34 +00:00
|
|
|
// Package criutil contains utility functions for interacting with the
|
|
|
|
// Container Runtime Interface (CRI), principally via the crictl command line
|
|
|
|
// tool. This requires critools to be installed on the local system.
|
|
|
|
package criutil
|
2018-11-02 01:28:12 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
|
|
|
"os"
|
|
|
|
"os/exec"
|
2020-05-06 05:00:14 +00:00
|
|
|
"path"
|
|
|
|
"regexp"
|
|
|
|
"strconv"
|
2018-11-02 01:28:12 +00:00
|
|
|
"strings"
|
|
|
|
"time"
|
2019-09-04 05:01:34 +00:00
|
|
|
|
2020-04-23 18:32:08 +00:00
|
|
|
"gvisor.dev/gvisor/pkg/test/dockerutil"
|
|
|
|
"gvisor.dev/gvisor/pkg/test/testutil"
|
2018-11-02 01:28:12 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// Crictl contains information required to run the crictl utility.
|
|
|
|
type Crictl struct {
|
2020-04-23 18:32:08 +00:00
|
|
|
logger testutil.Logger
|
|
|
|
endpoint string
|
|
|
|
cleanup []func()
|
|
|
|
}
|
|
|
|
|
2020-07-21 01:03:04 +00:00
|
|
|
// ResolvePath attempts to find binary paths. It may set the path to invalid,
|
2020-04-23 18:32:08 +00:00
|
|
|
// which will cause the execution to fail with a sensible error.
|
2020-07-21 01:03:04 +00:00
|
|
|
func ResolvePath(executable string) string {
|
2020-05-06 05:00:14 +00:00
|
|
|
runtime, err := dockerutil.RuntimePath()
|
|
|
|
if err == nil {
|
|
|
|
// Check first the directory of the runtime itself.
|
|
|
|
if dir := path.Dir(runtime); dir != "" && dir != "." {
|
|
|
|
guess := path.Join(dir, executable)
|
|
|
|
if fi, err := os.Stat(guess); err == nil && (fi.Mode()&0111) != 0 {
|
|
|
|
return guess
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-11-30 21:38:31 +00:00
|
|
|
// Favor /usr/local/bin, if it exists.
|
|
|
|
localBin := fmt.Sprintf("/usr/local/bin/%s", executable)
|
|
|
|
if _, err := os.Stat(localBin); err == nil {
|
|
|
|
return localBin
|
|
|
|
}
|
|
|
|
|
2020-05-06 05:00:14 +00:00
|
|
|
// Try to find via the path.
|
2020-11-30 21:38:31 +00:00
|
|
|
guess, _ := exec.LookPath(executable)
|
2020-05-06 05:00:14 +00:00
|
|
|
if err == nil {
|
|
|
|
return guess
|
2020-04-23 18:32:08 +00:00
|
|
|
}
|
2020-05-06 05:00:14 +00:00
|
|
|
|
2020-11-30 21:38:31 +00:00
|
|
|
// Return a bare path; this generates a suitable error.
|
|
|
|
return executable
|
2018-11-02 01:28:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewCrictl returns a Crictl configured with a timeout and an endpoint over
|
|
|
|
// which it will talk to containerd.
|
2021-01-13 01:50:33 +00:00
|
|
|
func NewCrictl(logger testutil.Logger, endpoint string) *Crictl {
|
2020-04-23 18:32:08 +00:00
|
|
|
// Attempt to find the executable, but don't bother propagating the
|
|
|
|
// error at this point. The first command executed will return with a
|
|
|
|
// binary not found error.
|
2018-11-02 01:28:12 +00:00
|
|
|
return &Crictl{
|
2020-04-23 18:32:08 +00:00
|
|
|
logger: logger,
|
|
|
|
endpoint: endpoint,
|
2018-11-02 01:28:12 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-23 18:32:08 +00:00
|
|
|
// CleanUp executes cleanup functions.
|
|
|
|
func (cc *Crictl) CleanUp() {
|
|
|
|
for _, c := range cc.cleanup {
|
|
|
|
c()
|
|
|
|
}
|
|
|
|
cc.cleanup = nil
|
2018-11-02 01:28:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// RunPod creates a sandbox. It corresponds to `crictl runp`.
|
2020-05-06 05:00:14 +00:00
|
|
|
func (cc *Crictl) RunPod(runtime, sbSpecFile string) (string, error) {
|
|
|
|
podID, err := cc.run("runp", "--runtime", runtime, sbSpecFile)
|
2018-11-02 01:28:12 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("runp failed: %v", err)
|
|
|
|
}
|
|
|
|
// Strip the trailing newline from crictl output.
|
|
|
|
return strings.TrimSpace(podID), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create creates a container within a sandbox. It corresponds to `crictl
|
|
|
|
// create`.
|
|
|
|
func (cc *Crictl) Create(podID, contSpecFile, sbSpecFile string) (string, error) {
|
2020-05-06 05:00:14 +00:00
|
|
|
// In version 1.16.0, crictl annoying starting attempting to pull the
|
|
|
|
// container, even if it was already available locally. We therefore
|
|
|
|
// need to parse the version and add an appropriate --no-pull argument
|
|
|
|
// since the image has already been loaded locally.
|
|
|
|
out, err := cc.run("-v")
|
2020-07-13 23:10:58 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
2020-05-06 05:00:14 +00:00
|
|
|
r := regexp.MustCompile("crictl version ([0-9]+)\\.([0-9]+)\\.([0-9+])")
|
|
|
|
vs := r.FindStringSubmatch(out)
|
|
|
|
if len(vs) != 4 {
|
|
|
|
return "", fmt.Errorf("crictl -v had unexpected output: %s", out)
|
|
|
|
}
|
|
|
|
major, err := strconv.ParseUint(vs[1], 10, 64)
|
2018-11-02 01:28:12 +00:00
|
|
|
if err != nil {
|
2020-05-06 05:00:14 +00:00
|
|
|
return "", fmt.Errorf("crictl had invalid version: %v (%s)", err, out)
|
|
|
|
}
|
|
|
|
minor, err := strconv.ParseUint(vs[2], 10, 64)
|
2018-11-02 01:28:12 +00:00
|
|
|
if err != nil {
|
2020-05-06 05:00:14 +00:00
|
|
|
return "", fmt.Errorf("crictl had invalid version: %v (%s)", err, out)
|
|
|
|
}
|
|
|
|
|
|
|
|
args := []string{"create"}
|
|
|
|
if (major == 1 && minor >= 16) || major > 1 {
|
|
|
|
args = append(args, "--no-pull")
|
|
|
|
}
|
|
|
|
args = append(args, podID)
|
|
|
|
args = append(args, contSpecFile)
|
|
|
|
args = append(args, sbSpecFile)
|
|
|
|
|
|
|
|
podID, err = cc.run(args...)
|
|
|
|
if err != nil {
|
|
|
|
time.Sleep(10 * time.Minute) // XXX
|
2018-11-02 01:28:12 +00:00
|
|
|
return "", fmt.Errorf("create failed: %v", err)
|
|
|
|
}
|
2020-05-06 05:00:14 +00:00
|
|
|
|
2018-11-02 01:28:12 +00:00
|
|
|
// Strip the trailing newline from crictl output.
|
|
|
|
return strings.TrimSpace(podID), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Start starts a container. It corresponds to `crictl start`.
|
|
|
|
func (cc *Crictl) Start(contID string) (string, error) {
|
|
|
|
output, err := cc.run("start", contID)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("start failed: %v", err)
|
|
|
|
}
|
|
|
|
return output, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stop stops a container. It corresponds to `crictl stop`.
|
|
|
|
func (cc *Crictl) Stop(contID string) error {
|
|
|
|
_, err := cc.run("stop", contID)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-11-08 07:28:37 +00:00
|
|
|
// Exec execs a program inside a container. It corresponds to `crictl exec`.
|
|
|
|
func (cc *Crictl) Exec(contID string, args ...string) (string, error) {
|
|
|
|
a := []string{"exec", contID}
|
|
|
|
a = append(a, args...)
|
|
|
|
output, err := cc.run(a...)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("exec failed: %v", err)
|
|
|
|
}
|
|
|
|
return output, nil
|
|
|
|
}
|
|
|
|
|
2020-06-12 02:29:34 +00:00
|
|
|
// Logs retrieves the container logs. It corresponds to `crictl logs`.
|
|
|
|
func (cc *Crictl) Logs(contID string, args ...string) (string, error) {
|
|
|
|
a := []string{"logs", contID}
|
|
|
|
a = append(a, args...)
|
|
|
|
output, err := cc.run(a...)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("logs failed: %v", err)
|
|
|
|
}
|
|
|
|
return output, nil
|
|
|
|
}
|
|
|
|
|
2018-11-02 01:28:12 +00:00
|
|
|
// Rm removes a container. It corresponds to `crictl rm`.
|
|
|
|
func (cc *Crictl) Rm(contID string) error {
|
|
|
|
_, err := cc.run("rm", contID)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// StopPod stops a pod. It corresponds to `crictl stopp`.
|
|
|
|
func (cc *Crictl) StopPod(podID string) error {
|
|
|
|
_, err := cc.run("stopp", podID)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// containsConfig is a minimal copy of
|
|
|
|
// https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto
|
|
|
|
// It only contains fields needed for testing.
|
|
|
|
type containerConfig struct {
|
|
|
|
Status containerStatus
|
|
|
|
}
|
|
|
|
|
|
|
|
type containerStatus struct {
|
|
|
|
Network containerNetwork
|
|
|
|
}
|
|
|
|
|
|
|
|
type containerNetwork struct {
|
|
|
|
IP string
|
|
|
|
}
|
|
|
|
|
|
|
|
// PodIP returns a pod's IP address.
|
|
|
|
func (cc *Crictl) PodIP(podID string) (string, error) {
|
|
|
|
output, err := cc.run("inspectp", podID)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
conf := &containerConfig{}
|
|
|
|
if err := json.Unmarshal([]byte(output), conf); err != nil {
|
|
|
|
return "", fmt.Errorf("failed to unmarshal JSON: %v, %s", err, output)
|
|
|
|
}
|
|
|
|
if conf.Status.Network.IP == "" {
|
|
|
|
return "", fmt.Errorf("no IP found in config: %s", output)
|
|
|
|
}
|
|
|
|
return conf.Status.Network.IP, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// RmPod removes a container. It corresponds to `crictl rmp`.
|
|
|
|
func (cc *Crictl) RmPod(podID string) error {
|
|
|
|
_, err := cc.run("rmp", podID)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2020-04-23 18:32:08 +00:00
|
|
|
// Import imports the given container from the local Docker instance.
|
|
|
|
func (cc *Crictl) Import(image string) error {
|
|
|
|
// Note that we provide a 10 minute timeout after connect because we may
|
|
|
|
// be pushing a lot of bytes in order to import the image. The connect
|
|
|
|
// timeout stays the same and is inherited from the Crictl instance.
|
|
|
|
cmd := testutil.Command(cc.logger,
|
2020-07-21 01:03:04 +00:00
|
|
|
ResolvePath("ctr"),
|
2020-04-23 18:32:08 +00:00
|
|
|
fmt.Sprintf("--connect-timeout=%s", 30*time.Second),
|
|
|
|
fmt.Sprintf("--address=%s", cc.endpoint),
|
|
|
|
"-n", "k8s.io", "images", "import", "-")
|
|
|
|
cmd.Stderr = os.Stderr // Pass through errors.
|
|
|
|
|
|
|
|
// Create a pipe and start the program.
|
|
|
|
w, err := cmd.StdinPipe()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := cmd.Start(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Save the image on the other end.
|
|
|
|
if err := dockerutil.Save(cc.logger, image, w); err != nil {
|
|
|
|
cmd.Wait()
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close our pipe reference & see if it was loaded.
|
|
|
|
if err := w.Close(); err != nil {
|
|
|
|
return w.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
return cmd.Wait()
|
|
|
|
}
|
|
|
|
|
2019-10-07 22:54:13 +00:00
|
|
|
// StartContainer pulls the given image ands starts the container in the
|
|
|
|
// sandbox with the given podID.
|
2020-04-23 18:32:08 +00:00
|
|
|
//
|
|
|
|
// Note that the image will always be imported from the local docker daemon.
|
2019-10-07 22:54:13 +00:00
|
|
|
func (cc *Crictl) StartContainer(podID, image, sbSpec, contSpec string) (string, error) {
|
2020-04-23 18:32:08 +00:00
|
|
|
if err := cc.Import(image); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
2019-10-07 22:54:13 +00:00
|
|
|
// Write the specs to files that can be read by crictl.
|
2020-04-23 18:32:08 +00:00
|
|
|
sbSpecFile, cleanup, err := testutil.WriteTmpFile("sbSpec", sbSpec)
|
2019-10-07 22:54:13 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("failed to write sandbox spec: %v", err)
|
|
|
|
}
|
2020-04-23 18:32:08 +00:00
|
|
|
cc.cleanup = append(cc.cleanup, cleanup)
|
|
|
|
contSpecFile, cleanup, err := testutil.WriteTmpFile("contSpec", contSpec)
|
2019-10-07 22:54:13 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("failed to write container spec: %v", err)
|
|
|
|
}
|
2020-04-23 18:32:08 +00:00
|
|
|
cc.cleanup = append(cc.cleanup, cleanup)
|
2019-10-07 22:54:13 +00:00
|
|
|
|
|
|
|
return cc.startContainer(podID, image, sbSpecFile, contSpecFile)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (cc *Crictl) startContainer(podID, image, sbSpecFile, contSpecFile string) (string, error) {
|
|
|
|
contID, err := cc.Create(podID, contSpecFile, sbSpecFile)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("failed to create container in pod %q: %v", podID, err)
|
2018-11-02 01:28:12 +00:00
|
|
|
}
|
|
|
|
|
2019-10-07 22:54:13 +00:00
|
|
|
if _, err := cc.Start(contID); err != nil {
|
|
|
|
return "", fmt.Errorf("failed to start container %q in pod %q: %v", contID, podID, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return contID, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// StopContainer stops and deletes the container with the given container ID.
|
|
|
|
func (cc *Crictl) StopContainer(contID string) error {
|
|
|
|
if err := cc.Stop(contID); err != nil {
|
|
|
|
return fmt.Errorf("failed to stop container %q: %v", contID, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := cc.Rm(contID); err != nil {
|
|
|
|
return fmt.Errorf("failed to remove container %q: %v", contID, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-04-23 18:32:08 +00:00
|
|
|
// StartPodAndContainer starts a sandbox and container in that sandbox. It
|
|
|
|
// returns the pod ID and container ID.
|
2020-05-06 05:00:14 +00:00
|
|
|
func (cc *Crictl) StartPodAndContainer(runtime, image, sbSpec, contSpec string) (string, string, error) {
|
2020-04-23 18:32:08 +00:00
|
|
|
if err := cc.Import(image); err != nil {
|
|
|
|
return "", "", err
|
|
|
|
}
|
|
|
|
|
2018-11-02 01:28:12 +00:00
|
|
|
// Write the specs to files that can be read by crictl.
|
2020-04-23 18:32:08 +00:00
|
|
|
sbSpecFile, cleanup, err := testutil.WriteTmpFile("sbSpec", sbSpec)
|
2018-11-02 01:28:12 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", "", fmt.Errorf("failed to write sandbox spec: %v", err)
|
|
|
|
}
|
2020-04-23 18:32:08 +00:00
|
|
|
cc.cleanup = append(cc.cleanup, cleanup)
|
|
|
|
contSpecFile, cleanup, err := testutil.WriteTmpFile("contSpec", contSpec)
|
2018-11-02 01:28:12 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", "", fmt.Errorf("failed to write container spec: %v", err)
|
|
|
|
}
|
2020-04-23 18:32:08 +00:00
|
|
|
cc.cleanup = append(cc.cleanup, cleanup)
|
2018-11-02 01:28:12 +00:00
|
|
|
|
2020-05-06 05:00:14 +00:00
|
|
|
podID, err := cc.RunPod(runtime, sbSpecFile)
|
2018-11-02 01:28:12 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", "", err
|
|
|
|
}
|
|
|
|
|
2019-10-07 22:54:13 +00:00
|
|
|
contID, err := cc.startContainer(podID, image, sbSpecFile, contSpecFile)
|
2018-11-02 01:28:12 +00:00
|
|
|
|
2019-10-07 22:54:13 +00:00
|
|
|
return podID, contID, err
|
2018-11-02 01:28:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// StopPodAndContainer stops a container and pod.
|
|
|
|
func (cc *Crictl) StopPodAndContainer(podID, contID string) error {
|
2019-10-07 22:54:13 +00:00
|
|
|
if err := cc.StopContainer(contID); err != nil {
|
2018-11-02 01:28:12 +00:00
|
|
|
return fmt.Errorf("failed to stop container %q in pod %q: %v", contID, podID, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := cc.StopPod(podID); err != nil {
|
|
|
|
return fmt.Errorf("failed to stop pod %q: %v", podID, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := cc.RmPod(podID); err != nil {
|
|
|
|
return fmt.Errorf("failed to remove pod %q: %v", podID, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-04-23 18:32:08 +00:00
|
|
|
// run runs crictl with the given args.
|
2018-11-02 01:28:12 +00:00
|
|
|
func (cc *Crictl) run(args ...string) (string, error) {
|
|
|
|
defaultArgs := []string{
|
2020-07-21 01:03:04 +00:00
|
|
|
ResolvePath("crictl"),
|
2020-04-23 18:32:08 +00:00
|
|
|
"--image-endpoint", fmt.Sprintf("unix://%s", cc.endpoint),
|
|
|
|
"--runtime-endpoint", fmt.Sprintf("unix://%s", cc.endpoint),
|
2018-11-02 01:28:12 +00:00
|
|
|
}
|
2020-04-23 18:32:08 +00:00
|
|
|
fullArgs := append(defaultArgs, args...)
|
|
|
|
out, err := testutil.Command(cc.logger, fullArgs...).CombinedOutput()
|
|
|
|
return string(out), err
|
2018-11-02 01:28:12 +00:00
|
|
|
}
|