2018-06-04 18:25:40 +00:00
|
|
|
// Copyright 2018 Google Inc.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
// Package testutil contains utility functions for runsc tests.
|
|
|
|
package testutil
|
|
|
|
|
|
|
|
import (
|
2018-07-26 00:36:52 +00:00
|
|
|
"context"
|
2018-06-04 18:25:40 +00:00
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
2018-06-20 21:37:56 +00:00
|
|
|
"io"
|
2018-06-04 18:25:40 +00:00
|
|
|
"io/ioutil"
|
2018-08-07 20:47:16 +00:00
|
|
|
"net/http"
|
2018-06-04 18:25:40 +00:00
|
|
|
"os"
|
2018-08-27 18:09:06 +00:00
|
|
|
"os/exec"
|
2018-09-18 22:20:19 +00:00
|
|
|
"os/signal"
|
2018-06-04 18:25:40 +00:00
|
|
|
"path/filepath"
|
2018-08-27 18:09:06 +00:00
|
|
|
"runtime"
|
|
|
|
"syscall"
|
2018-06-04 18:25:40 +00:00
|
|
|
"time"
|
|
|
|
|
2018-07-26 00:36:52 +00:00
|
|
|
"github.com/cenkalti/backoff"
|
2018-06-04 18:25:40 +00:00
|
|
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
|
|
|
"gvisor.googlesource.com/gvisor/runsc/boot"
|
|
|
|
"gvisor.googlesource.com/gvisor/runsc/specutils"
|
|
|
|
)
|
|
|
|
|
2018-07-23 20:30:29 +00:00
|
|
|
// RaceEnabled is set to true if it was built with '--race' option.
|
|
|
|
var RaceEnabled = false
|
|
|
|
|
2018-08-20 18:25:42 +00:00
|
|
|
// TmpDir returns the absolute path to a writable directory that can be used as
|
|
|
|
// scratch by the test.
|
|
|
|
func TmpDir() string {
|
|
|
|
dir := os.Getenv("TEST_TMPDIR")
|
|
|
|
if dir == "" {
|
|
|
|
dir = "/tmp"
|
|
|
|
}
|
|
|
|
return dir
|
|
|
|
}
|
|
|
|
|
2018-06-04 18:25:40 +00:00
|
|
|
// ConfigureExePath configures the executable for runsc in the test environment.
|
|
|
|
func ConfigureExePath() error {
|
2018-08-16 17:54:21 +00:00
|
|
|
path, err := FindFile("runsc/runsc")
|
2018-06-04 18:25:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-08-16 17:54:21 +00:00
|
|
|
specutils.ExePath = path
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// FindFile searchs for a file inside the test run environment. It returns the
|
|
|
|
// full path to the file. It fails if none or more than one file is found.
|
|
|
|
func FindFile(path string) (string, error) {
|
|
|
|
wd, err := os.Getwd()
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
// The test root is demarcated by a path element called "__main__". Search for
|
|
|
|
// it backwards from the in the working directory.
|
|
|
|
root := wd
|
|
|
|
for {
|
|
|
|
dir, name := filepath.Split(root)
|
|
|
|
if name == "__main__" {
|
|
|
|
break
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
2018-08-16 17:54:21 +00:00
|
|
|
if len(dir) == 0 {
|
|
|
|
return "", fmt.Errorf("directory __main__ not found in %q", wd)
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
2018-08-16 17:54:21 +00:00
|
|
|
// Remove ending slash to loop around.
|
|
|
|
root = dir[:len(dir)-1]
|
|
|
|
}
|
|
|
|
|
|
|
|
// bazel adds the build type to the directory structure. Since I don't want
|
|
|
|
// to guess what build type it's, just place '*' to match anything.
|
|
|
|
//
|
|
|
|
// The pattern goes like: /test-path/__main__/directories/*/file.
|
|
|
|
pattern := filepath.Join(root, filepath.Dir(path), "*", filepath.Base(path))
|
|
|
|
matches, err := filepath.Glob(pattern)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("error globbing %q: %v", pattern, err)
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
2018-08-16 17:54:21 +00:00
|
|
|
if len(matches) == 0 {
|
|
|
|
return "", fmt.Errorf("file %q not found", path)
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
2018-08-16 17:54:21 +00:00
|
|
|
if len(matches) != 1 {
|
|
|
|
return "", fmt.Errorf("more than one match found for %q: %s", path, matches)
|
|
|
|
}
|
|
|
|
return matches[0], nil
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
|
|
|
|
2018-07-23 20:30:29 +00:00
|
|
|
// TestConfig return the default configuration to use in tests.
|
|
|
|
func TestConfig() *boot.Config {
|
|
|
|
return &boot.Config{
|
2018-09-07 17:15:34 +00:00
|
|
|
Debug: true,
|
|
|
|
LogFormat: "text",
|
|
|
|
LogPackets: true,
|
|
|
|
Network: boot.NetworkNone,
|
|
|
|
Strace: true,
|
|
|
|
MultiContainer: true,
|
2018-09-07 19:27:44 +00:00
|
|
|
FileAccess: boot.FileAccessExclusive,
|
2018-09-07 17:15:34 +00:00
|
|
|
TestOnlyAllowRunAsCurrentUserWithoutChroot: true,
|
2018-07-23 20:30:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-04 18:25:40 +00:00
|
|
|
// NewSpecWithArgs creates a simple spec with the given args suitable for use
|
|
|
|
// in tests.
|
|
|
|
func NewSpecWithArgs(args ...string) *specs.Spec {
|
2018-08-20 18:25:42 +00:00
|
|
|
return &specs.Spec{
|
2018-06-04 18:25:40 +00:00
|
|
|
// The host filesystem root is the container root.
|
|
|
|
Root: &specs.Root{
|
|
|
|
Path: "/",
|
|
|
|
Readonly: true,
|
|
|
|
},
|
|
|
|
Process: &specs.Process{
|
|
|
|
Args: args,
|
|
|
|
Env: []string{
|
|
|
|
"PATH=" + os.Getenv("PATH"),
|
|
|
|
},
|
|
|
|
},
|
2018-08-20 18:25:42 +00:00
|
|
|
Mounts: []specs.Mount{
|
|
|
|
// Root is readonly, but many tests want to write to tmpdir.
|
|
|
|
// This creates a writable mount inside the root. Also, when tmpdir points
|
|
|
|
// to "/tmp", it makes the the actual /tmp to be mounted and not a tmpfs
|
|
|
|
// inside the sentry.
|
|
|
|
specs.Mount{
|
|
|
|
Type: "bind",
|
|
|
|
Destination: TmpDir(),
|
|
|
|
Source: TmpDir(),
|
|
|
|
},
|
|
|
|
},
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-06 23:12:58 +00:00
|
|
|
// SetupRootDir creates a root directory for containers.
|
|
|
|
func SetupRootDir() (string, error) {
|
2018-08-20 18:25:42 +00:00
|
|
|
rootDir, err := ioutil.TempDir(TmpDir(), "containers")
|
2018-06-06 23:12:58 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("error creating root dir: %v", err)
|
|
|
|
}
|
|
|
|
return rootDir, nil
|
|
|
|
}
|
|
|
|
|
2018-06-04 18:25:40 +00:00
|
|
|
// SetupContainer creates a bundle and root dir for the container, generates a
|
|
|
|
// test config, and writes the spec to config.json in the bundle dir.
|
2018-07-23 20:30:29 +00:00
|
|
|
func SetupContainer(spec *specs.Spec, conf *boot.Config) (rootDir, bundleDir string, err error) {
|
2018-06-06 23:12:58 +00:00
|
|
|
rootDir, err = SetupRootDir()
|
2018-06-04 18:25:40 +00:00
|
|
|
if err != nil {
|
2018-07-23 20:30:29 +00:00
|
|
|
return "", "", err
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
2018-07-23 20:30:29 +00:00
|
|
|
bundleDir, err = SetupContainerInRoot(rootDir, spec, conf)
|
|
|
|
return rootDir, bundleDir, err
|
2018-06-06 23:12:58 +00:00
|
|
|
}
|
2018-06-04 18:25:40 +00:00
|
|
|
|
2018-06-06 23:12:58 +00:00
|
|
|
// SetupContainerInRoot creates a bundle for the container, generates a test
|
|
|
|
// config, and writes the spec to config.json in the bundle dir.
|
2018-07-23 20:30:29 +00:00
|
|
|
func SetupContainerInRoot(rootDir string, spec *specs.Spec, conf *boot.Config) (bundleDir string, err error) {
|
2018-08-20 18:25:42 +00:00
|
|
|
bundleDir, err = ioutil.TempDir(TmpDir(), "bundle")
|
2018-06-04 18:25:40 +00:00
|
|
|
if err != nil {
|
2018-07-23 20:30:29 +00:00
|
|
|
return "", fmt.Errorf("error creating bundle dir: %v", err)
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if err = writeSpec(bundleDir, spec); err != nil {
|
2018-07-23 20:30:29 +00:00
|
|
|
return "", fmt.Errorf("error writing spec: %v", err)
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
|
|
|
|
2018-07-23 20:30:29 +00:00
|
|
|
conf.RootDir = rootDir
|
2018-09-05 03:08:41 +00:00
|
|
|
conf.SpecFile = filepath.Join(bundleDir, "config.json")
|
2018-07-23 20:30:29 +00:00
|
|
|
return bundleDir, nil
|
2018-06-04 18:25:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// writeSpec writes the spec to disk in the given directory.
|
|
|
|
func writeSpec(dir string, spec *specs.Spec) error {
|
|
|
|
b, err := json.Marshal(spec)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return ioutil.WriteFile(filepath.Join(dir, "config.json"), b, 0755)
|
|
|
|
}
|
|
|
|
|
|
|
|
// UniqueContainerID generates a unique container id for each test.
|
|
|
|
//
|
|
|
|
// The container id is used to create an abstract unix domain socket, which must
|
|
|
|
// be unique. While the container forbids creating two containers with the same
|
|
|
|
// name, sometimes between test runs the socket does not get cleaned up quickly
|
|
|
|
// enough, causing container creation to fail.
|
|
|
|
func UniqueContainerID() string {
|
|
|
|
return fmt.Sprintf("test-container-%d", time.Now().UnixNano())
|
|
|
|
}
|
2018-06-20 21:37:56 +00:00
|
|
|
|
|
|
|
// Copy copies file from src to dst.
|
|
|
|
func Copy(src, dst string) error {
|
|
|
|
in, err := os.Open(src)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer in.Close()
|
|
|
|
|
|
|
|
out, err := os.Create(dst)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer out.Close()
|
|
|
|
|
|
|
|
_, err = io.Copy(out, in)
|
|
|
|
return err
|
|
|
|
}
|
2018-07-26 00:36:52 +00:00
|
|
|
|
|
|
|
// Poll is a shorthand function to poll for something with given timeout.
|
|
|
|
func Poll(cb func() error, timeout time.Duration) error {
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
|
|
defer cancel()
|
|
|
|
b := backoff.WithContext(backoff.NewConstantBackOff(100*time.Millisecond), ctx)
|
|
|
|
return backoff.Retry(cb, b)
|
|
|
|
}
|
2018-08-07 20:47:16 +00:00
|
|
|
|
|
|
|
// WaitForHTTP tries GET requests on a port until the call succeeds or timeout.
|
|
|
|
func WaitForHTTP(port int, timeout time.Duration) error {
|
|
|
|
cb := func() error {
|
|
|
|
_, err := http.Get(fmt.Sprintf("http://localhost:%d/", port))
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return Poll(cb, timeout)
|
|
|
|
}
|
2018-08-27 18:09:06 +00:00
|
|
|
|
|
|
|
// RunAsRoot ensures the test runs with CAP_SYS_ADMIN. If need it will create
|
|
|
|
// a new user namespace and reexecute the test as root inside of the namespace.
|
2018-09-18 22:20:19 +00:00
|
|
|
// This functionr returns when it's running as root. If it needs to create
|
|
|
|
// another process, it will exit from there and not return.
|
|
|
|
func RunAsRoot() {
|
2018-09-07 17:15:34 +00:00
|
|
|
if specutils.HasCapSysAdmin() {
|
2018-09-18 22:20:19 +00:00
|
|
|
return
|
2018-08-27 18:09:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Current process doesn't have CAP_SYS_ADMIN, create user namespace and run
|
|
|
|
// as root inside that namespace to get it.
|
|
|
|
runtime.LockOSThread()
|
|
|
|
defer runtime.UnlockOSThread()
|
|
|
|
|
2018-08-31 16:44:31 +00:00
|
|
|
cmd := exec.Command("/proc/self/exe", os.Args[1:]...)
|
2018-08-27 18:09:06 +00:00
|
|
|
cmd.SysProcAttr = &syscall.SysProcAttr{
|
|
|
|
Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNS,
|
|
|
|
// Set current user/group as root inside the namespace.
|
|
|
|
UidMappings: []syscall.SysProcIDMap{
|
|
|
|
{ContainerID: 0, HostID: os.Getuid(), Size: 1},
|
|
|
|
},
|
|
|
|
GidMappings: []syscall.SysProcIDMap{
|
|
|
|
{ContainerID: 0, HostID: os.Getgid(), Size: 1},
|
|
|
|
},
|
|
|
|
GidMappingsEnableSetgroups: false,
|
|
|
|
Credential: &syscall.Credential{
|
|
|
|
Uid: 0,
|
|
|
|
Gid: 0,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
cmd.Env = os.Environ()
|
|
|
|
cmd.Stdin = os.Stdin
|
|
|
|
cmd.Stdout = os.Stdout
|
|
|
|
cmd.Stderr = os.Stderr
|
|
|
|
if err := cmd.Run(); err != nil {
|
|
|
|
if exit, ok := err.(*exec.ExitError); ok {
|
|
|
|
if ws, ok := exit.Sys().(syscall.WaitStatus); ok {
|
|
|
|
os.Exit(ws.ExitStatus())
|
|
|
|
}
|
|
|
|
os.Exit(-1)
|
|
|
|
}
|
|
|
|
panic(fmt.Sprint("error running child process:", err.Error()))
|
|
|
|
}
|
|
|
|
os.Exit(0)
|
|
|
|
}
|
2018-09-18 22:20:19 +00:00
|
|
|
|
|
|
|
// StartReaper starts a gorouting that will reap all children processes created
|
|
|
|
// by the tests. Caller must call the returned function to stop it.
|
|
|
|
func StartReaper() func() {
|
|
|
|
ch := make(chan os.Signal, 1)
|
|
|
|
signal.Notify(ch, syscall.SIGCHLD)
|
|
|
|
stop := make(chan struct{})
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ch:
|
|
|
|
case <-stop:
|
|
|
|
return
|
|
|
|
}
|
|
|
|
for {
|
|
|
|
cpid, _ := syscall.Wait4(-1, nil, syscall.WNOHANG, nil)
|
|
|
|
if cpid < 1 {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
return func() { stop <- struct{}{} }
|
|
|
|
}
|
|
|
|
|
|
|
|
// RetryEintr retries the function until an error different than EINTR is
|
|
|
|
// returned.
|
|
|
|
func RetryEintr(f func() (uintptr, uintptr, error)) (uintptr, uintptr, error) {
|
|
|
|
for {
|
|
|
|
r1, r2, err := f()
|
|
|
|
if err != syscall.EINTR {
|
|
|
|
return r1, r2, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|