Support STAGED_BINARIES to run prebuilt binaries with the test pipeline.

In some cases, it may be desirable to prebuild binaries and run all tests,
for example to run benchmarks with various experiments. Allow the top-level
Makefile to support this by checking for a STAGED_BINARIES variable.

PiperOrigin-RevId: 410673120
This commit is contained in:
Adin Scannell 2021-11-17 17:47:00 -08:00 committed by gVisor bot
parent ce194f2c1c
commit f1a46c928f
17 changed files with 126 additions and 199 deletions

View File

@ -17,7 +17,6 @@ _templates:
# BENCHMARKS_OFFICIAL is set from hooks/pre-command, based
# on whether this is executing on the master branch.
BENCHMARKS_DATASET: buildkite
BENCHMARKS_PLATFORMS: "ptrace kvm"
BENCHMARKS_PROJECT: gvisor-benchmarks
BENCHMARKS_TABLE: benchmarks
BENCHMARKS_UPLOAD: true
@ -61,8 +60,8 @@ steps:
label: ":test_tube: Unit tests"
command: make unit-tests
- <<: *common
label: ":test_tube: runsc tests"
command: make runsc-tests
label: ":test_tube: Container tests"
command: make container-tests
# All system call tests.
- <<: *common

View File

@ -99,6 +99,8 @@ endif
## RUNTIME_BIN - The runtime binary (default: $RUNTIME_DIR/runsc).
## RUNTIME_LOG_DIR - The logs directory (default: $RUNTIME_DIR/logs).
## RUNTIME_LOGS - The log pattern (default: $RUNTIME_LOG_DIR/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%).
## STAGED_BINARIES - A tarball of staged binaries. If this is set, then binaries
## will be installed from this staged bundle instead of built.
##
ifeq (,$(BRANCH_NAME))
RUNTIME := runsc
@ -113,7 +115,13 @@ RUNTIME_LOGS := $(RUNTIME_LOG_DIR)/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%
$(RUNTIME_BIN): # See below.
@mkdir -p "$(RUNTIME_DIR)"
ifeq (,$(STAGED_BINARIES))
@$(call copy,//runsc,$(RUNTIME_BIN))
else
gsutil cat "${STAGED_BINARIES}" | \
tar -C "$(RUNTIME_DIR)" -zxvf - runsc && \
chmod a+rx "$(RUNTIME_BIN)"
endif
.PHONY: $(RUNTIME_BIN) # Real file, but force rebuild.
# Configure helpers for below.
@ -179,16 +187,19 @@ nogo-tests:
@$(call test,--build_tag_filters=nogo --test_tag_filters=nogo //:all pkg/... tools/...)
.PHONY: nogo-tests
# For unit tests, we take everything in the root, pkg/... and tools/..., and
# pull in all directories in runsc except runsc/container.
unit-tests: ## Local package unit tests in pkg/..., tools/.., etc.
@$(call test,--build_tag_filters=-nogo --test_tag_filters=-nogo //:all pkg/... tools/...)
@$(call test,--build_tag_filters=-nogo --test_tag_filters=-nogo --test_filter=-//runsc/container/... //:all pkg/... tools/... runsc/...)
.PHONY: unit-tests
runsc-tests: ## Run all tests in runsc/...
@$(call test,runsc/...)
.PHONY: runsc-tests
# See unit-tests: this includes runsc/container.
container-tests: $(RUNTIME_BIN) ## Run all tests in runsc/container/...
@$(call test,--test_arg=--runsc=$(RUNTIME_BIN) runsc/container/...)
.PHONY: container-tests
tests: ## Runs all unit tests and syscall tests.
tests: unit-tests nogo-tests runsc-tests syscall-tests
tests: unit-tests nogo-tests container-tests syscall-tests
.PHONY: tests
integration-tests: ## Run all standard integration tests.
@ -211,7 +222,7 @@ syscall-native-tests:
.PHONY: syscall-native-tests
syscall-tests: ## Run all system call tests.
@$(call test,$(PARTITIONS) $(SYSCALL_TARGETS))
@$(call test,$(PARTITIONS) test/syscalls/...)
.PHONY: syscall-tests
%-runtime-tests: load-runtimes_% $(RUNTIME_BIN)
@ -222,17 +233,17 @@ syscall-tests: ## Run all system call tests.
@$(call install_runtime,$(RUNTIME),--vfs2)
@$(call test_runtime,$(RUNTIME),--test_timeout=10800 //test/runtimes:$*)
do-tests:
@$(call run,//runsc,--rootless do true)
@$(call run,//runsc,--rootless -network=none do true)
@$(call sudo,//runsc,do true)
do-tests: $(RUNTIME_BIN)
@$(RUNTIME_BIN) --rootless do true
@$(RUNTIME_BIN) --rootless -network=none do true
@sudo $(RUNTIME_BIN) do true
.PHONY: do-tests
arm-qemu-smoke-test: BAZEL_OPTIONS=--config=cross-aarch64
arm-qemu-smoke-test: load-arm-qemu
arm-qemu-smoke-test: $(RUNTIME_BIN) load-arm-qemu
export T=$$(mktemp -d --tmpdir release.XXXXXX); \
mkdir -p $$T/bin/arm64/ && \
$(call copy,//runsc:runsc,$$T/bin/arm64) && \
cp $(RUNTIME_BIN) $$T/bin/arm64 && \
docker run --rm -v $$T/bin/arm64/runsc:/workdir/initramfs/runsc gvisor.dev/images/arm-qemu
.PHONY: arm-qemu-smoke-test
@ -300,8 +311,13 @@ fsstress-test: load-basic $(RUNTIME_BIN)
# Specific containerd version tests.
containerd-test-%: load-basic_alpine load-basic_python load-basic_busybox load-basic_resolv load-basic_httpd load-basic_ubuntu $(RUNTIME_BIN)
@$(call install_runtime,$(RUNTIME),) # Clear flags.
@$(call sudo,tools/installers:containerd,$*)
@$(call sudo,tools/installers:shim)
@sudo tools/install_containerd.sh $*
ifeq (,$(STAGED_BINARIES))
@$(call sudocopy,//shim:containerd-shim-runsc-v1,"$$(dirname $$(which containerd))")
else
gsutil cat "$(STAGED_BINARIES)" | \
sudo tar -C "$$(dirname $$(which containerd))" -zxvf - containerd-shim-runsc-v1
endif
@$(call sudo,test/root:root_test,--runtime=$(RUNTIME) -test.v)
# The shim builds with containerd 1.3.9 and it's not backward compatible. Test
@ -323,7 +339,6 @@ containerd-tests: containerd-test-1.5.4
## BENCHMARKS_SUITE - name of the benchmark suite. See //tools/bigquery/bigquery.go.
## BENCHMARKS_UPLOAD - if true, upload benchmark data from the run.
## BENCHMARKS_OFFICIAL - marks the data as official.
## BENCHMARKS_PLATFORMS - platforms to run benchmarks (e.g. ptrace kvm).
## BENCHMARKS_FILTER - filter to be applied to the test suite.
## BENCHMARKS_OPTIONS - options to be passed to the test.
## BENCHMARKS_PROFILE - profile options to be passed to the test.
@ -335,7 +350,6 @@ BENCHMARKS_TABLE ?= benchmarks
BENCHMARKS_SUITE ?= ffmpeg
BENCHMARKS_UPLOAD ?= false
BENCHMARKS_OFFICIAL ?= false
BENCHMARKS_PLATFORMS ?= ptrace
BENCHMARKS_TARGETS := //test/benchmarks/media:ffmpeg_test
BENCHMARKS_FILTER := .
BENCHMARKS_OPTIONS := -test.benchtime=30s
@ -355,17 +369,18 @@ run_benchmark = \
export T=$$(mktemp --tmpdir logs.$(1).XXXXXX); \
if test "$(1)" = "runc"; then $(call sudo,$(BENCHMARKS_TARGETS),-runtime=$(1) $(BENCHMARKS_ARGS)) | tee $$T; fi; \
if test "$(1)" != "runc"; then $(call install_runtime,$(1),--profile $(2)); \
$(call sudo,$(BENCHMARKS_TARGETS),-runtime=$(1) $(BENCHMARKS_ARGS) $(BENCHMARKS_PROFILE)) | tee $$T; fi; \
$(call sudo,$(BENCHMARKS_TARGETS),-runtime=$(1) $(BENCHMARKS_ARGS) $(BENCHMARKS_PROFILE)) | tee $$T; fi; \
if test "$(BENCHMARKS_UPLOAD)" = "true"; then \
$(call run,tools/parsers:parser,parse --debug --file=$$T --runtime=$(1) --suite_name=$(BENCHMARKS_SUITE) --project=$(BENCHMARKS_PROJECT) --dataset=$(BENCHMARKS_DATASET) --table=$(BENCHMARKS_TABLE) --official=$(BENCHMARKS_OFFICIAL)); \
fi; \
rm -rf $$T)
benchmark-platforms: load-benchmarks $(RUNTIME_BIN) ## Runs benchmarks for runc and all given platforms in BENCHMARK_PLATFORMS.
@$(foreach PLATFORM,$(BENCHMARKS_PLATFORMS), \
$(call run_benchmark,$(PLATFORM),--platform=$(PLATFORM) $(BENCH_RUNTIME_ARGS) --vfs2) && \
$(call run_benchmark,$(PLATFORM)_vfs1,--platform=$(PLATFORM) $(BENCH_RUNTIME_ARGS)) && \
) true
benchmark-platforms: load-benchmarks $(RUNTIME_BIN) ## Runs benchmarks for runc and all platforms.
@set -xe; for PLATFORM in $$($(RUNTIME_BIN) help platforms); do \
export PLATFORM; \
$(call run_benchmark,$${PLATFORM},--platform=$${PLATFORM} $(BENCH_RUNTIME_ARGS) --vfs2); \
$(call run_benchmark,$${PLATFORM}_vfs1,--platform=$${PLATFORM} $(BENCH_RUNTIME_ARGS)); \
done
@$(call run_benchmark,runc)
.PHONY: benchmark-platforms

View File

@ -447,6 +447,14 @@ func Register(name string, platform Constructor) {
platforms[name] = platform
}
// List lists available platforms.
func List() (available []string) {
for name, _ := range platforms {
available = append(available, name)
}
return
}
// Lookup looks up the platform constructor by name.
func Lookup(name string) (Constructor, error) {
p, ok := platforms[name]

View File

@ -33,12 +33,7 @@ import (
var (
// runtime is the runtime to use for tests. This will be applied to all
// containers. Note that the default here ("runsc") corresponds to the
// default used by the installations. This is important, because the
// default installer for vm_tests (in tools/installers:head, invoked
// via tools/vm:defs.bzl) will install with this name. So without
// changing anything, tests should have a runsc runtime available to
// them. Otherwise installers should update the existing runtime
// instead of installing a new one.
// default used by the installations.
runtime = flag.String("runtime", "runsc", "specify which runtime to use")
// config is the default Docker daemon configuration path.

View File

@ -53,6 +53,7 @@ var (
partition = flag.Int("partition", 1, "partition number, this is 1-indexed")
totalPartitions = flag.Int("total_partitions", 1, "total number of partitions")
isRunningWithHostNet = flag.Bool("hostnet", false, "whether test is running with hostnet")
runscPath = flag.String("runsc", "", "path to runsc binary")
)
// IsCheckpointSupported returns the relevant command line flag.
@ -73,11 +74,14 @@ func ImageByName(name string) string {
// ConfigureExePath configures the executable for runsc in the test environment.
func ConfigureExePath() error {
path, err := FindFile("runsc/runsc")
if err != nil {
return err
if *runscPath == "" {
path, err := FindFile("runsc/runsc")
if err != nil {
return err
}
*runscPath = path
}
specutils.ExePath = path
specutils.ExePath = *runscPath
return nil
}

View File

@ -60,6 +60,7 @@ func Main(version string) {
// Help and flags commands are generated automatically.
help := cmd.NewHelp(subcommands.DefaultCommander)
help.Register(new(cmd.Syscalls))
help.Register(new(cmd.Platforms))
subcommands.Register(help, "")
subcommands.Register(subcommands.FlagsCommand(), "")

View File

@ -26,6 +26,7 @@ go_library(
"mitigate_extras.go",
"path.go",
"pause.go",
"platforms.go",
"ps.go",
"restore.go",
"resume.go",

55
runsc/cmd/platforms.go Normal file
View File

@ -0,0 +1,55 @@
// Copyright 2021 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"fmt"
"os"
"github.com/google/subcommands"
"gvisor.dev/gvisor/pkg/sentry/platform"
"gvisor.dev/gvisor/runsc/flag"
)
// Platforms implements subcommands.Command for the "platforms" command.
type Platforms struct{}
// Name implements subcommands.Command.Name.
func (*Platforms) Name() string {
return "platforms"
}
// Synopsis implements subcommands.Command.Synopsis.
func (*Platforms) Synopsis() string {
return "Print a list of available platforms."
}
// Usage implements subcommands.Command.Usage.
func (*Platforms) Usage() string {
return `platforms [options] - Print available platforms.
`
}
// SetFlags implements subcommands.Command.SetFlags.
func (*Platforms) SetFlags(f *flag.FlagSet) {}
// Execute implements subcommands.Command.Execute.
func (*Platforms) Execute(_ context.Context, f *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
for _, p := range platform.List() {
fmt.Fprintf(os.Stdout, "%s\n", p)
}
return subcommands.ExitSuccess
}

View File

@ -26,8 +26,7 @@ To run, use the Makefile:
- Additionally, you can benchmark several platforms in one command:
```
make benchmark-platforms BENCHMARKS_PLATFORMS=ptrace,kvm \
BENCHMARKS_TARGET=//path/to/target"
make benchmark-platforms BENCHMARKS_TARGET=//path/to/target"
```
The above command will install runtimes/run benchmarks on ptrace and kvm as well

View File

@ -51,7 +51,6 @@ var (
fuse = flag.Bool("fuse", false, "enable FUSE")
container = flag.Bool("container", false, "run tests in their own namespaces (user ns, network ns, etc), pretending to be root")
setupContainerPath = flag.String("setup-container", "", "path to setup_container binary (for use with --container)")
runscPath = flag.String("runsc", "", "path to runsc binary")
addUDSTree = flag.Bool("add-uds-tree", false, "expose a tree of UDS utilities for use in tests")
// TODO(gvisor.dev/issue/4572): properly support leak checking for runsc, and
@ -222,7 +221,7 @@ func runRunsc(tc gtest.TestCase, spec *specs.Spec) error {
// Current process doesn't have CAP_SYS_ADMIN, create user namespace and run
// as root inside that namespace to get it.
rArgs := append(args, "run", "--bundle", bundleDir, id)
cmd := exec.Command(*runscPath, rArgs...)
cmd := exec.Command(specutils.ExePath, rArgs...)
cmd.SysProcAttr = &unix.SysProcAttr{
Cloneflags: unix.CLONE_NEWUSER | unix.CLONE_NEWNS,
// Set current user/group as root inside the namespace.
@ -254,7 +253,7 @@ func runRunsc(tc gtest.TestCase, spec *specs.Spec) error {
dArgs := append([]string{}, args...)
dArgs = append(dArgs, "-alsologtostderr=true", "debug", "--stacks", id)
go func(dArgs []string) {
debug := exec.Command(*runscPath, dArgs...)
debug := exec.Command(specutils.ExePath, dArgs...)
debug.Stdout = os.Stdout
debug.Stderr = os.Stderr
debug.Run()
@ -272,7 +271,7 @@ func runRunsc(tc gtest.TestCase, spec *specs.Spec) error {
dArgs = append(args, "debug",
fmt.Sprintf("--signal=%d", unix.SIGTERM),
id)
signal := exec.Command(*runscPath, dArgs...)
signal := exec.Command(specutils.ExePath, dArgs...)
signal.Stdout = os.Stdout
signal.Stderr = os.Stderr
signal.Run()
@ -470,11 +469,10 @@ func main() {
log.SetLevel(log.Debug)
}
if *platform != "native" && *runscPath == "" {
if *platform != "native" {
if err := testutil.ConfigureExePath(); err != nil {
panic(err.Error())
}
*runscPath = specutils.ExePath
}
if *container && *setupContainerPath == "" {
setupContainer, err := testutil.FindFile("test/runner/setup_container/setup_container")

View File

@ -64,7 +64,7 @@ STARTUP_OPTIONS :=
BAZEL_OPTIONS :=
BAZEL := bazel $(STARTUP_OPTIONS)
BASE_OPTIONS := --color=no --curses=no
TEST_OPTIONS := $(BASE_OPTIONS) \
TEST_OPTIONS += $(BASE_OPTIONS) \
--test_output=errors \
--keep_going \
--verbose_failures=true \
@ -189,12 +189,13 @@ build_paths = \
| xargs -r -I {} bash -c 'test -e "{}" || exit 0; readlink -f "{}"' \
| xargs -r -I {} bash -c 'set -euo pipefail; $(2)')
clean = $(call header,CLEAN) && $(call wrapper,$(BAZEL) clean)
build = $(call header,BUILD $(1)) && $(call build_paths,$(1),echo {})
copy = $(call header,COPY $(1) $(2)) && $(call build_paths,$(1),cp -fa {} $(2))
run = $(call header,RUN $(1) $(2)) && $(call build_paths,$(1),{} $(2))
sudo = $(call header,SUDO $(1) $(2)) && $(call build_paths,$(1),sudo -E {} $(2))
test = $(call header,TEST $(1)) && $(call wrapper,$(BAZEL) test $(TEST_OPTIONS) $(1))
clean = $(call header,CLEAN) && $(call wrapper,$(BAZEL) clean)
build = $(call header,BUILD $(1)) && $(call build_paths,$(1),echo {})
copy = $(call header,COPY $(1) $(2)) && $(call build_paths,$(1),cp -fa {} $(2))
run = $(call header,RUN $(1) $(2)) && $(call build_paths,$(1),{} $(2))
sudo = $(call header,SUDO $(1) $(2)) && $(call build_paths,$(1),sudo -E {} $(2))
test = $(call header,TEST $(1)) && $(call wrapper,$(BAZEL) test $(TEST_OPTIONS) $(1))
sudocopy = $(call header,COPY $(1) $(2)) && $(call build_paths,$(1),sudo cp -fa {} $(2))
clean: ## Cleans the bazel cache.
@$(call clean)

View File

@ -1,32 +0,0 @@
# Installers for use by top-level scripts.
package(
default_visibility = ["//:sandbox"],
licenses = ["notice"],
)
sh_binary(
name = "head",
srcs = ["head.sh"],
data = [
"//runsc",
],
)
sh_binary(
name = "master",
srcs = ["master.sh"],
)
sh_binary(
name = "containerd",
srcs = ["containerd.sh"],
)
sh_binary(
name = "shim",
srcs = ["shim.sh"],
data = [
"//shim:containerd-shim-runsc-v1",
],
)

View File

@ -1,27 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Install our runtime.
runfiles=.
if [[ -d "$0.runfiles" ]]; then
runfiles="$0.runfiles"
fi
$(find -L "${runfiles}" -executable -type f -name runsc) install
# Restart docker.
if service docker status 2>/dev/null; then
service docker restart
fi

View File

@ -1,24 +0,0 @@
#!/bin/bash
# Copyright 2020 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -xeuo pipefail
# Find the images directory.
for images in $(find . -type d -name images); do
if [[ -f "${images}"/Makefile ]]; then
make -C "${images}" load-all-images
fi
done

View File

@ -1,34 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Install runsc from the master branch.
set -e
curl -fsSL https://gvisor.dev/archive.key | sudo apt-key add -
add-apt-repository "deb https://storage.googleapis.com/gvisor/releases release main"
while true; do
if (apt-get update && apt-get install -y runsc); then
break
fi
result=$?
if [[ $result -ne 100 ]]; then
exit $result
fi
done
runsc install
service docker restart

View File

@ -1,32 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Install all the shims.
#
# Note that containerd looks at the current executable directory
# in order to find the shim binary. So we need to check in order
# of preference. The local containerd installer will install to
# /usr/local, so we use that first.
if [[ -x /usr/local/bin/containerd ]]; then
containerd_install_dir=/usr/local/bin
else
containerd_install_dir=/usr/bin
fi
runfiles=.
if [[ -d "$0.runfiles" ]]; then
runfiles="$0.runfiles"
fi
find -L "${runfiles}" -executable -type f -name containerd-shim-runsc-v1 -exec cp -L {} "${containerd_install_dir}" \;