Support releasing aarch64 builds.

This change works around an issue in rules_pkg, described here:
  https://github.com/bazelbuild/rules_pkg/pull/263

PiperOrigin-RevId: 350869030
This commit is contained in:
Adin Scannell 2021-01-08 18:01:05 -08:00 committed by gVisor bot
parent 70de1db82e
commit 0c99ab7090
12 changed files with 116 additions and 99 deletions

View File

@ -40,7 +40,10 @@ steps:
# Release workflow.
- <<: *common
label: ":ship: Release tests"
commands: make release
commands:
- make artifacts/x86_64
- make BAZEL_OPTIONS=--config=cross-aarch64 artifacts/aarch64
- make release
# Basic unit tests.
- <<: *common

View File

@ -404,20 +404,22 @@ website-deploy: website-push ## Deploy a new version of the website.
## Repository builders.
##
## This builds a local apt repository. The following variables may be set:
## RELEASE_ROOT - The repository root (default: "repo" directory).
## RELEASE_KEY - The repository GPG private key file (default: dummy key is created).
## RELEASE_NIGHTLY - Set to true if a nightly release (default: false).
## RELEASE_COMMIT - The commit or Change-Id for the release (needed for tag).
## RELEASE_NAME - The name of the release in the proper format (needed for tag).
## RELEASE_NOTES - The file containing release notes (needed for tag).
## RELEASE_ROOT - The repository root (default: "repo" directory).
## RELEASE_KEY - The repository GPG private key file (default: dummy key is created).
## RELEASE_ARTIFACTS - The release artifacts directory. May contain multiple.
## RELEASE_NIGHTLY - Set to true if a nightly release (default: false).
## RELEASE_COMMIT - The commit or Change-Id for the release (needed for tag).
## RELEASE_NAME - The name of the release in the proper format (needed for tag).
## RELEASE_NOTES - The file containing release notes (needed for tag).
##
RELEASE_ROOT := $(CURDIR)/repo
RELEASE_KEY := repo.key
RELEASE_NIGHTLY := false
RELEASE_COMMIT :=
RELEASE_NAME :=
RELEASE_NOTES :=
GPG_TEST_OPTIONS := $(shell if gpg --pinentry-mode loopback --version >/dev/null 2>&1; then echo --pinentry-mode loopback; fi)
RELEASE_ROOT := repo
RELEASE_KEY := repo.key
RELEASE_ARTIFACTS := artifacts
RELEASE_NIGHTLY := false
RELEASE_COMMIT :=
RELEASE_NAME :=
RELEASE_NOTES :=
GPG_TEST_OPTIONS := $(shell if gpg --pinentry-mode loopback --version >/dev/null 2>&1; then echo --pinentry-mode loopback; fi)
$(RELEASE_KEY):
@echo "WARNING: Generating a key for testing ($@); don't use this."
@ -433,15 +435,16 @@ $(RELEASE_KEY):
gpg --batch $(GPG_TEST_OPTIONS) --export-secret-keys --no-default-keyring --secret-keyring $$T > $@; \
rc=$$?; rm -f $$T $$C; exit $$rc
release: $(RELEASE_KEY) ## Builds a release.
@mkdir -p $(RELEASE_ROOT)
@export T=$$(mktemp -d --tmpdir release.XXXXXX); \
$(call copy,//runsc:runsc,$$T) && \
$(call copy,//shim/v1:gvisor-containerd-shim,$$T) && \
$(call copy,//shim/v2:containerd-shim-runsc-v1,$$T) && \
$(call copy,//debian:debian,$$T) && \
NIGHTLY=$(RELEASE_NIGHTLY) tools/make_release.sh $(RELEASE_KEY) $(RELEASE_ROOT) $$T/*; \
rc=$$?; rm -rf $$T; exit $$rc
$(RELEASE_ARTIFACTS)/%:
@mkdir -p $@
@$(call copy,//runsc:runsc,$@)
@$(call copy,//shim/v1:gvisor-containerd-shim,$@)
@$(call copy,//shim/v2:containerd-shim-runsc-v1,$@)
@$(call copy,//debian:debian,$@)
release: $(RELEASE_KEY) $(RELEASE_ARTIFACTS)/$(ARCH)
@rm -rf $(RELEASE_ROOT) && mkdir -p $(RELEASE_ROOT)
@NIGHTLY=$(RELEASE_NIGHTLY) tools/make_release.sh $(RELEASE_KEY) $(RELEASE_ROOT) $$(find $(RELEASE_ARTIFACTS) -type f)
.PHONY: release
tag: ## Creates and pushes a release tag.

View File

@ -40,10 +40,10 @@ http_archive(
# binaries of symbols, which we don't want.
"//tools:rules_go.patch",
],
sha256 = "b725e6497741d7fc2d55fcc29a276627d10e43fa5d0bb692692890ae30d98d00",
sha256 = "a515569b4903776eae90ac2696b34ee1dd45600cf9dfd7d16475e2df32867521",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.24.3/rules_go-v0.24.3.tar.gz",
"https://github.com/bazelbuild/rules_go/releases/download/v0.24.3/rules_go-v0.24.3.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.24.10/rules_go-v0.24.10.tar.gz",
"https://github.com/bazelbuild/rules_go/releases/download/v0.24.10/rules_go-v0.24.10.tar.gz",
],
)
@ -144,8 +144,8 @@ rbe_autoconfig(name = "rbe_default")
http_archive(
name = "rules_pkg",
sha256 = "5bdc04987af79bd27bc5b00fe30f59a858f77ffa0bd2d8143d5b31ad8b1bd71c",
url = "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.0/rules_pkg-0.2.0.tar.gz",
sha256 = "6b5969a7acd7b60c02f816773b06fcf32fbe8ba0c7919ccdc2df4f8fb923804a",
url = "https://github.com/bazelbuild/rules_pkg/releases/download/0.3.0/rules_pkg-0.3.0.tar.gz",
)
load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")

29
debian/BUILD vendored
View File

@ -1,4 +1,4 @@
load("//tools:defs.bzl", "pkg_deb", "pkg_tar")
load("//tools:defs.bzl", "pkg_deb", "pkg_tar", "select_arch", "version")
package(licenses = ["notice"])
@ -22,27 +22,16 @@ pkg_tar(
],
)
genrule(
name = "debian-version",
# Note that runsc must appear in the srcs parameter and not the tools
# parameter, otherwise it will not be stamped. This is reasonable, as tools
# may be encoded differently in the build graph (cached more aggressively
# because they are assumes to be hermetic).
srcs = ["//runsc"],
outs = ["version.txt"],
# Note that the little dance here is necessary because files in the $(SRCS)
# attribute are not executable by default, and we can't touch in place.
cmd = "cp $(location //runsc:runsc) $(@D)/runsc && \
chmod a+x $(@D)/runsc && \
$(@D)/runsc -version | grep version | sed 's/^[^0-9]*//' > $@ && \
rm -f $(@D)/runsc",
stamp = 1,
)
pkg_deb(
name = "debian",
architecture = "amd64",
out = "runsc-latest.deb",
architecture = select_arch(
amd64 = "amd64",
arm64 = "arm64",
),
changes = "runsc.changes",
data = ":debian-data",
deb = "runsc.deb",
# Note that the description_file will be flatten (all newlines removed),
# and therefore it is kept to a simple one-line description. The expected
# format for debian packages is "short summary\nLonger explanation of
@ -52,7 +41,7 @@ pkg_deb(
maintainer = "The gVisor Authors <gvisor-dev@googlegroups.com>",
package = "runsc",
postinst = "postinst.sh",
version_file = ":version.txt",
version_file = version,
visibility = [
"//visibility:public",
],

View File

@ -12,7 +12,8 @@ To download and install the latest release manually follow these steps:
```bash
(
set -e
URL=https://storage.googleapis.com/gvisor/releases/release/latest
ARCH=$(uname -m)
URL=https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH}
wget ${URL}/runsc ${URL}/runsc.sha512 \
${URL}/gvisor-containerd-shim ${URL}/gvisor-containerd-shim.sha512 \
${URL}/containerd-shim-runsc-v1 ${URL}/containerd-shim-runsc-v1.sha512
@ -29,7 +30,7 @@ To install gVisor as a Docker runtime, run the following commands:
```bash
/usr/local/bin/runsc install
sudo systemctl restart docker
sudo systemctl reload docker
docker run --rm --runtime=runsc hello-world
```
@ -81,13 +82,15 @@ latest release is recommended.
After selecting an appropriate release channel from the options below, proceed
to the preferred installation mechanism: manual or from an `apt` repository.
> Note: Older releases are still available but may not have an `${ARCH}`
> component in the URL. These release were available for `x86_64` only.
### HEAD
Binaries are available for every commit on the `master` branch, and are
available at the following URL:
`https://storage.googleapis.com/gvisor/releases/master/latest/runsc`
`https://storage.googleapis.com/gvisor/releases/master/latest/runsc.sha512`
`https://storage.googleapis.com/gvisor/releases/master/latest/${ARCH}`
You can use this link with the steps described in
[Install latest release](#install-latest).
@ -103,15 +106,14 @@ sudo add-apt-repository "deb https://storage.googleapis.com/gvisor/releases mast
Nightly releases are built most nights from the master branch, and are available
at the following URL:
`https://storage.googleapis.com/gvisor/releases/nightly/latest/runsc`
`https://storage.googleapis.com/gvisor/releases/nightly/latest/runsc.sha512`
`https://storage.googleapis.com/gvisor/releases/nightly/latest/${ARCH}`
You can use this link with the steps described in
[Install latest release](#install-latest).
Specific nightly releases can be found at:
`https://storage.googleapis.com/gvisor/releases/nightly/${yyyy-mm-dd}/runsc`
`https://storage.googleapis.com/gvisor/releases/nightly/${yyyy-mm-dd}/${ARCH}`
Note that a release may not be available for every day.
@ -125,7 +127,7 @@ sudo add-apt-repository "deb https://storage.googleapis.com/gvisor/releases nigh
The latest official release is available at the following URL:
`https://storage.googleapis.com/gvisor/releases/release/latest`
`https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH}`
You can use this link with the steps described in
[Install latest release](#install-latest).
@ -140,7 +142,7 @@ sudo add-apt-repository "deb https://storage.googleapis.com/gvisor/releases rele
A given release release is available at the following URL:
`https://storage.googleapis.com/gvisor/releases/release/${yyyymmdd}`
`https://storage.googleapis.com/gvisor/releases/release/${yyyymmdd}/${ARCH}`
You can use this link with the steps described in
[Install latest release](#install-latest).
@ -161,7 +163,7 @@ sudo add-apt-repository "deb https://storage.googleapis.com/gvisor/releases yyyy
A given point release is available at the following URL:
`https://storage.googleapis.com/gvisor/releases/release/${yyyymmdd}.${rc}`
`https://storage.googleapis.com/gvisor/releases/release/${yyyymmdd}.${rc}/${ARCH}`
You can use this link with the steps described in
[Install latest release](#install-latest).

View File

@ -160,6 +160,12 @@ bazel-image: load-default ## Ensures that the local builder exists.
@docker commit $(BUILDER_NAME) gvisor.dev/images/builder >&2
.PHONY: bazel-image
# Note: when starting the bazel server, we tie the life of the container to the
# bazel server's life, so that the container disappears naturally. We also call
# bazel shutdown prior to startup, to ensure that any existing bazel instance in
# the workspace (perhaps of a different architecture) stops. If the instance is
# compatible and the container is already running, then the wrapper if statement
# here will succeed, and we wouldn't have needed a new server at all.
ifneq (true,$(shell $(wrapper echo true)))
bazel-server: bazel-image ## Ensures that the server exists.
@$(call header,DOCKER RUN)
@ -171,7 +177,7 @@ bazel-server: bazel-image ## Ensures that the server exists.
--workdir "$(CURDIR)" \
$(DOCKER_RUN_OPTIONS) \
gvisor.dev/images/builder \
bash -c "set -x; tail -f --pid=\$$($(BAZEL) info server_pid) /dev/null" >&2
bash -c "set -x; $(BAZEL) shutdown; tail -f --pid=\$$($(BAZEL) info server_pid) /dev/null"
else
bazel-server:
@

View File

@ -37,3 +37,11 @@ config_setting(
},
visibility = ["//visibility:private"],
)
genrule(
name = "version",
outs = ["version.txt"],
cmd = "cat bazel-out/stable-status.txt | grep STABLE_VERSION | cut -d' ' -f2- >$@",
stamp = True,
visibility = ["//:sandbox"],
)

View File

@ -7,6 +7,7 @@ build_test = _build_test
bzl_library = _bzl_library
more_shards = 4
most_shards = 8
version = "//tools/bazeldefs:version"
def short_path(path):
return path

View File

@ -1,6 +1,7 @@
"""Packaging rules."""
load("@rules_pkg//:pkg.bzl", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar")
# N.B. We refer to pkg_deb_impl to avoid the macro, which cannot use select.
load("@rules_pkg//:pkg.bzl", _pkg_deb = "pkg_deb_impl", _pkg_tar = "pkg_tar")
pkg_deb = _pkg_deb
pkg_tar = _pkg_tar

View File

@ -8,7 +8,7 @@ change for Google-internal and bazel-compatible rules.
load("//tools/go_stateify:defs.bzl", "go_stateify")
load("//tools/go_marshal:defs.bzl", "go_marshal", "marshal_deps", "marshal_test_deps")
load("//tools/nogo:defs.bzl", "nogo_test")
load("//tools/bazeldefs:defs.bzl", _arch_genrule = "arch_genrule", _build_test = "build_test", _bzl_library = "bzl_library", _coreutil = "coreutil", _default_installer = "default_installer", _default_net_util = "default_net_util", _more_shards = "more_shards", _most_shards = "most_shards", _proto_library = "proto_library", _select_arch = "select_arch", _select_system = "select_system", _short_path = "short_path")
load("//tools/bazeldefs:defs.bzl", _arch_genrule = "arch_genrule", _build_test = "build_test", _bzl_library = "bzl_library", _coreutil = "coreutil", _default_installer = "default_installer", _default_net_util = "default_net_util", _more_shards = "more_shards", _most_shards = "most_shards", _proto_library = "proto_library", _select_arch = "select_arch", _select_system = "select_system", _short_path = "short_path", _version = "version")
load("//tools/bazeldefs:cc.bzl", _cc_binary = "cc_binary", _cc_flags_supplier = "cc_flags_supplier", _cc_grpc_library = "cc_grpc_library", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test", _cc_toolchain = "cc_toolchain", _gbenchmark = "gbenchmark", _grpcpp = "grpcpp", _gtest = "gtest", _vdso_linker_option = "vdso_linker_option")
load("//tools/bazeldefs:go.bzl", _gazelle = "gazelle", _go_binary = "go_binary", _go_embed_data = "go_embed_data", _go_grpc_and_proto_libraries = "go_grpc_and_proto_libraries", _go_library = "go_library", _go_path = "go_path", _go_proto_library = "go_proto_library", _go_test = "go_test", _select_goarch = "select_goarch", _select_goos = "select_goos")
load("//tools/bazeldefs:pkg.bzl", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar")
@ -27,6 +27,7 @@ short_path = _short_path
coreutil = _coreutil
more_shards = _more_shards
most_shards = _most_shards
version = _version
# C++ rules.
cc_binary = _cc_binary

View File

@ -18,9 +18,16 @@ if [[ "$#" -le 3 ]]; then
echo "usage: $0 <private-key> <suite> <root> <packages...>"
exit 1
fi
declare -r private_key=$(readlink -e "$1"); shift
declare -r suite="$1"; shift
declare -r root="$1"; shift
declare private_key
declare suite
declare root
private_key="$(readlink -e "$1")"
suite="$2"
root="$(readlink -m "$3")"
readonly private_key
readonly suite
readonly root
shift; shift; shift # For "$@" below.
# Ensure that we have the correct packages installed.
function apt_install() {
@ -56,9 +63,15 @@ mkdir -p "${release}"
# Create a temporary keyring, and ensure it is cleaned up.
# Using separate homedir allows us to install apt repositories multiple times
# using the same key. This is a limitation in GnuPG pre-2.1.
declare -r keyring=$(mktemp /tmp/keyringXXXXXX.gpg)
declare -r homedir=$(mktemp -d /tmp/homedirXXXXXX)
declare -r gpg_opts=("--no-default-keyring" "--secret-keyring" "${keyring}" "--homedir" "${homedir}")
declare keyring
declare homedir
declare gpg_opts
keyring="$(mktemp /tmp/keyringXXXXXX.gpg)"
homedir="$(mktemp -d /tmp/homedirXXXXXX)"
gpg_opts=("--no-default-keyring" "--secret-keyring" "${keyring}" "--homedir" "${homedir}")
readonly keyring
readonly homedir
readonly gpg_opts
cleanup() {
rm -rf "${keyring}" "${homedir}"
}
@ -73,40 +86,29 @@ gpg "${gpg_opts[@]}" --import "${private_key}" || \
# Copy the packages into the root.
for pkg in "$@"; do
if ! [[ -f "${pkg}" ]]; then
continue
fi
ext=${pkg##*.}
name=$(basename "${pkg}" ".${ext}")
arch=${name##*_}
if [[ "${name}" == "${arch}" ]]; then
continue # Not a regular package.
fi
if [[ "${pkg}" =~ ^.*\.deb$ ]]; then
# Extract from the debian file.
version=$(dpkg --info "${pkg}" | grep -E 'Version:' | cut -d':' -f2)
elif [[ "${pkg}" =~ ^.*\.changes$ ]]; then
# Extract from the changes file.
version=$(grep -E 'Version:' "${pkg}" | cut -d':' -f2)
else
# Unsupported file type.
echo "Unknown file type: ${pkg}"
exit 1
fi
# The package may already exist, in which case we leave it alone.
version=${version// /} # Trim whitespace.
destdir="${root}/pool/${version}/binary-${arch}"
target="${destdir}/${name}.${ext}"
if [[ -f "${target}" ]]; then
if [[ "${ext}" != "deb" ]]; then
continue
fi
# Extract package information.
name=$(basename "${pkg}" ".${ext}")
arch=$(dpkg --info "${pkg}" | grep 'Architecture:' | cut -d':' -f2)
version=$(dpkg --info "${pkg}" | grep 'Version:' | cut -d':' -f2)
arch=${arch// /} # Trim whitespace.
version=${version// /} # Ditto.
destdir="${root}/pool/${version}/binary-${arch}"
# Copy & sign the package.
mkdir -p "${destdir}"
cp -a "${pkg}" "${target}"
chmod 0644 "${target}"
if [[ "${ext}" == "deb" ]]; then
# We use [*] here to expand the gpg_opts array into a single shell-word.
dpkg-sig -g "${gpg_opts[*]}" --sign builder "${target}"
fi
cp -a -L "$(dirname "${pkg}")/${name}.deb" "${destdir}"
cp -a -L "$(dirname "${pkg}")/${name}.changes" "${destdir}"
chmod 0644 "${destdir}"/"${name}".*
# We use [*] here to expand the gpg_opts array into a single shell-word.
dpkg-sig -g "${gpg_opts[*]}" --sign builder "${destdir}/${name}.deb"
done
# Build the package list.

View File

@ -38,12 +38,13 @@ done
# install_raw installs raw artifacts.
install_raw() {
mkdir -p "${root}/$1"
for binary in "${binaries[@]}"; do
# Copy the raw file & generate a sha512sum.
# Copy the raw file & generate a sha512sum, sorted by architecture.
arch=$(file "${binary}" | cut -d',' -f2 | awk '{print $NF}' | tr '-' '_')
name=$(basename "${binary}")
cp -f "${binary}" "${root}/$1"
(cd "${root}/$1" && sha512sum "${name}" > "${name}.sha512")
mkdir -p "${root}/$1/${arch}"
cp -f "${binary}" "${root}/$1/${arch}"
(cd "${root}/$1/${arch}" && sha512sum "${name}" > "${name}.sha512")
done
}