From f1061aabaf57e2d3344129dcbcc4841cba0c2616 Mon Sep 17 00:00:00 2001 From: Nicolas Lacasse Date: Wed, 9 Oct 2019 11:10:08 -0700 Subject: [PATCH] Add blacklists for remaining runtime tests, and test that they parse correctly. PiperOrigin-RevId: 273781112 --- test/runtimes/BUILD | 13 ++- test/runtimes/blacklist_go1.12.csv | 16 +++ test/runtimes/blacklist_java11.csv | 126 +++++++++++++++++++++++ test/runtimes/blacklist_nodejs12.4.0.csv | 2 +- test/runtimes/blacklist_php7.3.6.csv | 29 ++++++ test/runtimes/blacklist_python3.7.3.csv | 27 +++++ test/runtimes/blacklist_test.go | 37 +++++++ test/runtimes/build_defs.bzl | 17 ++- 8 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 test/runtimes/blacklist_go1.12.csv create mode 100644 test/runtimes/blacklist_java11.csv create mode 100644 test/runtimes/blacklist_php7.3.6.csv create mode 100644 test/runtimes/blacklist_python3.7.3.csv create mode 100644 test/runtimes/blacklist_test.go diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD index 1cde74cfc..2e125525b 100644 --- a/test/runtimes/BUILD +++ b/test/runtimes/BUILD @@ -1,6 +1,6 @@ # These packages are used to run language runtime tests inside gVisor sandboxes. -load("@io_bazel_rules_go//go:def.bzl", "go_binary") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_test") load("//test/runtimes:build_defs.bzl", "runtime_test") package(licenses = ["notice"]) @@ -16,11 +16,13 @@ go_binary( ) runtime_test( + blacklist_file = "blacklist_go1.12.csv", image = "gcr.io/gvisor-presubmit/go1.12", lang = "go", ) runtime_test( + blacklist_file = "blacklist_java11.csv", image = "gcr.io/gvisor-presubmit/java11", lang = "java", ) @@ -32,11 +34,20 @@ runtime_test( ) runtime_test( + blacklist_file = "blacklist_php7.3.6.csv", image = "gcr.io/gvisor-presubmit/php7.3.6", lang = "php", ) runtime_test( + blacklist_file = "blacklist_python3.7.3.csv", image = "gcr.io/gvisor-presubmit/python3.7.3", lang = "python", ) + +go_test( + name = "blacklist_test", + size = "small", + srcs = ["blacklist_test.go"], + embed = [":runner"], +) diff --git a/test/runtimes/blacklist_go1.12.csv b/test/runtimes/blacklist_go1.12.csv new file mode 100644 index 000000000..8c8ae0c5d --- /dev/null +++ b/test/runtimes/blacklist_go1.12.csv @@ -0,0 +1,16 @@ +test name,bug id,comment +cgo_errors,,FLAKY +cgo_test,,FLAKY +go_test:cmd/go,,FLAKY +go_test:cmd/vendor/golang.org/x/sys/unix,b/118783622,/dev devices missing +go_test:net,b/118784196,socket: invalid argument. Works as intended: see bug. +go_test:os,b/118780122,we have a pollable filesystem but that's a surprise +go_test:os/signal,b/118780860,/dev/pts not properly supported +go_test:runtime,b/118782341,sigtrap not reported or caught or something +go_test:syscall,b/118781998,bad bytes -- bad mem addr +race,b/118782931,thread sanitizer. Works as intended: b/62219744. +runtime:cpu124,b/118778254,segmentation fault +test:0_1,,FLAKY +testasan,, +testcarchive,b/118782924,no sigpipe +testshared,,FLAKY diff --git a/test/runtimes/blacklist_java11.csv b/test/runtimes/blacklist_java11.csv new file mode 100644 index 000000000..c012e5a56 --- /dev/null +++ b/test/runtimes/blacklist_java11.csv @@ -0,0 +1,126 @@ +test name,bug id,comment +com/sun/crypto/provider/Cipher/PBE/PKCS12Cipher.java,,Fails in Docker +com/sun/jdi/NashornPopFrameTest.java,, +com/sun/jdi/ProcessAttachTest.java,, +com/sun/management/HotSpotDiagnosticMXBean/CheckOrigin.java,,Fails in Docker +com/sun/management/OperatingSystemMXBean/GetCommittedVirtualMemorySize.java,, +com/sun/management/UnixOperatingSystemMXBean/GetMaxFileDescriptorCount.sh,, +com/sun/tools/attach/AttachSelf.java,, +com/sun/tools/attach/BasicTests.java,, +com/sun/tools/attach/PermissionTest.java,, +com/sun/tools/attach/StartManagementAgent.java,, +com/sun/tools/attach/TempDirTest.java,, +com/sun/tools/attach/modules/Driver.java,, +java/lang/Character/CheckScript.java,,Fails in Docker +java/lang/Character/CheckUnicode.java,,Fails in Docker +java/lang/Class/GetPackageBootLoaderChildLayer.java,, +java/lang/ClassLoader/nativeLibrary/NativeLibraryTest.java,,Fails in Docker +java/lang/String/nativeEncoding/StringPlatformChars.java,, +java/net/DatagramSocket/ReuseAddressTest.java,, +java/net/DatagramSocket/SendDatagramToBadAddress.java,b/78473345, +java/net/Inet4Address/PingThis.java,, +java/net/InterfaceAddress/NetworkPrefixLength.java,b/78507103, +java/net/MulticastSocket/MulticastTTL.java,, +java/net/MulticastSocket/Promiscuous.java,, +java/net/MulticastSocket/SetLoopbackMode.java,, +java/net/MulticastSocket/SetTTLAndGetTTL.java,, +java/net/MulticastSocket/Test.java,, +java/net/MulticastSocket/TestDefaults.java,, +java/net/MulticastSocket/TimeToLive.java,, +java/net/NetworkInterface/NetworkInterfaceStreamTest.java,, +java/net/Socket/SetSoLinger.java,b/78527327,SO_LINGER is not yet supported +java/net/Socket/TrafficClass.java,b/78527818,Not supported on gVisor +java/net/Socket/UrgentDataTest.java,b/111515323, +java/net/Socket/setReuseAddress/Basic.java,b/78519214,SO_REUSEADDR enabled by default +java/net/SocketOption/OptionsTest.java,,Fails in Docker +java/net/SocketOption/TcpKeepAliveTest.java,, +java/net/SocketPermission/SocketPermissionTest.java,, +java/net/URLConnection/6212146/TestDriver.java,,Fails in Docker +java/net/httpclient/RequestBuilderTest.java,,Fails in Docker +java/net/httpclient/ShortResponseBody.java,, +java/net/httpclient/ShortResponseBodyWithRetry.java,, +java/nio/channels/AsyncCloseAndInterrupt.java,, +java/nio/channels/AsynchronousServerSocketChannel/Basic.java,, +java/nio/channels/AsynchronousSocketChannel/Basic.java,b/77921528,SO_KEEPALIVE is not settable +java/nio/channels/DatagramChannel/BasicMulticastTests.java,, +java/nio/channels/DatagramChannel/SocketOptionTests.java,,Fails in Docker +java/nio/channels/DatagramChannel/UseDGWithIPv6.java,, +java/nio/channels/FileChannel/directio/DirectIOTest.java,,Fails in Docker +java/nio/channels/Selector/OutOfBand.java,, +java/nio/channels/Selector/SelectWithConsumer.java,,Flaky +java/nio/channels/ServerSocketChannel/SocketOptionTests.java,, +java/nio/channels/SocketChannel/LingerOnClose.java,, +java/nio/channels/SocketChannel/SocketOptionTests.java,b/77965901, +java/nio/channels/spi/SelectorProvider/inheritedChannel/InheritedChannelTest.java,,Fails in Docker +java/rmi/activation/Activatable/extLoadedImpl/ext.sh,, +java/rmi/transport/checkLeaseInfoLeak/CheckLeaseLeak.java,, +java/text/Format/NumberFormat/CurrencyFormat.java,,Fails in Docker +java/text/Format/NumberFormat/CurrencyFormat.java,,Fails in Docker +java/util/Calendar/JapaneseEraNameTest.java,, +java/util/Currency/CurrencyTest.java,,Fails in Docker +java/util/Currency/ValidateISO4217.java,,Fails in Docker +java/util/Locale/LSRDataTest.java,, +java/util/concurrent/locks/Lock/TimedAcquireLeak.java,, +java/util/jar/JarFile/mrjar/MultiReleaseJarAPI.java,,Fails in Docker +java/util/logging/LogManager/Configuration/updateConfiguration/SimpleUpdateConfigWithInputStreamTest.java,, +java/util/logging/TestLoggerWeakRefLeak.java,, +javax/imageio/AppletResourceTest.java,, +javax/management/security/HashedPasswordFileTest.java,, +javax/net/ssl/SSLSession/JSSERenegotiate.java,,Fails in Docker +javax/sound/sampled/AudioInputStream/FrameLengthAfterConversion.java,, +jdk/jfr/event/runtime/TestNetworkUtilizationEvent.java,, +jdk/jfr/event/runtime/TestThreadParkEvent.java,, +jdk/jfr/event/sampling/TestNative.java,, +jdk/jfr/jcmd/TestJcmdChangeLogLevel.java,, +jdk/jfr/jcmd/TestJcmdConfigure.java,, +jdk/jfr/jcmd/TestJcmdDump.java,, +jdk/jfr/jcmd/TestJcmdDumpGeneratedFilename.java,, +jdk/jfr/jcmd/TestJcmdDumpLimited.java,, +jdk/jfr/jcmd/TestJcmdDumpPathToGCRoots.java,, +jdk/jfr/jcmd/TestJcmdLegacy.java,, +jdk/jfr/jcmd/TestJcmdSaveToFile.java,, +jdk/jfr/jcmd/TestJcmdStartDirNotExist.java,, +jdk/jfr/jcmd/TestJcmdStartInvaldFile.java,, +jdk/jfr/jcmd/TestJcmdStartPathToGCRoots.java,, +jdk/jfr/jcmd/TestJcmdStartStopDefault.java,, +jdk/jfr/jcmd/TestJcmdStartWithOptions.java,, +jdk/jfr/jcmd/TestJcmdStartWithSettings.java,, +jdk/jfr/jcmd/TestJcmdStopInvalidFile.java,, +jdk/jfr/jvm/TestJfrJavaBase.java,, +jdk/jfr/startupargs/TestStartRecording.java,, +jdk/modules/incubator/ImageModules.java,, +jdk/net/Sockets/ExtOptionTest.java,, +jdk/net/Sockets/QuickAckTest.java,, +lib/security/cacerts/VerifyCACerts.java,, +sun/management/jmxremote/bootstrap/CustomLauncherTest.java,, +sun/management/jmxremote/bootstrap/JvmstatCountersTest.java,, +sun/management/jmxremote/bootstrap/LocalManagementTest.java,, +sun/management/jmxremote/bootstrap/RmiRegistrySslTest.java,, +sun/management/jmxremote/bootstrap/RmiSslBootstrapTest.sh,, +sun/management/jmxremote/startstop/JMXStartStopTest.java,, +sun/management/jmxremote/startstop/JMXStatusPerfCountersTest.java,, +sun/management/jmxremote/startstop/JMXStatusTest.java,, +sun/text/resources/LocaleDataTest.java,, +sun/tools/jcmd/TestJcmdSanity.java,, +sun/tools/jhsdb/AlternateHashingTest.java,, +sun/tools/jhsdb/BasicLauncherTest.java,, +sun/tools/jhsdb/HeapDumpTest.java,, +sun/tools/jhsdb/heapconfig/JMapHeapConfigTest.java,, +sun/tools/jinfo/BasicJInfoTest.java,, +sun/tools/jinfo/JInfoTest.java,, +sun/tools/jmap/BasicJMapTest.java,, +sun/tools/jstack/BasicJStackTest.java,, +sun/tools/jstack/DeadlockDetectionTest.java,, +sun/tools/jstatd/TestJstatdExternalRegistry.java,, +sun/tools/jstatd/TestJstatdPort.java,,Flaky +sun/tools/jstatd/TestJstatdPortAndServer.java,,Flaky +sun/util/calendar/zi/TestZoneInfo310.java,, +tools/jar/modularJar/Basic.java,, +tools/jar/multiRelease/Basic.java,, +tools/jimage/JImageExtractTest.java,, +tools/jimage/JImageTest.java,, +tools/jlink/JLinkTest.java,, +tools/jlink/plugins/IncludeLocalesPluginTest.java,, +tools/jmod/hashes/HashesTest.java,, +tools/launcher/BigJar.java,b/111611473, +tools/launcher/modules/patch/systemmodules/PatchSystemModules.java,, diff --git a/test/runtimes/blacklist_nodejs12.4.0.csv b/test/runtimes/blacklist_nodejs12.4.0.csv index 9135d763c..4ab4e2927 100644 --- a/test/runtimes/blacklist_nodejs12.4.0.csv +++ b/test/runtimes/blacklist_nodejs12.4.0.csv @@ -15,7 +15,7 @@ parallel/test-dgram-create-socket-handle-fd.js,b/132447238, parallel/test-dgram-createSocket-type.js,b/68847739, parallel/test-dgram-socket-buffer-size.js,b/68847921, parallel/test-fs-access.js,, -parallel/test-fs-write-stream-double-close.js +parallel/test-fs-write-stream-double-close.js,, parallel/test-fs-write-stream-throw-type-error.js,b/110226209, parallel/test-fs-write-stream.js,, parallel/test-http2-respond-file-error-pipe-offset.js,, diff --git a/test/runtimes/blacklist_php7.3.6.csv b/test/runtimes/blacklist_php7.3.6.csv new file mode 100644 index 000000000..456bf7487 --- /dev/null +++ b/test/runtimes/blacklist_php7.3.6.csv @@ -0,0 +1,29 @@ +test name,bug id,comment +ext/intl/tests/bug77895.phpt,, +ext/intl/tests/dateformat_bug65683_2.phpt,, +ext/mbstring/tests/bug76319.phpt,, +ext/mbstring/tests/bug76958.phpt,, +ext/mbstring/tests/bug77025.phpt,, +ext/mbstring/tests/bug77165.phpt,, +ext/mbstring/tests/bug77454.phpt,, +ext/mbstring/tests/mb_convert_encoding_leak.phpt,, +ext/mbstring/tests/mb_strrpos_encoding_3rd_param.phpt,, +ext/standard/tests/file/filetype_variation.phpt,, +ext/standard/tests/file/fopen_variation19.phpt,, +ext/standard/tests/file/php_fd_wrapper_01.phpt,, +ext/standard/tests/file/php_fd_wrapper_02.phpt,, +ext/standard/tests/file/php_fd_wrapper_03.phpt,, +ext/standard/tests/file/php_fd_wrapper_04.phpt,, +ext/standard/tests/file/realpath_bug77484.phpt,, +ext/standard/tests/file/rename_variation.phpt,b/68717309, +ext/standard/tests/file/symlink_link_linkinfo_is_link_variation4.phpt,, +ext/standard/tests/file/symlink_link_linkinfo_is_link_variation8.phpt,, +ext/standard/tests/general_functions/escapeshellarg_bug71270.phpt,, +ext/standard/tests/general_functions/escapeshellcmd_bug71270.phpt,, +ext/standard/tests/network/bug20134.phpt,, +tests/output/stream_isatty_err.phpt,b/68720279, +tests/output/stream_isatty_in-err.phpt,b/68720282, +tests/output/stream_isatty_in-out-err.phpt,, +tests/output/stream_isatty_in-out.phpt,b/68720299, +tests/output/stream_isatty_out-err.phpt,b/68720311, +tests/output/stream_isatty_out.phpt,b/68720325, diff --git a/test/runtimes/blacklist_python3.7.3.csv b/test/runtimes/blacklist_python3.7.3.csv new file mode 100644 index 000000000..2b9947212 --- /dev/null +++ b/test/runtimes/blacklist_python3.7.3.csv @@ -0,0 +1,27 @@ +test name,bug id,comment +test_asynchat,b/76031995,SO_REUSEADDR +test_asyncio,,Fails on Docker. +test_asyncore,b/76031995,SO_REUSEADDR +test_epoll,, +test_fcntl,,fcntl invalid argument -- artificial test to make sure something works in 64 bit mode. +test_ftplib,,Fails in Docker +test_httplib,b/76031995,SO_REUSEADDR +test_imaplib,, +test_logging,, +test_multiprocessing_fork,,Flaky. Sometimes times out. +test_multiprocessing_forkserver,,Flaky. Sometimes times out. +test_multiprocessing_main_handling,,Flaky. Sometimes times out. +test_multiprocessing_spawn,,Flaky. Sometimes times out. +test_nntplib,b/76031995,tests should not set SO_REUSEADDR +test_poplib,,Fails on Docker +test_posix,b/76174079,posix.sched_get_priority_min not implemented + posix.sched_rr_get_interval not permitted +test_pty,b/76157709,out of pty devices +test_readline,b/76157709,out of pty devices +test_resource,b/76174079, +test_selectors,b/76116849,OSError not raised with epoll +test_smtplib,b/76031995,SO_REUSEADDR and unclosed sockets +test_socket,b/75983380, +test_ssl,b/76031995,SO_REUSEADDR +test_subprocess,, +test_support,b/76031995,SO_REUSEADDR +test_telnetlib,b/76031995,SO_REUSEADDR diff --git a/test/runtimes/blacklist_test.go b/test/runtimes/blacklist_test.go new file mode 100644 index 000000000..52f49b984 --- /dev/null +++ b/test/runtimes/blacklist_test.go @@ -0,0 +1,37 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + "testing" +) + +func TestMain(m *testing.M) { + flag.Parse() + os.Exit(m.Run()) +} + +// Test that the blacklist parses without error. +func TestBlacklists(t *testing.T) { + bl, err := getBlacklist() + if err != nil { + t.Fatalf("error parsing blacklist: %v", err) + } + if *blacklistFile != "" && len(bl) == 0 { + t.Errorf("got empty blacklist for file %q", blacklistFile) + } +} diff --git a/test/runtimes/build_defs.bzl b/test/runtimes/build_defs.bzl index 7edd12c17..7c11624b4 100644 --- a/test/runtimes/build_defs.bzl +++ b/test/runtimes/build_defs.bzl @@ -1,4 +1,6 @@ -"""Defines a rule for runsc test targets.""" +"""Defines a rule for runtime test targets.""" + +load("@io_bazel_rules_go//go:def.bzl", "go_test") # runtime_test is a macro that will create targets to run the given test target # with different runtime options. @@ -21,6 +23,9 @@ def runtime_test( args += ["--blacklist_file", "test/runtimes/" + blacklist_file] data += [blacklist_file] + # Add a test that the blacklist parses correctly. + blacklist_test(lang, blacklist_file) + sh_test( name = lang + "_test", srcs = ["runner.sh"], @@ -35,6 +40,16 @@ def runtime_test( ], ) +def blacklist_test(lang, blacklist_file): + """Test that a blacklist parses correctly.""" + go_test( + name = lang + "_blacklist_test", + embed = [":runner"], + srcs = ["blacklist_test.go"], + args = ["--blacklist_file", "test/runtimes/" + blacklist_file], + data = [blacklist_file], + ) + def sh_test(**kwargs): """Wraps the standard sh_test.""" native.sh_test(