269 lines
9.1 KiB
C++
269 lines
9.1 KiB
C++
// Copyright 2020 The gVisor Authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#include <errno.h>
|
|
#include <signal.h>
|
|
#include <sys/syscall.h>
|
|
#include <sys/types.h>
|
|
#include <unistd.h>
|
|
|
|
#include <atomic>
|
|
|
|
#include "absl/time/clock.h"
|
|
#include "absl/time/time.h"
|
|
#include "test/util/cleanup.h"
|
|
#include "test/util/logging.h"
|
|
#include "test/util/memory_util.h"
|
|
#include "test/util/posix_error.h"
|
|
#include "test/util/test_util.h"
|
|
#include "test/util/thread_util.h"
|
|
|
|
namespace gvisor {
|
|
namespace testing {
|
|
|
|
namespace {
|
|
|
|
// This is the classic test case for memory fences on architectures with total
|
|
// store ordering; see e.g. Intel SDM Vol. 3A Sec. 8.2.3.4 "Loads May Be
|
|
// Reordered with Earlier Stores to Different Locations". In each iteration of
|
|
// the test, given two variables X and Y initially set to 0
|
|
// (MembarrierTestSharedState::local_var and remote_var in the code), two
|
|
// threads execute as follows:
|
|
//
|
|
// T1 T2
|
|
// -- --
|
|
//
|
|
// X = 1 Y = 1
|
|
// T1fence() T2fence()
|
|
// read Y read X
|
|
//
|
|
// On architectures where memory writes may be locally buffered by each CPU
|
|
// (essentially all architectures), if T1fence() and T2fence() are omitted or
|
|
// ineffective, it is possible for both T1 and T2 to read 0 because the memory
|
|
// write from the other CPU is not yet visible outside that CPU. T1fence() and
|
|
// T2fence() are expected to perform the necessary synchronization to restore
|
|
// sequential consistency: both threads agree on a order of memory accesses that
|
|
// is consistent with program order in each thread, such that at least one
|
|
// thread reads 1.
|
|
//
|
|
// In the NoMembarrier test, T1fence() and T2fence() are both ordinary memory
|
|
// fences establishing ordering between memory accesses before and after the
|
|
// fence (std::atomic_thread_fence). In all other test cases, T1fence() is not a
|
|
// memory fence at all, but only prevents compiler reordering of memory accesses
|
|
// (std::atomic_signal_fence); T2fence() is an invocation of the membarrier()
|
|
// syscall, which establishes ordering of memory accesses before and after the
|
|
// syscall on both threads.
|
|
|
|
template <typename F>
|
|
int DoMembarrierTestSide(std::atomic<int>* our_var,
|
|
std::atomic<int> const& their_var,
|
|
F const& test_fence) {
|
|
our_var->store(1, std::memory_order_relaxed);
|
|
test_fence();
|
|
return their_var.load(std::memory_order_relaxed);
|
|
}
|
|
|
|
struct MembarrierTestSharedState {
|
|
std::atomic<int64_t> remote_iter_cur;
|
|
std::atomic<int64_t> remote_iter_done;
|
|
std::atomic<int> local_var;
|
|
std::atomic<int> remote_var;
|
|
int remote_obs_of_local_var;
|
|
|
|
void Init() {
|
|
remote_iter_cur.store(-1, std::memory_order_relaxed);
|
|
remote_iter_done.store(-1, std::memory_order_relaxed);
|
|
}
|
|
};
|
|
|
|
// Special value for MembarrierTestSharedState::remote_iter_cur indicating that
|
|
// the remote thread should terminate.
|
|
constexpr int64_t kRemoteIterStop = -2;
|
|
|
|
// Must be async-signal-safe.
|
|
template <typename F>
|
|
void RunMembarrierTestRemoteSide(MembarrierTestSharedState* state,
|
|
F const& test_fence) {
|
|
int64_t i = 0;
|
|
int64_t cur;
|
|
while (true) {
|
|
while ((cur = state->remote_iter_cur.load(std::memory_order_acquire)) < i) {
|
|
if (cur == kRemoteIterStop) {
|
|
return;
|
|
}
|
|
// spin
|
|
}
|
|
state->remote_obs_of_local_var =
|
|
DoMembarrierTestSide(&state->remote_var, state->local_var, test_fence);
|
|
state->remote_iter_done.store(i, std::memory_order_release);
|
|
i++;
|
|
}
|
|
}
|
|
|
|
template <typename F>
|
|
void RunMembarrierTestLocalSide(MembarrierTestSharedState* state,
|
|
F const& test_fence) {
|
|
// On test completion, instruct the remote thread to terminate.
|
|
Cleanup cleanup_remote([&] {
|
|
state->remote_iter_cur.store(kRemoteIterStop, std::memory_order_relaxed);
|
|
});
|
|
|
|
int64_t i = 0;
|
|
absl::Time end = absl::Now() + absl::Seconds(5); // arbitrary test duration
|
|
while (absl::Now() < end) {
|
|
// Reset both vars to 0.
|
|
state->local_var.store(0, std::memory_order_relaxed);
|
|
state->remote_var.store(0, std::memory_order_relaxed);
|
|
// Instruct the remote thread to begin this iteration.
|
|
state->remote_iter_cur.store(i, std::memory_order_release);
|
|
// Perform our side of the test.
|
|
auto local_obs_of_remote_var =
|
|
DoMembarrierTestSide(&state->local_var, state->remote_var, test_fence);
|
|
// Wait for the remote thread to finish this iteration.
|
|
while (state->remote_iter_done.load(std::memory_order_acquire) < i) {
|
|
// spin
|
|
}
|
|
ASSERT_TRUE(local_obs_of_remote_var != 0 ||
|
|
state->remote_obs_of_local_var != 0);
|
|
i++;
|
|
}
|
|
}
|
|
|
|
TEST(MembarrierTest, NoMembarrier) {
|
|
MembarrierTestSharedState state;
|
|
state.Init();
|
|
|
|
ScopedThread remote_thread([&] {
|
|
RunMembarrierTestRemoteSide(
|
|
&state, [] { std::atomic_thread_fence(std::memory_order_seq_cst); });
|
|
});
|
|
RunMembarrierTestLocalSide(
|
|
&state, [] { std::atomic_thread_fence(std::memory_order_seq_cst); });
|
|
}
|
|
|
|
enum membarrier_cmd {
|
|
MEMBARRIER_CMD_QUERY = 0,
|
|
MEMBARRIER_CMD_GLOBAL = (1 << 0),
|
|
MEMBARRIER_CMD_GLOBAL_EXPEDITED = (1 << 1),
|
|
MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED = (1 << 2),
|
|
MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3),
|
|
MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4),
|
|
};
|
|
|
|
int membarrier(membarrier_cmd cmd, int flags) {
|
|
return syscall(SYS_membarrier, cmd, flags);
|
|
}
|
|
|
|
PosixErrorOr<int> SupportedMembarrierCommands() {
|
|
int cmds = membarrier(MEMBARRIER_CMD_QUERY, 0);
|
|
if (cmds < 0) {
|
|
if (errno == ENOSYS) {
|
|
// No commands are supported.
|
|
return 0;
|
|
}
|
|
return PosixError(errno, "membarrier(MEMBARRIER_CMD_QUERY) failed");
|
|
}
|
|
return cmds;
|
|
}
|
|
|
|
TEST(MembarrierTest, Global) {
|
|
SKIP_IF((ASSERT_NO_ERRNO_AND_VALUE(SupportedMembarrierCommands()) &
|
|
MEMBARRIER_CMD_GLOBAL) == 0);
|
|
|
|
Mapping m = ASSERT_NO_ERRNO_AND_VALUE(
|
|
MmapAnon(kPageSize, PROT_READ | PROT_WRITE, MAP_SHARED));
|
|
auto state = static_cast<MembarrierTestSharedState*>(m.ptr());
|
|
state->Init();
|
|
|
|
pid_t const child_pid = fork();
|
|
if (child_pid == 0) {
|
|
// In child process.
|
|
RunMembarrierTestRemoteSide(
|
|
state, [] { TEST_PCHECK(membarrier(MEMBARRIER_CMD_GLOBAL, 0) == 0); });
|
|
_exit(0);
|
|
}
|
|
// In parent process.
|
|
ASSERT_THAT(child_pid, SyscallSucceeds());
|
|
Cleanup cleanup_child([&] {
|
|
int status;
|
|
ASSERT_THAT(waitpid(child_pid, &status, 0),
|
|
SyscallSucceedsWithValue(child_pid));
|
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0)
|
|
<< " status " << status;
|
|
});
|
|
RunMembarrierTestLocalSide(
|
|
state, [] { std::atomic_signal_fence(std::memory_order_seq_cst); });
|
|
}
|
|
|
|
TEST(MembarrierTest, GlobalExpedited) {
|
|
constexpr int kRequiredCommands = MEMBARRIER_CMD_GLOBAL_EXPEDITED |
|
|
MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED;
|
|
SKIP_IF((ASSERT_NO_ERRNO_AND_VALUE(SupportedMembarrierCommands()) &
|
|
kRequiredCommands) != kRequiredCommands);
|
|
|
|
ASSERT_THAT(membarrier(MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED, 0),
|
|
SyscallSucceeds());
|
|
|
|
Mapping m = ASSERT_NO_ERRNO_AND_VALUE(
|
|
MmapAnon(kPageSize, PROT_READ | PROT_WRITE, MAP_SHARED));
|
|
auto state = static_cast<MembarrierTestSharedState*>(m.ptr());
|
|
state->Init();
|
|
|
|
pid_t const child_pid = fork();
|
|
if (child_pid == 0) {
|
|
// In child process.
|
|
RunMembarrierTestRemoteSide(state, [] {
|
|
TEST_PCHECK(membarrier(MEMBARRIER_CMD_GLOBAL_EXPEDITED, 0) == 0);
|
|
});
|
|
_exit(0);
|
|
}
|
|
// In parent process.
|
|
ASSERT_THAT(child_pid, SyscallSucceeds());
|
|
Cleanup cleanup_child([&] {
|
|
int status;
|
|
ASSERT_THAT(waitpid(child_pid, &status, 0),
|
|
SyscallSucceedsWithValue(child_pid));
|
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0)
|
|
<< " status " << status;
|
|
});
|
|
RunMembarrierTestLocalSide(
|
|
state, [] { std::atomic_signal_fence(std::memory_order_seq_cst); });
|
|
}
|
|
|
|
TEST(MembarrierTest, PrivateExpedited) {
|
|
constexpr int kRequiredCommands = MEMBARRIER_CMD_PRIVATE_EXPEDITED |
|
|
MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED;
|
|
SKIP_IF((ASSERT_NO_ERRNO_AND_VALUE(SupportedMembarrierCommands()) &
|
|
kRequiredCommands) != kRequiredCommands);
|
|
|
|
ASSERT_THAT(membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0),
|
|
SyscallSucceeds());
|
|
|
|
MembarrierTestSharedState state;
|
|
state.Init();
|
|
|
|
ScopedThread remote_thread([&] {
|
|
RunMembarrierTestRemoteSide(&state, [] {
|
|
TEST_PCHECK(membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0) == 0);
|
|
});
|
|
});
|
|
RunMembarrierTestLocalSide(
|
|
&state, [] { std::atomic_signal_fence(std::memory_order_seq_cst); });
|
|
}
|
|
|
|
} // namespace
|
|
|
|
} // namespace testing
|
|
} // namespace gvisor
|