go
/
gvisor
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Consolidate most synchronization primitive linknames in the sync package. 

PiperOrigin-RevId: 345359823
This commit is contained in:
Jamie Liu 2020-12-02 19:06:41 -08:00 committed by gVisor bot
parent bdaae08ee2
commit ed8bdf461b
21 changed files with 161 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/sleep/BUILD
View File

@ -5,10 +5,6 @@ package(licenses = ["notice"])
go_library(
name = "sleep",
srcs = [
"commit_amd64.s",
"commit_arm64.s",
"commit_asm.go",
"commit_noasm.go",
"sleep_unsafe.go",
],
visibility = ["//:sandbox"],

35
pkg/sleep/commit_amd64.s
View File

@ -1,35 +0,0 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "textflag.h"
#define preparingG 1
// See commit_noasm.go for a description of commitSleep.
//
// func commitSleep(g uintptr, waitingG *uintptr) bool
TEXT ·commitSleep(SB),NOSPLIT,$0-24
MOVQ waitingG+8(FP), CX
MOVQ g+0(FP), DX
// Store the G in waitingG if it's still preparingG. If it's anything
// else it means a waker has aborted the sleep.
MOVQ $preparingG, AX
LOCK
CMPXCHGQ DX, 0(CX)
SETEQ AX
MOVB AX, ret+16(FP)
RET

38
pkg/sleep/commit_arm64.s
View File

@ -1,38 +0,0 @@
// Copyright 2019 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "textflag.h"
#define preparingG 1
// See commit_noasm.go for a description of commitSleep.
//
// func commitSleep(g uintptr, waitingG *uintptr) bool
TEXT ·commitSleep(SB),NOSPLIT,$0-24
MOVD waitingG+8(FP), R0
MOVD $preparingG, R1
MOVD G+0(FP), R2
// Store the G in waitingG if it's still preparingG. If it's anything
// else it means a waker has aborted the sleep.
again:
LDAXR (R0), R3
CMP R1, R3
BNE ok
STLXR R2, (R0), R3
CBNZ R3, again
ok:
CSET EQ, R0
MOVB R0, ret+16(FP)
RET

20
pkg/sleep/commit_asm.go
View File

@ -1,20 +0,0 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build amd64 arm64
package sleep
// See commit_noasm.go for a description of commitSleep.
func commitSleep(g uintptr, waitingG *uintptr) bool

33
pkg/sleep/commit_noasm.go
View File

@ -1,33 +0,0 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !race
// +build !amd64,!arm64
package sleep
import "sync/atomic"
// commitSleep signals to wakers that the given g is now sleeping. Wakers can
// then fetch it and wake it.
//
// The commit may fail if wakers have been asserted after our last check, in
// which case they will have set s.waitingG to zero.
//
// It is written in assembly because it is called from g0, so it doesn't have
// a race context.
func commitSleep(g uintptr, waitingG *uintptr) bool {
// Try to store the G so that wakers know who to wake.
return atomic.CompareAndSwapUintptr(waitingG, preparingG, g)
}

15
pkg/sleep/empty.s
View File

@ -1,15 +0,0 @@
// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Empty assembly file so empty func definitions work.

27
pkg/sleep/sleep_unsafe.go
View File

@ -12,11 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.11
// +build !go1.17
// Check go:linkname function signatures when updating Go version.
// Package sleep allows goroutines to efficiently sleep on multiple sources of
// notifications (wakers). It offers O(1) complexity, which is different from
// multi-channel selects which have O(n) complexity (where n is the number of
@ -91,12 +86,6 @@ var (
assertedSleeper Sleeper
)
//go:linkname gopark runtime.gopark
func gopark(unlockf func(uintptr, *uintptr) bool, wg *uintptr, reason uint8, traceEv byte, traceskip int)
//go:linkname goready runtime.goready
func goready(g uintptr, traceskip int)
// Sleeper allows a goroutine to sleep and receive wake up notifications from
// Wakers in an efficient way.
//
@ -189,7 +178,7 @@ func (s *Sleeper) nextWaker(block bool) *Waker {
// See:runtime2.go in the go runtime package for
// the values to pass as the waitReason here.
const waitReasonSelect = 9
gopark(commitSleep, &s.waitingG, waitReasonSelect, traceEvGoBlockSelect, 0)
sync.Gopark(commitSleep, unsafe.Pointer(&s.waitingG), sync.WaitReasonSelect, sync.TraceEvGoBlockSelect, 0)
}
// Pull the shared list out and reverse it in the local
@ -212,6 +201,18 @@ func (s *Sleeper) nextWaker(block bool) *Waker {
return w
}
// commitSleep signals to wakers that the given g is now sleeping. Wakers can
// then fetch it and wake it.
//
// The commit may fail if wakers have been asserted after our last check, in
// which case they will have set s.waitingG to zero.
//
//go:norace
//go:nosplit
func commitSleep(g uintptr, waitingG unsafe.Pointer) bool {
return sync.RaceUncheckedAtomicCompareAndSwapUintptr((*uintptr)(waitingG), preparingG, g)
}
// Fetch fetches the next wake-up notification. If a notification is immediately
// available, it is returned right away. Otherwise, the behavior depends on the
// value of 'block': if true, the current goroutine blocks until a notification
@ -311,7 +312,7 @@ func (s *Sleeper) enqueueAssertedWaker(w *Waker) {
case 0, preparingG:
default:
// We managed to get a G. Wake it up.
goready(g, 0)
sync.Goready(g, 0)
}
}

8
pkg/sync/BUILD
View File

@ -33,15 +33,17 @@ go_library(
"aliases.go",
"checklocks_off_unsafe.go",
"checklocks_on_unsafe.go",
"memmove_unsafe.go",
"goyield_go113_unsafe.go",
"goyield_unsafe.go",
"mutex_unsafe.go",
"nocopy.go",
"norace_unsafe.go",
"race_amd64.s",
"race_arm64.s",
"race_unsafe.go",
"runtime_unsafe.go",
"rwmutex_unsafe.go",
"seqcount.go",
"spin_legacy_unsafe.go",
"spin_unsafe.go",
"sync.go",
],
marshal = False,

7
pkg/sync/spin_legacy_unsafe.go → pkg/sync/goyield_go113_unsafe.go
View File

@ -10,15 +10,8 @@ package sync
import (
"runtime"
_ "unsafe" // for go:linkname
)
//go:linkname canSpin sync.runtime_canSpin
func canSpin(i int) bool
//go:linkname doSpin sync.runtime_doSpin
func doSpin()
func goyield() {
// goyield is not available until Go 1.14.
runtime.Gosched()

6
pkg/sync/spin_unsafe.go → pkg/sync/goyield_unsafe.go
View File

@ -14,11 +14,5 @@ import (
_ "unsafe" // for go:linkname
)
//go:linkname canSpin sync.runtime_canSpin
func canSpin(i int) bool
//go:linkname doSpin sync.runtime_doSpin
func doSpin()
//go:linkname goyield runtime.goyield
func goyield()

28
pkg/sync/memmove_unsafe.go
View File

@ -1,28 +0,0 @@
// Copyright 2019 The gVisor Authors.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.12
// +build !go1.17
// Check go:linkname function signatures when updating Go version.
package sync
import (
"unsafe"
)
//go:linkname memmove runtime.memmove
//go:noescape
func memmove(to, from unsafe.Pointer, n uintptr)
// Memmove is exported for SeqAtomicLoad/SeqAtomicTryLoad<T>, which can't
// define it because go_generics can't update the go:linkname annotation.
// Furthermore, go:linkname silently doesn't work if the local name is exported
// (this is of course undocumented), which is why this indirection is
// necessary.
func Memmove(to, from unsafe.Pointer, n uintptr) {
memmove(to, from, n)
}

11
pkg/sync/norace_unsafe.go
View File

@ -8,6 +8,7 @@
package sync
import (
"sync/atomic"
"unsafe"
)
@ -33,3 +34,13 @@ func RaceRelease(addr unsafe.Pointer) {
// RaceReleaseMerge has the same semantics as runtime.RaceReleaseMerge.
func RaceReleaseMerge(addr unsafe.Pointer) {
}
// RaceUncheckedAtomicCompareAndSwapUintptr is equivalent to
// sync/atomic.CompareAndSwapUintptr, but is not checked by the race detector.
// This is necessary when implementing gopark callbacks, since no race context
// is available during their execution.
func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool {
// Use atomic.CompareAndSwapUintptr outside of race builds for
// inlinability.
return atomic.CompareAndSwapUintptr(ptr, old, new)
}

17
pkg/syncevent/waiter_amd64.s → pkg/sync/race_amd64.s
View File

@ -12,21 +12,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// +build race
// +build amd64
#include "textflag.h"
// See waiter_noasm_unsafe.go for a description of waiterUnlock.
//
// func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool
TEXT ·waiterUnlock(SB),NOSPLIT,$0-24
// func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
TEXT ·RaceUncheckedAtomicCompareAndSwapUintptr(SB),NOSPLIT,$0-25
MOVQ ptr+0(FP), DI
MOVQ wg+8(FP), SI
MOVQ old+8(FP), AX
MOVQ new+16(FP), SI
MOVQ $·preparingG(SB), AX
LOCK
CMPXCHGQ DI, 0(SI)
CMPXCHGQ SI, 0(DI)
SETEQ AX
MOVB AX, ret+16(FP)
MOVB AX, ret+24(FP)
RET

17
pkg/syncevent/waiter_arm64.s → pkg/sync/race_arm64.s
View File

@ -12,15 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// +build race
// +build arm64
#include "textflag.h"
// See waiter_noasm_unsafe.go for a description of waiterUnlock.
//
// func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool
TEXT ·waiterUnlock(SB),NOSPLIT,$0-24
MOVD wg+8(FP), R0
MOVD $·preparingG(SB), R1
MOVD ptr+0(FP), R2
// func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
TEXT ·RaceUncheckedAtomicCompareAndSwapUintptr(SB),NOSPLIT,$0-25
MOVD ptr+0(FP), R0
MOVD old+8(FP), R1
MOVD new+16(FP), R1
again:
LDAXR (R0), R3
CMP R1, R3
@ -29,6 +30,6 @@ again:
CBNZ R3, again
ok:
CSET EQ, R0
MOVB R0, ret+16(FP)
MOVB R0, ret+24(FP)
RET

6
pkg/sync/race_unsafe.go
View File

@ -39,3 +39,9 @@ func RaceRelease(addr unsafe.Pointer) {
func RaceReleaseMerge(addr unsafe.Pointer) {
runtime.RaceReleaseMerge(addr)
}
// RaceUncheckedAtomicCompareAndSwapUintptr is equivalent to
// sync/atomic.CompareAndSwapUintptr, but is not checked by the race detector.
// This is necessary when implementing gopark callbacks, since no race context
// is available during their execution.
func RaceUncheckedAtomicCompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool

76
pkg/sync/runtime_unsafe.go Normal file
View File

@ -0,0 +1,76 @@
// Copyright 2020 The gVisor Authors.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.13
// +build !go1.17
// Check function signatures and constants when updating Go version.
package sync
import (
"unsafe"
)
// Note that go:linkname silently doesn't work if the local name is exported,
// necessitating an indirection for exported functions.
// Memmove is runtime.memmove, exported for SeqAtomicLoad/SeqAtomicTryLoad<T>.
//
//go:nosplit
func Memmove(to, from unsafe.Pointer, n uintptr) {
memmove(to, from, n)
}
//go:linkname memmove runtime.memmove
//go:noescape
func memmove(to, from unsafe.Pointer, n uintptr)
// Gopark is runtime.gopark. Gopark calls unlockf(pointer to runtime.g, lock);
// if unlockf returns true, Gopark blocks until Goready(pointer to runtime.g)
// is called. unlockf and its callees must be nosplit and norace, since stack
// splitting and race context are not available where it is called.
//
//go:nosplit
func Gopark(unlockf func(uintptr, unsafe.Pointer) bool, lock unsafe.Pointer, reason uint8, traceEv byte, traceskip int) {
gopark(unlockf, lock, reason, traceEv, traceskip)
}
//go:linkname gopark runtime.gopark
func gopark(unlockf func(uintptr, unsafe.Pointer) bool, lock unsafe.Pointer, reason uint8, traceEv byte, traceskip int)
// Goready is runtime.goready.
//
//go:nosplit
func Goready(gp uintptr, traceskip int) {
goready(gp, traceskip)
}
//go:linkname goready runtime.goready
func goready(gp uintptr, traceskip int)
// Values for the reason argument to gopark, from Go's src/runtime/runtime2.go.
const (
WaitReasonSelect uint8 = 9
)
// Values for the traceEv argument to gopark, from Go's src/runtime/trace.go.
const (
TraceEvGoBlockSelect byte = 24
)
// These functions are only used within the sync package.
//go:linkname semacquire sync.runtime_Semacquire
func semacquire(s *uint32)
//go:linkname semrelease sync.runtime_Semrelease
func semrelease(s *uint32, handoff bool, skipframes int)
//go:linkname canSpin sync.runtime_canSpin
func canSpin(i int) bool
//go:linkname doSpin sync.runtime_doSpin
func doSpin()

23
pkg/sync/rwmutex_unsafe.go
View File

@ -3,11 +3,6 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.13
// +build !go1.17
// Check go:linkname function signatures when updating Go version.
// This is mostly copied from the standard library's sync/rwmutex.go.
//
// Happens-before relationships indicated to the race detector:
@ -23,16 +18,10 @@ import (
"unsafe"
)
//go:linkname runtimeSemacquire sync.runtime_Semacquire
func runtimeSemacquire(s *uint32)
//go:linkname runtimeSemrelease sync.runtime_Semrelease
func runtimeSemrelease(s *uint32, handoff bool, skipframes int)
// RWMutex is identical to sync.RWMutex, but adds the DowngradeLock,
// TryLock and TryRLock methods.
type RWMutex struct {
// w is held if there are pending writers
// w is held by writers.
//
// We use CrossGoroutineMutex rather than Mutex because the lock
// annotation instrumentation in Mutex will trigger false positives in
@ -78,7 +67,7 @@ func (rw *RWMutex) RLock() {
}
if atomic.AddInt32(&rw.readerCount, 1) < 0 {
// A writer is pending, wait for it.
runtimeSemacquire(&rw.readerSem)
semacquire(&rw.readerSem)
}
if RaceEnabled {
RaceEnable()
@ -99,7 +88,7 @@ func (rw *RWMutex) RUnlock() {
// A writer is pending.
if atomic.AddInt32(&rw.readerWait, -1) == 0 {
// The last reader unblocks the writer.
runtimeSemrelease(&rw.writerSem, false, 0)
semrelease(&rw.writerSem, false, 0)
}
}
if RaceEnabled {
@ -146,7 +135,7 @@ func (rw *RWMutex) Lock() {
r := atomic.AddInt32(&rw.readerCount, -rwmutexMaxReaders) + rwmutexMaxReaders
// Wait for active readers.
if r != 0 && atomic.AddInt32(&rw.readerWait, r) != 0 {
runtimeSemacquire(&rw.writerSem)
semacquire(&rw.writerSem)
}
if RaceEnabled {
RaceEnable()
@ -168,7 +157,7 @@ func (rw *RWMutex) Unlock() {
}
// Unblock blocked readers, if any.
for i := 0; i < int(r); i++ {
runtimeSemrelease(&rw.readerSem, false, 0)
semrelease(&rw.readerSem, false, 0)
}
// Allow other writers to proceed.
rw.w.Unlock()
@ -191,7 +180,7 @@ func (rw *RWMutex) DowngradeLock() {
// Unblock blocked readers, if any. Note that this loop starts as 1 since r
// includes this goroutine.
for i := 1; i < int(r); i++ {
runtimeSemrelease(&rw.readerSem, false, 0)
semrelease(&rw.readerSem, false, 0)
}
// Allow other writers to proceed to rw.w.Lock(). Note that they will still
// block on rw.writerSem since at least this reader exists, such that

4
pkg/syncevent/BUILD
View File

@ -9,10 +9,6 @@ go_library(
"receiver.go",
"source.go",
"syncevent.go",
"waiter_amd64.s",
"waiter_arm64.s",
"waiter_asm_unsafe.go",
"waiter_noasm_unsafe.go",
"waiter_unsafe.go",
],
visibility = ["//:sandbox"],

24
pkg/syncevent/waiter_asm_unsafe.go
View File

@ -1,24 +0,0 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build amd64 arm64
package syncevent
import (
"unsafe"
)
// See waiter_noasm_unsafe.go for a description of waiterUnlock.
func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool

39
pkg/syncevent/waiter_noasm_unsafe.go
View File

@ -1,39 +0,0 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// waiterUnlock is called from g0, so when the race detector is enabled,
// waiterUnlock must be implemented in assembly since no race context is
// available.
//
// +build !race
// +build !amd64,!arm64
package syncevent
import (
"sync/atomic"
"unsafe"
)
// waiterUnlock is the "unlock function" passed to runtime.gopark by
// Waiter.Wait*. wg is &Waiter.g, and g is a pointer to the calling runtime.g.
// waiterUnlock returns true if Waiter.Wait should sleep and false if sleeping
// should be aborted.
//
//go:nosplit
func waiterUnlock(ptr unsafe.Pointer, wg *unsafe.Pointer) bool {
// The only way this CAS can fail is if a call to Waiter.NotifyPending()
// has replaced *wg with nil, in which case we should not sleep.
return atomic.CompareAndSwapPointer(wg, (unsafe.Pointer)(&preparingG), ptr)
}

59
pkg/syncevent/waiter_unsafe.go
View File

@ -12,11 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.11
// +build !go1.17
// Check go:linkname function signatures when updating Go version.
package syncevent
import (
@ -26,17 +21,6 @@ import (
"gvisor.dev/gvisor/pkg/sync"
)
//go:linkname gopark runtime.gopark
func gopark(unlockf func(unsafe.Pointer, *unsafe.Pointer) bool, wg *unsafe.Pointer, reason uint8, traceEv byte, traceskip int)
//go:linkname goready runtime.goready
func goready(g unsafe.Pointer, traceskip int)
const (
waitReasonSelect = 9 // Go: src/runtime/runtime2.go
traceEvGoBlockSelect = 24 // Go: src/runtime/trace.go
)
// Waiter allows a goroutine to block on pending events received by a Receiver.
//
// Waiter.Init() must be called before first use.
@ -45,20 +29,19 @@ type Waiter struct {
// g is one of:
//
// - nil: No goroutine is blocking in Wait.
// - 0: No goroutine is blocking in Wait.
//
// - &preparingG: A goroutine is in Wait preparing to sleep, but hasn't yet
// - preparingG: A goroutine is in Wait preparing to sleep, but hasn't yet
// completed waiterUnlock(). Thus the wait can only be interrupted by
// replacing the value of g with nil (the G may not be in state Gwaiting
// yet, so we can't call goready.)
// replacing the value of g with 0 (the G may not be in state Gwaiting yet,
// so we can't call goready.)
//
// - Otherwise: g is a pointer to the runtime.g in state Gwaiting for the
// goroutine blocked in Wait, which can only be woken by calling goready.
g unsafe.Pointer `state:"zerovalue"`
g uintptr `state:"zerovalue"`
}
// Sentinel object for Waiter.g.
var preparingG struct{}
const preparingG = 1
// Init must be called before first use of w.
func (w *Waiter) Init() {
@ -99,21 +82,29 @@ func (w *Waiter) WaitFor(es Set) Set {
}
// Indicate that we're preparing to go to sleep.
atomic.StorePointer(&w.g, (unsafe.Pointer)(&preparingG))
atomic.StoreUintptr(&w.g, preparingG)
// If an event is pending, abort the sleep.
if p := w.r.Pending(); p&es != NoEvents {
atomic.StorePointer(&w.g, nil)
atomic.StoreUintptr(&w.g, 0)
return p
}
// If w.g is still preparingG (i.e. w.NotifyPending() has not been
// called or has not reached atomic.SwapPointer()), go to sleep until
// called or has not reached atomic.SwapUintptr()), go to sleep until
// w.NotifyPending() => goready().
gopark(waiterUnlock, &w.g, waitReasonSelect, traceEvGoBlockSelect, 0)
sync.Gopark(waiterCommit, unsafe.Pointer(&w.g), sync.WaitReasonSelect, sync.TraceEvGoBlockSelect, 0)
}
}
//go:norace
//go:nosplit
func waiterCommit(g uintptr, wg unsafe.Pointer) bool {
// The only way this CAS can fail is if a call to Waiter.NotifyPending()
// has replaced *wg with nil, in which case we should not sleep.
return sync.RaceUncheckedAtomicCompareAndSwapUintptr((*uintptr)(wg), preparingG, g)
}
// Ack marks the given events as not pending.
func (w *Waiter) Ack(es Set) {
w.r.Ack(es)
@ -135,20 +126,20 @@ func (w *Waiter) WaitAndAckAll() Set {
for {
// Indicate that we're preparing to go to sleep.
atomic.StorePointer(&w.g, (unsafe.Pointer)(&preparingG))
atomic.StoreUintptr(&w.g, preparingG)
// If an event is pending, abort the sleep.
if w.r.Pending() != NoEvents {
if p := w.r.PendingAndAckAll(); p != NoEvents {
atomic.StorePointer(&w.g, nil)
atomic.StoreUintptr(&w.g, 0)
return p
}
}
// If w.g is still preparingG (i.e. w.NotifyPending() has not been
// called or has not reached atomic.SwapPointer()), go to sleep until
// called or has not reached atomic.SwapUintptr()), go to sleep until
// w.NotifyPending() => goready().
gopark(waiterUnlock, &w.g, waitReasonSelect, traceEvGoBlockSelect, 0)
sync.Gopark(waiterCommit, unsafe.Pointer(&w.g), sync.WaitReasonSelect, sync.TraceEvGoBlockSelect, 0)
// Check for pending events. We call PendingAndAckAll() directly now since
// we only expect to be woken after events become pending.
@ -171,14 +162,14 @@ func (w *Waiter) NotifyPending() {
// goroutine. NotifyPending is called after w.r.Pending() is updated, so
// concurrent and future calls to w.Wait() will observe pending events and
// abort sleeping.
if atomic.LoadPointer(&w.g) == nil {
if atomic.LoadUintptr(&w.g) == 0 {
return
}
// Wake a sleeping G, or prevent a G that is preparing to sleep from doing
// so. Swap is needed here to ensure that only one call to NotifyPending
// calls goready.
if g := atomic.SwapPointer(&w.g, nil); g != nil && g != (unsafe.Pointer)(&preparingG) {
goready(g, 0)
if g := atomic.SwapUintptr(&w.g, 0); g > preparingG {
sync.Goready(g, 0)
}
}