runtime: implement Pinner API for object pinning

Some C APIs require the use or structures that contain pointers to
buffers (iovec, io_uring, ...).  The pointer passing rules would
require that these buffers are allocated in C memory and to process
this data with Go libraries it would need to be copied.

In order to provide a zero-copy way to use these C APIs, this CL
implements a Pinner API that allows to pin Go objects, which
guarantees that the garbage collector does not move these objects
while pinned.  This allows to relax the pointer passing rules so that
pinned pointers can be stored in C allocated memory or can be
contained in Go memory that is passed to C functions.

The Pin() method accepts pointers to objects of any type and
unsafe.Pointer.  Slices and arrays can be pinned by calling Pin()
with the pointer to the first element.  Pinning of maps is not
supported.

If the GC collects unreachable Pinner holding pinned objects it
panics.  If Pin() is called with the other non-pointer types it
panics as well.

Performance considerations: This change has no impact on execution
time on existing code, because checks are only done in code paths,
that would panic otherwise.  The memory footprint on existing code is
one pointer per memory span.

Fixes: #46787

Signed-off-by: Sven Anderson <sven@anderson.de>
Change-Id: I110031fe789b92277ae45a9455624687bd1c54f2
Reviewed-on: https://go-review.googlesource.com/c/go/+/367296
Auto-Submit: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Than McIntosh <thanm@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>

api/next/46787.txt

@@ -0,0 +1,3 @@
+pkg runtime, method (*Pinner) Pin(interface{}) #46787
+pkg runtime, method (*Pinner) Unpin() #46787
+pkg runtime, type Pinner struct #46787

src/cmd/cgo/internal/testerrors/ptr_test.go

@@ -163,6 +163,17 @@ var ptrTests = []ptrTest{
 		fail:      true,
 		expensive: true,
 	},
+	{
+		// Storing a pinned Go pointer into C memory should succeed.
+		name: "barrierpinnedok",
+		c: `#include <stdlib.h>
+		    char **f14a2() { return malloc(sizeof(char*)); }
+		    void f14b2(char **p) {}`,
+		imports:   []string{"runtime"},
+		body:      `var pinr runtime.Pinner; p := C.f14a2(); x := new(C.char); pinr.Pin(x); *p = x; C.f14b2(p); pinr.Unpin()`,
+		fail:      false,
+		expensive: true,
+	},
 	{
 		// Storing a Go pointer into C memory by assigning a
 		// large value should fail.

src/runtime/cgocall.go

@@ -376,12 +376,12 @@ var racecgosync uint64 // represents possible synchronization in C code
 
 // We want to detect all cases where a program that does not use
 // unsafe makes a cgo call passing a Go pointer to memory that
-// contains a Go pointer. Here a Go pointer is defined as a pointer
-// to memory allocated by the Go runtime. Programs that use unsafe
-// can evade this restriction easily, so we don't try to catch them.
-// The cgo program will rewrite all possibly bad pointer arguments to
-// call cgoCheckPointer, where we can catch cases of a Go pointer
-// pointing to a Go pointer.
+// contains an unpinned Go pointer. Here a Go pointer is defined as a
+// pointer to memory allocated by the Go runtime. Programs that use
+// unsafe can evade this restriction easily, so we don't try to catch
+// them. The cgo program will rewrite all possibly bad pointer
+// arguments to call cgoCheckPointer, where we can catch cases of a Go
+// pointer pointing to an unpinned Go pointer.
 
 // Complicating matters, taking the address of a slice or array
 // element permits the C program to access all elements of the slice
@@ -403,7 +403,7 @@ var racecgosync uint64 // represents possible synchronization in C code
 // pointers.)
 
 // cgoCheckPointer checks if the argument contains a Go pointer that
-// points to a Go pointer, and panics if it does.
+// points to an unpinned Go pointer, and panics if it does.
 func cgoCheckPointer(ptr any, arg any) {
 	if !goexperiment.CgoCheck2 && debug.cgocheck == 0 {
 		return
@@ -450,13 +450,14 @@ func cgoCheckPointer(ptr any, arg any) {
 	cgoCheckArg(t, ep.data, t.Kind_&kindDirectIface == 0, top, cgoCheckPointerFail)
 }
 
-const cgoCheckPointerFail = "cgo argument has Go pointer to Go pointer"
+const cgoCheckPointerFail = "cgo argument has Go pointer to unpinned Go pointer"
 const cgoResultFail = "cgo result has Go pointer"
 
 // cgoCheckArg is the real work of cgoCheckPointer. The argument p
 // is either a pointer to the value (of type t), or the value itself,
 // depending on indir. The top parameter is whether we are at the top
-// level, where Go pointers are allowed.
+// level, where Go pointers are allowed. Go pointers to pinned objects are
+// always allowed.
 func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) {
 	if t.PtrBytes == 0 || p == nil {
 		// If the type has no pointers there is nothing to do.
@@ -507,7 +508,7 @@ func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) {
 		if !cgoIsGoPointer(p) {
 			return
 		}
-		if !top {
+		if !top && !isPinned(p) {
 			panic(errorString(msg))
 		}
 		cgoCheckArg(it, p, it.Kind_&kindDirectIface == 0, false, msg)
@@ -518,7 +519,7 @@ func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) {
 		if p == nil || !cgoIsGoPointer(p) {
 			return
 		}
-		if !top {
+		if !top && !isPinned(p) {
 			panic(errorString(msg))
 		}
 		if st.Elem.PtrBytes == 0 {
@@ -533,7 +534,7 @@ func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) {
 		if !cgoIsGoPointer(ss.str) {
 			return
 		}
-		if !top {
+		if !top && !isPinned(ss.str) {
 			panic(errorString(msg))
 		}
 	case kindStruct:
@@ -562,7 +563,7 @@ func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) {
 		if !cgoIsGoPointer(p) {
 			return
 		}
-		if !top {
+		if !top && !isPinned(p) {
 			panic(errorString(msg))
 		}
 
@@ -572,7 +573,7 @@ func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) {
 
 // cgoCheckUnknownPointer is called for an arbitrary pointer into Go
 // memory. It checks whether that Go memory contains any other
-// pointer into Go memory. If it does, we panic.
+// pointer into unpinned Go memory. If it does, we panic.
 // The return values are unused but useful to see in panic tracebacks.
 func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) {
 	if inheap(uintptr(p)) {
@@ -588,7 +589,8 @@ func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) {
 			if hbits, addr = hbits.next(); addr == 0 {
 				break
 			}
-			if cgoIsGoPointer(*(*unsafe.Pointer)(unsafe.Pointer(addr))) {
+			pp := *(*unsafe.Pointer)(unsafe.Pointer(addr))
+			if cgoIsGoPointer(pp) && !isPinned(pp) {
 				panic(errorString(msg))
 			}
 		}

src/runtime/cgocheck.go

@@ -12,10 +12,11 @@ import (
 	"unsafe"
 )
 
-const cgoWriteBarrierFail = "Go pointer stored into non-Go memory"
+const cgoWriteBarrierFail = "unpinned Go pointer stored into non-Go memory"
 
 // cgoCheckPtrWrite is called whenever a pointer is stored into memory.
-// It throws if the program is storing a Go pointer into non-Go memory.
+// It throws if the program is storing an unpinned Go pointer into non-Go
+// memory.
 //
 // This is called from generated code when GOEXPERIMENT=cgocheck2 is enabled.
 //
@@ -48,6 +49,12 @@ func cgoCheckPtrWrite(dst *unsafe.Pointer, src unsafe.Pointer) {
 		return
 	}
 
+	// If the object is pinned, it's safe to store it in C memory. The GC
+	// ensures it will not be moved or freed.
+	if isPinned(src) {
+		return
+	}
+
 	// It's OK if writing to memory allocated by persistentalloc.
 	// Do this check last because it is more expensive and rarely true.
 	// If it is false the expense doesn't matter since we are crashing.
@@ -56,14 +63,14 @@ func cgoCheckPtrWrite(dst *unsafe.Pointer, src unsafe.Pointer) {
 	}
 
 	systemstack(func() {
-		println("write of Go pointer", hex(uintptr(src)), "to non-Go memory", hex(uintptr(unsafe.Pointer(dst))))
+		println("write of unpinned Go pointer", hex(uintptr(src)), "to non-Go memory", hex(uintptr(unsafe.Pointer(dst))))
 		throw(cgoWriteBarrierFail)
 	})
 }
 
 // cgoCheckMemmove is called when moving a block of memory.
-// It throws if the program is copying a block that contains a Go pointer
-// into non-Go memory.
+// It throws if the program is copying a block that contains an unpinned Go
+// pointer into non-Go memory.
 //
 // This is called from generated code when GOEXPERIMENT=cgocheck2 is enabled.
 //
@@ -76,8 +83,8 @@ func cgoCheckMemmove(typ *_type, dst, src unsafe.Pointer) {
 // cgoCheckMemmove2 is called when moving a block of memory.
 // dst and src point off bytes into the value to copy.
 // size is the number of bytes to copy.
-// It throws if the program is copying a block that contains a Go pointer
-// into non-Go memory.
+// It throws if the program is copying a block that contains an unpinned Go
+// pointer into non-Go memory.
 //
 //go:nosplit
 //go:nowritebarrier
@@ -97,8 +104,8 @@ func cgoCheckMemmove2(typ *_type, dst, src unsafe.Pointer, off, size uintptr) {
 // cgoCheckSliceCopy is called when copying n elements of a slice.
 // src and dst are pointers to the first element of the slice.
 // typ is the element type of the slice.
-// It throws if the program is copying slice elements that contain Go pointers
-// into non-Go memory.
+// It throws if the program is copying slice elements that contain unpinned Go
+// pointers into non-Go memory.
 //
 //go:nosplit
 //go:nowritebarrier
@@ -120,7 +127,7 @@ func cgoCheckSliceCopy(typ *_type, dst, src unsafe.Pointer, n int) {
 }
 
 // cgoCheckTypedBlock checks the block of memory at src, for up to size bytes,
-// and throws if it finds a Go pointer. The type of the memory is typ,
+// and throws if it finds an unpinned Go pointer. The type of the memory is typ,
 // and src is off bytes into that type.
 //
 //go:nosplit
@@ -177,14 +184,14 @@ func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) {
 			break
 		}
 		v := *(*unsafe.Pointer)(unsafe.Pointer(addr))
-		if cgoIsGoPointer(v) {
+		if cgoIsGoPointer(v) && !isPinned(v) {
 			throw(cgoWriteBarrierFail)
 		}
 	}
 }
 
 // cgoCheckBits checks the block of memory at src, for up to size
-// bytes, and throws if it finds a Go pointer. The gcbits mark each
+// bytes, and throws if it finds an unpinned Go pointer. The gcbits mark each
 // pointer value. The src pointer is off bytes into the gcbits.
 //
 //go:nosplit
@@ -209,7 +216,7 @@ func cgoCheckBits(src unsafe.Pointer, gcbits *byte, off, size uintptr) {
 		} else {
 			if bits&1 != 0 {
 				v := *(*unsafe.Pointer)(add(src, i))
-				if cgoIsGoPointer(v) {
+				if cgoIsGoPointer(v) && !isPinned(v) {
 					throw(cgoWriteBarrierFail)
 				}
 			}

src/runtime/export_test.go

@@ -47,6 +47,8 @@ var NetpollGenericInit = netpollGenericInit
 var Memmove = memmove
 var MemclrNoHeapPointers = memclrNoHeapPointers
 
+var CgoCheckPointer = cgoCheckPointer
+
 const TracebackInnerFrames = tracebackInnerFrames
 const TracebackOuterFrames = tracebackOuterFrames
 
@@ -1826,3 +1828,15 @@ func PersistentAlloc(n uintptr) unsafe.Pointer {
 func FPCallers(pcBuf []uintptr) int {
 	return fpTracebackPCs(unsafe.Pointer(getcallerfp()), pcBuf)
 }
+
+var (
+	IsPinned      = isPinned
+	GetPinCounter = pinnerGetPinCounter
+)
+
+func SetPinnerLeakPanic(f func()) {
+	pinnerLeakPanic = f
+}
+func GetPinnerLeakPanic() func() {
+	return pinnerLeakPanic
+}

src/runtime/mbitmap.go

@@ -202,6 +202,10 @@ func (s *mspan) isFree(index uintptr) bool {
 // n must be within [0, s.npages*_PageSize),
 // or may be exactly s.npages*_PageSize
 // if s.elemsize is from sizeclasses.go.
+//
+// nosplit, because it is called by objIndex, which is nosplit
+//
+//go:nosplit
 func (s *mspan) divideByElemSize(n uintptr) uintptr {
 	const doubleCheck = false
 
@@ -215,6 +219,9 @@ func (s *mspan) divideByElemSize(n uintptr) uintptr {
 	return q
 }
 
+// nosplit, because it is called by other nosplit code like findObject
+//
+//go:nosplit
 func (s *mspan) objIndex(p uintptr) uintptr {
 	return s.divideByElemSize(p - s.base())
 }

src/runtime/mfinal.go

@@ -274,6 +274,31 @@ func runfinq() {
 	}
 }
 
+func isGoPointerWithoutSpan(p unsafe.Pointer) bool {
+	// 0-length objects are okay.
+	if p == unsafe.Pointer(&zerobase) {
+		return true
+	}
+
+	// Global initializers might be linker-allocated.
+	//	var Foo = &Object{}
+	//	func main() {
+	//		runtime.SetFinalizer(Foo, nil)
+	//	}
+	// The relevant segments are: noptrdata, data, bss, noptrbss.
+	// We cannot assume they are in any order or even contiguous,
+	// due to external linking.
+	for datap := &firstmoduledata; datap != nil; datap = datap.next {
+		if datap.noptrdata <= uintptr(p) && uintptr(p) < datap.enoptrdata ||
+			datap.data <= uintptr(p) && uintptr(p) < datap.edata ||
+			datap.bss <= uintptr(p) && uintptr(p) < datap.ebss ||
+			datap.noptrbss <= uintptr(p) && uintptr(p) < datap.enoptrbss {
+			return true
+		}
+	}
+	return false
+}
+
 // SetFinalizer sets the finalizer associated with obj to the provided
 // finalizer function. When the garbage collector finds an unreachable block
 // with an associated finalizer, it clears the association and runs
@@ -388,27 +413,9 @@ func SetFinalizer(obj any, finalizer any) {
 	base, _, _ := findObject(uintptr(e.data), 0, 0)
 
 	if base == 0 {
-		// 0-length objects are okay.
-		if e.data == unsafe.Pointer(&zerobase) {
+		if isGoPointerWithoutSpan(e.data) {
 			return
 		}
-
-		// Global initializers might be linker-allocated.
-		//	var Foo = &Object{}
-		//	func main() {
-		//		runtime.SetFinalizer(Foo, nil)
-		//	}
-		// The relevant segments are: noptrdata, data, bss, noptrbss.
-		// We cannot assume they are in any order or even contiguous,
-		// due to external linking.
-		for datap := &firstmoduledata; datap != nil; datap = datap.next {
-			if datap.noptrdata <= uintptr(e.data) && uintptr(e.data) < datap.enoptrdata ||
-				datap.data <= uintptr(e.data) && uintptr(e.data) < datap.edata ||
-				datap.bss <= uintptr(e.data) && uintptr(e.data) < datap.ebss ||
-				datap.noptrbss <= uintptr(e.data) && uintptr(e.data) < datap.enoptrbss {
-				return
-			}
-		}
 		throw("runtime.SetFinalizer: pointer not in allocated block")
 	}