unique: add unique package and implement Make/Handle

This change adds the unique package for canonicalizing values, as
described by the proposal in #62483.

Fixes #62483.

Change-Id: I1dc3d34ec12351cb4dc3838a8ea29a5368d59e99
Reviewed-on: https://go-review.googlesource.com/c/go/+/574355
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Ingo Oeser <nightlyone@googlemail.com>
Reviewed-by: David Chase <drchase@google.com>

api/next/62483.txt

@@ -0,0 +1,3 @@
+pkg unique, func Make[$0 comparable]($0) Handle[$0] #62483
+pkg unique, method (Handle[$0]) Value() $0 #62483
+pkg unique, type Handle[$0 comparable] struct #62483

doc/next/6-stdlib/2-unique.md

@@ -0,0 +1,13 @@
+### New unique package
+
+The new [unique](/pkg/unique) package provides facilites for
+canonicalizing values (like "interning" or "hash-consing").
+
+Any value of comparable type may be canonicalized with the new
+`Make[T]` function, which produces a reference to a canonical copy of
+the value in the form of a `Handle[T]`.
+Two `Handle[T]` are equal if and only if the values used to produce the
+handles are equal, allowing programs to deduplicate values and reduce
+their memory footprint.
+Comparing two `Handle[T]` values is efficient, reducing down to a simple
+pointer comparison.

doc/next/6-stdlib/99-minor/unique/62483.md

@@ -0,0 +1 @@
+<!-- This is a new package; covered in 6-stdlib/2-unique.md. -->

src/go/build/deps_test.go

@@ -164,6 +164,9 @@ var depsRules = `
 	bufio, path, strconv
 	< STR;
 
+	RUNTIME, internal/concurrent
+	< unique;
+
 	# OS is basic OS access, including helpers (path/filepath, os/exec, etc).
 	# OS includes string routines, but those must be layered above package os.
 	# OS does not include reflection.

src/runtime/mgc.go

@@ -1694,7 +1694,8 @@ func gcResetMarkState() {
 // Hooks for other packages
 
 var poolcleanup func()
-var boringCaches []unsafe.Pointer // for crypto/internal/boring
+var boringCaches []unsafe.Pointer  // for crypto/internal/boring
+var uniqueMapCleanup chan struct{} // for unique
 
 //go:linkname sync_runtime_registerPoolCleanup sync.runtime_registerPoolCleanup
 func sync_runtime_registerPoolCleanup(f func()) {
@@ -1706,6 +1707,18 @@ func boring_registerCache(p unsafe.Pointer) {
 	boringCaches = append(boringCaches, p)
 }
 
+//go:linkname unique_runtime_registerUniqueMapCleanup unique.runtime_registerUniqueMapCleanup
+func unique_runtime_registerUniqueMapCleanup(f func()) {
+	// Start the goroutine in the runtime so it's counted as a system goroutine.
+	uniqueMapCleanup = make(chan struct{}, 1)
+	go func(cleanup func()) {
+		for {
+			<-uniqueMapCleanup
+			cleanup()
+		}
+	}(f)
+}
+
 func clearpools() {
 	// clear sync.Pools
 	if poolcleanup != nil {
@@ -1717,6 +1730,14 @@ func clearpools() {
 		atomicstorep(p, nil)
 	}
 
+	// clear unique maps
+	if uniqueMapCleanup != nil {
+		select {
+		case uniqueMapCleanup <- struct{}{}:
+		default:
+		}
+	}
+
 	// Clear central sudog cache.
 	// Leave per-P caches alone, they have strictly bounded size.
 	// Disconnect cached list before dropping it on the floor,

src/unique/clone.go

@@ -0,0 +1,100 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unique
+
+import (
+	"internal/abi"
+	"unsafe"
+)
+
+// clone makes a copy of value, and may update string values found in value
+// with a cloned version of those strings. The purpose of explicitly cloning
+// strings is to avoid accidentally giving a large string a long lifetime.
+//
+// Note that this will clone strings in structs and arrays found in value,
+// and will clone value if it itself is a string. It will not, however, clone
+// strings if value is of interface or slice type (that is, found via an
+// indirection).
+func clone[T comparable](value T, seq *cloneSeq) T {
+	for _, offset := range seq.stringOffsets {
+		ps := (*string)(unsafe.Pointer(uintptr(unsafe.Pointer(&value)) + offset))
+		*ps = cloneString(*ps)
+	}
+	return value
+}
+
+// singleStringClone describes how to clone a single string.
+var singleStringClone = cloneSeq{stringOffsets: []uintptr{0}}
+
+// cloneSeq describes how to clone a value of a particular type.
+type cloneSeq struct {
+	stringOffsets []uintptr
+}
+
+// makeCloneSeq creates a cloneSeq for a type.
+func makeCloneSeq(typ *abi.Type) cloneSeq {
+	if typ == nil {
+		return cloneSeq{}
+	}
+	if typ.Kind() == abi.String {
+		return singleStringClone
+	}
+	var seq cloneSeq
+	switch typ.Kind() {
+	case abi.Struct:
+		buildStructCloneSeq(typ, &seq, 0)
+	case abi.Array:
+		buildArrayCloneSeq(typ, &seq, 0)
+	}
+	return seq
+}
+
+// buildStructCloneSeq populates a cloneSeq for an abi.Type that has Kind abi.Struct.
+func buildStructCloneSeq(typ *abi.Type, seq *cloneSeq, baseOffset uintptr) {
+	styp := typ.StructType()
+	for i := range styp.Fields {
+		f := &styp.Fields[i]
+		switch f.Typ.Kind() {
+		case abi.String:
+			seq.stringOffsets = append(seq.stringOffsets, baseOffset+f.Offset)
+		case abi.Struct:
+			buildStructCloneSeq(f.Typ, seq, baseOffset+f.Offset)
+		case abi.Array:
+			buildArrayCloneSeq(f.Typ, seq, baseOffset+f.Offset)
+		}
+	}
+}
+
+// buildArrayCloneSeq populates a cloneSeq for an abi.Type that has Kind abi.Array.
+func buildArrayCloneSeq(typ *abi.Type, seq *cloneSeq, baseOffset uintptr) {
+	atyp := typ.ArrayType()
+	etyp := atyp.Elem
+	offset := baseOffset
+	for range atyp.Len {
+		switch etyp.Kind() {
+		case abi.String:
+			seq.stringOffsets = append(seq.stringOffsets, offset)
+		case abi.Struct:
+			buildStructCloneSeq(etyp, seq, offset)
+		case abi.Array:
+			buildArrayCloneSeq(etyp, seq, offset)
+		}
+		offset += etyp.Size()
+		align := uintptr(etyp.FieldAlign())
+		offset = (offset + align - 1) &^ (align - 1)
+	}
+}
+
+// cloneString is a copy of strings.Clone, because we can't depend on the strings
+// package here. Several packages that might make use of unique, like net, explicitly
+// forbid depending on unicode, which strings depends on.
+func cloneString(s string) string {
+	if len(s) == 0 {
+		return ""
+	}
+	b := make([]byte, len(s))
+	copy(b, s)
+	return unsafe.String(&b[0], len(b))
+}

src/unique/clone_test.go

@@ -0,0 +1,37 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unique
+
+import (
+	"internal/abi"
+	"internal/goarch"
+	"reflect"
+	"testing"
+)
+
+func TestMakeCloneSeq(t *testing.T) {
+	testCloneSeq[testString](t, cSeq(0))
+	testCloneSeq[testIntArray](t, cSeq())
+	testCloneSeq[testEface](t, cSeq())
+	testCloneSeq[testStringArray](t, cSeq(0, 2*goarch.PtrSize, 4*goarch.PtrSize))
+	testCloneSeq[testStringStruct](t, cSeq(0))
+	testCloneSeq[testStringStructArrayStruct](t, cSeq(0, 2*goarch.PtrSize))
+	testCloneSeq[testStruct](t, cSeq(8))
+}
+
+func cSeq(stringOffsets ...uintptr) cloneSeq {
+	return cloneSeq{stringOffsets: stringOffsets}
+}
+
+func testCloneSeq[T any](t *testing.T, want cloneSeq) {
+	typName := reflect.TypeFor[T]().Name()
+	typ := abi.TypeOf(*new(T))
+	t.Run(typName, func(t *testing.T) {
+		got := makeCloneSeq(typ)
+		if !reflect.DeepEqual(got, want) {
+			t.Errorf("unexpected cloneSeq for type %s: got %#v, want %#v", typName, got, want)
+		}
+	})
+}

src/unique/doc.go

@@ -0,0 +1,9 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+The unique package provides facilities for canonicalizing ("interning")
+comparable values.
+*/
+package unique