runtime: use vDSO clock_gettime on linux/mips64x

Speed up nanotime1 and walltime1 on MIPS64 with vDSO, just like the other vDSO-enabled targets. Benchmark numbers on Loongson 3A3000 (GOARCH=mips64le, 1.4GHz) against current master: benchmark old ns/op new ns/op delta BenchmarkNow 868 293 -66.24% BenchmarkNowUnixNano 851 296 -65.22% Performance hit on fallback case, tested by using a wrong vDSO symbol name: benchmark old ns/op new ns/op delta BenchmarkNow 868 889 +2.42% BenchmarkNowUnixNano 851 893 +4.94% Change-Id: Ibfb48893cd060536359863ffee7624c00def646b GitHub-Last-Rev: 03a58ac GitHub-Pull-Request: golang#35181 Reviewed-on: https://go-review.googlesource.com/c/go/+/203578 Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Cherry Zhang <cherryyz@google.com>
witchard · Nov 4, 2019 · 210e367 · 210e367
1 parent bf7e55b
commit 210e367
Show file tree

Hide file tree

Showing 6 changed files with 106 additions and 8 deletions.
diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build !386,!amd64,!arm,!arm64,!ppc64,!ppc64le
+// +build !386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le
 
 package runtime
 

diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s
@@ -211,23 +211,88 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28
 
 // func walltime1() (sec int64, nsec int32)
 TEXT runtime·walltime1(SB),NOSPLIT,$16
+	MOVV	R29, R16	// R16 is unchanged by C code
+	MOVV	R29, R1
+
+	MOVV	g_m(g), R17	// R17 = m
+
+	// Set vdsoPC and vdsoSP for SIGPROF traceback.
+	MOVV	R31, m_vdsoPC(R17)
+	MOVV	R29, m_vdsoSP(R17)
+
+	MOVV	m_curg(R17), R4
+	MOVV	g, R5
+	BNE	R4, R5, noswitch
+
+	MOVV	m_g0(R17), R4
+	MOVV	(g_sched+gobuf_sp)(R4), R1	// Set SP to g0 stack
+
+noswitch:
+	SUBV	$16, R1
+	AND	$~15, R1	// Align for C code
+	MOVV	R1, R29
+
 	MOVW	$0, R4 // CLOCK_REALTIME
 	MOVV	$0(R29), R5
-	MOVV	$SYS_clock_gettime, R2
-	SYSCALL
+
+	MOVV	runtime·vdsoClockgettimeSym(SB), R25
+	BEQ	R25, fallback
+
+	JAL	(R25)
+
+finish:
 	MOVV	0(R29), R3	// sec
 	MOVV	8(R29), R5	// nsec
+
+	MOVV	R16, R29	// restore SP
+	MOVV	R0, m_vdsoSP(R17)	// clear vdsoSP
+
 	MOVV	R3, sec+0(FP)
 	MOVW	R5, nsec+8(FP)
 	RET
 
+fallback:
+	MOVV	$SYS_clock_gettime, R2
+	SYSCALL
+	JMP finish
+
 TEXT runtime·nanotime1(SB),NOSPLIT,$16
+	MOVV	R29, R16	// R16 is unchanged by C code
+	MOVV	R29, R1
+
+	MOVV	g_m(g), R17	// R17 = m
+
+	// Set vdsoPC and vdsoSP for SIGPROF traceback.
+	MOVV	R31, m_vdsoPC(R17)
+	MOVV	R29, m_vdsoSP(R17)
+
+	MOVV	m_curg(R17), R4
+	MOVV	g, R5
+	BNE	R4, R5, noswitch
+
+	MOVV	m_g0(R17), R4
+	MOVV	(g_sched+gobuf_sp)(R4), R1	// Set SP to g0 stack
+
+noswitch:
+	SUBV	$16, R1
+	AND	$~15, R1	// Align for C code
+	MOVV	R1, R29
+
 	MOVW	$1, R4 // CLOCK_MONOTONIC
 	MOVV	$0(R29), R5
-	MOVV	$SYS_clock_gettime, R2
-	SYSCALL
+
+	MOVV	runtime·vdsoClockgettimeSym(SB), R25
+	BEQ	R25, fallback
+
+	JAL	(R25)
+
+finish:
 	MOVV	0(R29), R3	// sec
 	MOVV	8(R29), R5	// nsec
+
+	MOVV	R16, R29	// restore SP
+	MOVV	R0, m_vdsoSP(R17)	// clear vdsoSP
+
 	// sec is in R3, nsec in R5
 	// return nsec in R3
 	MOVV	$1000000000, R4
@@ -237,6 +302,11 @@ TEXT runtime·nanotime1(SB),NOSPLIT,$16
 	MOVV	R3, ret+0(FP)
 	RET
 
+fallback:
+	MOVV	$SYS_clock_gettime, R2
+	SYSCALL
+	JMP	finish
+
 TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28
 	MOVW	how+0(FP), R4
 	MOVV	new+8(FP), R5

diff --git a/src/runtime/vdso_elf64.go b/src/runtime/vdso_elf64.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build amd64 arm64 ppc64 ppc64le
+// +build amd64 arm64 mips64 mips64le ppc64 ppc64le
 
 package runtime
 

diff --git a/src/runtime/vdso_in_none.go b/src/runtime/vdso_in_none.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux,!386,!amd64,!arm,!arm64,!ppc64,!ppc64le !linux
+// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le !linux
 
 package runtime
 

diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386 amd64 arm arm64 ppc64 ppc64le
+// +build 386 amd64 arm arm64 mips64 mips64le ppc64 ppc64le
 
 package runtime
 

diff --git a/src/runtime/vdso_linux_mips64x.go b/src/runtime/vdso_linux_mips64x.go
@@ -0,0 +1,28 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+// +build mips64 mips64le
+
+package runtime
+
+const (
+	// vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
+	// See cmd/compile/internal/mips64/galign.go arch.MAXWIDTH initialization.
+	vdsoArrayMax = 1<<50 - 1
+)
+
+// see man 7 vdso : mips
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
+
+// The symbol name is not __kernel_clock_gettime as suggested by the manpage;
+// according to Linux source code it should be __vdso_clock_gettime instead.
+var vdsoSymbolKeys = []vdsoSymbolKey{
+	{"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
+}
+
+// initialize to fall back to syscall
+var (
+	vdsoClockgettimeSym uintptr = 0
+)