STOPSHIP: vdso: disable switches for experiments

Permit vdso to be enabled or disabled at will to manage performance
experiments on the dogfood population.  Parameters are accessible
from user space at /sys/module/vdso/parameters/enable_{32|64}:

    enable_64=0 -> 64 bit vdso disabled
    enable_32=0 -> 32 bit vdso disabled

Overhead appears to be ~2ns to perform the checking on every call.

Signed-off-by: Mark Salyzyn <salyzyn@google.com>
Test: manual, bionic-benchmarks --bionic_xml=vdso.xml to confirm.
Bug: 70518189
Change-Id: Ic0fefa61919c93ad809eb20c5a8c8c1590b4cfc3
Signed-off-by: khusika <khusikadhamar@gmail.com>

arch/arm64/include/asm/vdso_datapage.h

@@ -35,6 +35,10 @@ typedef __u64 vdso_xtime_clock_sec_t;
 typedef __u64 vdso_raw_time_sec_t;
 #endif
 
+#define USE_SYSCALL    0x1
+#define USE_SYSCALL_32 0x2
+#define USE_SYSCALL_64 0x4
+
 struct vdso_data {
 	__u64 cs_cycle_last;	/* Timebase at clocksource init */
 	vdso_raw_time_sec_t raw_time_sec;	/* Raw time */

arch/arm64/kernel/vdso.c

@@ -28,6 +28,7 @@
 #include <linux/gfp.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
+#include <linux/moduleparam.h>
 #include <linux/sched.h>
 #include <linux/signal.h>
 #include <linux/slab.h>
@@ -39,6 +40,19 @@
 #include <asm/vdso.h>
 #include <asm/vdso_datapage.h>
 
+#ifdef USE_SYSCALL
+#if defined(__LP64__)
+static int enable_64 = 1;
+module_param(enable_64, int, 0600);
+MODULE_PARM_DESC(enable_64, "enable vDSO for aarch64 0=off");
+#endif
+#if (!defined(__LP64) || (defined(CONFIG_COMPAT) && defined(CONFIG_VDSO32)))
+static int enable_32 = 1;
+module_param(enable_32, int, 0600);
+MODULE_PARM_DESC(enable_32, "enable vDSO for aarch64 0=off");
+#endif
+#endif
+
 struct vdso_mappings {
 	unsigned long num_code_pages;
 	struct vm_special_mapping data_mapping;
@@ -322,6 +336,23 @@ void update_vsyscall(struct timekeeper *tk)
 	struct timespec xtime_coarse;
 	u32 use_syscall = strcmp(tk->tkr_mono.clock->name, "arch_sys_counter");
 
+#ifdef USE_SYSCALL
+	if (use_syscall) {
+		use_syscall = USE_SYSCALL | USE_SYSCALL_32 | USE_SYSCALL_64;
+	} else {
+#if (defined(__LP64__))
+		if (!enable_64)
+#endif
+			use_syscall = USE_SYSCALL_64;
+#if (!defined(__LP64) || (defined(CONFIG_COMPAT) && defined(CONFIG_VDSO32)))
+		if (!enable_32)
+#endif
+			use_syscall |= USE_SYSCALL_32;
+		if (use_syscall == (USE_SYSCALL_32 | USE_SYSCALL_64))
+			use_syscall |= USE_SYSCALL;
+	}
+#endif
+
 	++vdso_data->tb_seq_count;
 	smp_wmb();
 
@@ -332,7 +363,11 @@ void update_vsyscall(struct timekeeper *tk)
 	vdso_data->wtm_clock_sec		= tk->wall_to_monotonic.tv_sec;
 	vdso_data->wtm_clock_nsec		= tk->wall_to_monotonic.tv_nsec;
 
+#ifdef USE_SYSCALL
+	if (!(use_syscall & USE_SYSCALL)) {
+#else
 	if (!use_syscall) {
+#endif
 		/* tkr_mono.cycle_last == tkr_raw.cycle_last */
 		vdso_data->cs_cycle_last	= tk->tkr_mono.cycle_last;
 		vdso_data->raw_time_sec		= tk->raw_sec;

lib/vdso/vgettimeofday.c

@@ -36,6 +36,16 @@ DEFINE_FALLBACK(gettimeofday, struct timeval *, tv, struct timezone *, tz)
 DEFINE_FALLBACK(clock_gettime, clockid_t, clock, struct timespec *, ts)
 DEFINE_FALLBACK(clock_getres, clockid_t, clock, struct timespec *, ts)
 
+#ifdef USE_SYSCALL
+#if defined(__LP64__)
+# define USE_SYSCALL_MASK (USE_SYSCALL | USE_SYSCALL_64)
+#else
+# define USE_SYSCALL_MASK (USE_SYSCALL | USE_SYSCALL_32)
+#endif
+#else
+# define USE_SYSCALL_MASK ((uint32_t)-1)
+#endif
+
 static notrace u32 vdso_read_begin(const struct vdso_data *vd)
 {
 	u32 seq;
@@ -140,7 +150,7 @@ static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
 	do {
 		seq = vdso_read_begin(vd);
 
-		if (vd->use_syscall)
+		if (vd->use_syscall & USE_SYSCALL_MASK)
 			return -1;
 
 		cycle_last = vd->cs_cycle_last;
@@ -180,7 +190,7 @@ static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
 	do {
 		seq = vdso_read_begin(vd);
 
-		if (vd->use_syscall)
+		if (vd->use_syscall & USE_SYSCALL_MASK)
 			return -1;
 
 		cycle_last = vd->cs_cycle_last;
@@ -224,7 +234,7 @@ static notrace int do_monotonic_raw(const struct vdso_data *vd,
 	do {
 		seq = vdso_read_begin(vd);
 
-		if (vd->use_syscall)
+		if (vd->use_syscall & USE_SYSCALL_MASK)
 			return -1;
 
 		cycle_last = vd->cs_cycle_last;
@@ -263,7 +273,7 @@ static notrace int do_boottime(const struct vdso_data *vd, struct timespec *ts)
 	do {
 		seq = vdso_read_begin(vd);
 
-		if (vd->use_syscall)
+		if (vd->use_syscall & USE_SYSCALL_MASK)
 			return -1;
 
 		cycle_last = vd->cs_cycle_last;
@@ -299,6 +309,12 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
 {
 	const struct vdso_data *vd = __get_datapage();
 
+#ifdef USE_SYSCALL
+	if (vd->use_syscall & USE_SYSCALL_MASK) {
+		goto fallback;
+	}
+#endif
+
 	switch (clock) {
 	case CLOCK_REALTIME_COARSE:
 		do_realtime_coarse(vd, ts);
@@ -361,6 +377,14 @@ int __vdso_clock_getres(clockid_t clock, struct timespec *res)
 {
 	long nsec;
 
+#ifdef USE_SYSCALL
+	const struct vdso_data *vd = __get_datapage();
+
+	if (vd->use_syscall & USE_SYSCALL_MASK) {
+		return clock_getres_fallback(clock, res);
+	}
+#endif
+
 	switch (clock) {
 	case CLOCK_REALTIME_COARSE:
 	case CLOCK_MONOTONIC_COARSE:
@@ -389,7 +413,24 @@ int __vdso_clock_getres(clockid_t clock, struct timespec *res)
 notrace time_t __vdso_time(time_t *t)
 {
 	const struct vdso_data *vd = __get_datapage();
+
+#ifdef USE_SYSCALL
+	time_t result;
+
+	if (vd->use_syscall & USE_SYSCALL_MASK) {
+		/* Facsimile of syscall implementation (faster by a few ns) */
+		struct timeval tv;
+		int ret = gettimeofday_fallback(&tv, NULL);
+
+		if (ret < 0)
+			return ret;
+		result = tv.tv_sec;
+	} else {
+		result = READ_ONCE(vd->xtime_coarse_sec);
+	}
+#else
 	time_t result = READ_ONCE(vd->xtime_coarse_sec);
+#endif
 
 	if (t)
 		*t = result;