Implement TLS variables

Thread-local storage (TLS) variables are a form of dynamic variables in
DTrace.  They are implemented using a global BPF hash map, indexed using
a key value that is derived from the task ID and the variable ID (with
some special magic to handle the idle task that has TID 0 on all CPUs).

Access to the TLS variables is handled through a pre-compiled BPF
function : dt_get_tvar.  It returns the address of the storage location
for the given variable in the current task.  This is used for both load
and store operations.

The dvars BPF map contains an element at key 0 that contains a value of
all zeros.  This is used to initialize new TLS variables.

One test (variables/tvar/err.limited_space.d) is marked XFAIL because
there is no support for reporting drops just yet.

Signed-off-by: Kris Van Hees <kris.van.hees@oracle.com>
Reviewed-by: Eugene Loh <eugene.loh@oracle.com>

bpf/Build

@@ -26,7 +26,7 @@ bpf_dlib_SOURCES = \
 	basename.S \
 	dirname.S \
 	get_bvar.c \
-	get_tvar.c set_tvar.c \
+	get_tvar.c \
 	index.S \
 	lltostr.S \
 	probe_error.c \

bpf/get_tvar.c

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  */
 #include <linux/bpf.h>
 #include <stdint.h>
@@ -10,12 +10,63 @@
 # define noinline	__attribute__((noinline))
 #endif
 
-extern struct bpf_map_def tvars;
+extern struct bpf_map_def dvars;
+extern uint64_t NCPUS;
 
-noinline uint64_t dt_get_tvar(uint32_t id)
+noinline void *dt_get_tvar(uint32_t id, uint64_t store, uint64_t nval)
 {
-	uint64_t	*val;
+	uint64_t	key;
+	uint64_t	dflt_key = 0;
+	void		*val;
 
-	val = bpf_map_lookup_elem(&tvars, &id);
-	return val ? *val : 0;
+	key = bpf_get_current_pid_tgid();
+	key &= 0x00000000ffffffffUL;
+	if (key == 0)
+		key = bpf_get_smp_processor_id();
+	else
+		key += (uint32_t)(uint64_t)&NCPUS;
+
+	key++;
+	key = (key << 32) | id;
+
+	/*
+	 * If we are going to store a zero-value, it is a request to delete the
+	 * TLS variable.
+	 */
+	if (store && !nval) {
+		bpf_map_delete_elem(&dvars, &key);
+		return 0;
+	}
+
+	/*
+	 * For load or non-zero store, we return the address of the value if
+	 * there is one.
+	 */
+	val = bpf_map_lookup_elem(&dvars, &key);
+	if (val != 0)
+		return val;
+
+	/*
+	 * If we are performing a load (not a store), and no var was found,
+	 * we are done.
+	 */
+	if (!store)
+		return 0;
+
+	/*
+	 * Not found and we are storing a non-zero value: create the variable
+	 * with the default value.
+	 */
+	val = bpf_map_lookup_elem(&dvars, &dflt_key);
+	if (val == 0)
+		return 0;
+
+	if (bpf_map_update_elem(&dvars, &key, val, BPF_ANY) < 0)
+		return 0;
+
+	val = bpf_map_lookup_elem(&dvars, &key);
+	if (val != 0)
+		return val;
+
+	return 0;
 }

libdtrace/dt_bpf.c

@@ -98,7 +98,7 @@ int dt_bpf_map_update(int fd, const void *key, const void *val)
 	attr.map_fd = fd;
 	attr.key = (uint64_t)(unsigned long)key;
 	attr.value = (uint64_t)(unsigned long)val;
-	attr.flags = 0;
+	attr.flags = BPF_ANY;
 
 	return bpf(BPF_MAP_UPDATE_ELEM, &attr);
 }
@@ -213,36 +213,25 @@ populate_probes_map(dtrace_hdl_t *dtp, int fd)
  *		use.
  * - gvars:	Global variables map.  This is a global map with a singleton
  *		element (key 0) addressed by variable offset.
+ * - dvars:	Dynamic variables map.  This is a global hash map indexed with
+ *		a unique numeric identifier for each variable per thread.  The
+ *		value of each element is sized to accomodate the largest thread
+ *		local ariable type found across all programsn the tracing
+ *		session..
  * - lvars:	Local variables map.  This is a per-CPU map with a singleton
  *		element (key 0) addressed by variable offset.
- *
- * FIXME: TLS variable storage is still being designed further so this is just
- *	  a temporary placeholder and will most likely be replaced by something
- *	  else.  If we stick to the legacy DTrace approach, we will need to
- *	  determine the maximum overall key size for TLS variables *and* the
- *	  maximum value size.  Based on these values, the legacy code would
- *	  take the memory size set aside for dynamic variables, and divide it by
- *	  the storage size needed for the largest dynamic variable (associative
- *	  array element or TLS variable).
- *
- * - tvars:	Thread-local (TLS) variables map, associating a 64-bit value
- *		with each thread-local variable.  The map is indexed by a value
- *		computed based on the thread-local variable id and execution
- *		thread information to ensure each thread has its own copy of a
- *		given thread-local variable.  The amount of TLS variable space
- *		to allocate for these dynamic variables is calculated based on
- *		the number of uniquely named TLS variables (next-to-be-assigned
- *		id minus the base id).
  */
 int
 dt_bpf_gmap_create(dtrace_hdl_t *dtp)
 {
-	int		stabsz, gvarsz, lvarsz, tvarc, aggsz, memsz;
+	int		stabsz, gvarsz, lvarsz, aggsz, memsz;
+	int		dvarc = 0;
 	int		ci_mapfd, st_mapfd, pr_mapfd;
 	uint32_t	key = 0;
 	size_t		strsize = dtp->dt_options[DTRACEOPT_STRSIZE];
 	uint8_t		*buf, *end;
 	char		*strtab;
+	size_t		strdatasz = P2ROUNDUP(DT_STRLEN_BYTES + strsize + 1, 8);
 
 	/* If we already created the global maps, return success. */
 	if (dt_gmap_done)
@@ -257,7 +246,9 @@ dt_bpf_gmap_create(dtrace_hdl_t *dtp)
 	/* Determine sizes for global, local, and TLS maps. */
 	gvarsz = P2ROUNDUP(dt_idhash_datasize(dtp->dt_globals), 8);
 	lvarsz = P2ROUNDUP(dtp->dt_maxlvaralloc, 8);
-	tvarc = dt_idhash_peekid(dtp->dt_tls) - DIF_VAR_OTHER_UBASE;
+	if (dtp->dt_maxtlslen)
+		dvarc = (dtp->dt_options[DTRACEOPT_DYNVARSIZE] /
+			 dtp->dt_maxtlslen) + 1;
 
 	/* Create global maps as long as there are no errors. */
 	dtp->dt_stmap_fd = create_gmap(dtp, "state", BPF_MAP_TYPE_ARRAY,
@@ -312,9 +303,7 @@ dt_bpf_gmap_create(dtrace_hdl_t *dtp)
 		8 +
 		roundup(dtp->dt_maxreclen, 8) +
 		MAX(sizeof(uint64_t) * dtp->dt_options[DTRACEOPT_MAXFRAMES],
-		    DT_TSTRING_SLOTS *
-			roundup(DT_STRLEN_BYTES +
-				dtp->dt_options[DTRACEOPT_STRSIZE] + 1, 8) +
+		    DT_TSTRING_SLOTS * strdatasz +
 		    dtp->dt_options[DTRACEOPT_STRSIZE] + 1
 		);
 	if (create_gmap(dtp, "mem", BPF_MAP_TYPE_PERCPU_ARRAY,
@@ -371,10 +360,19 @@ dt_bpf_gmap_create(dtrace_hdl_t *dtp)
 			sizeof(uint32_t), lvarsz, 1) == -1)
 		return -1;		/* dt_errno is set for us */
 
-	if (tvarc > 0 &&
-	    create_gmap(dtp, "tvars", BPF_MAP_TYPE_ARRAY,
-			sizeof(uint32_t), sizeof(uint64_t), tvarc) == -1)
-		return -1;		/* dt_errno is set for us */
+	if (dvarc > 0) {
+		int	fd;
+		char	dflt[dtp->dt_maxtlslen];
+
+		fd = create_gmap(dtp, "dvars", BPF_MAP_TYPE_HASH,
+				 sizeof(uint64_t), dtp->dt_maxtlslen, dvarc);
+		if (fd == -1)
+			return -1;	/* dt_errno is set for us */
+
+		/* Initialize the default value (key = 0). */
+		memset(dflt, 0, dtp->dt_maxtlslen);
+		dt_bpf_map_update(fd, &key, &dflt);
+	}
 
 	/* Populate the 'cpuinfo' map. */
 	dt_bpf_map_update(ci_mapfd, &key, dtp->dt_conf.cpus);

libdtrace/dt_bpf.h

@@ -27,6 +27,7 @@ extern "C" {
 #define DT_CONST_STKSIZ	7
 #define DT_CONST_BOOTTM	8
 #define DT_CONST_NSPEC	9
+#define DT_CONST_NCPUS	10
 
 extern int perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu,
 			   int group_fd, unsigned long flags);

libdtrace/dt_cc.c

@@ -2355,6 +2355,9 @@ dt_link_construct(dtrace_hdl_t *dtp, const dt_probe_t *prp, dtrace_difo_t *dp,
 			case DT_CONST_NSPEC:
 				nrp->dofr_data = dtp->dt_options[DTRACEOPT_NSPEC];
 				continue;
+			case DT_CONST_NCPUS:
+				nrp->dofr_data = dtp->dt_conf.max_cpuid + 1;
+				continue;
 			case DT_CONST_STKSIZ:
 				nrp->dofr_data = sizeof(uint64_t)
 				    * dtp->dt_options[DTRACEOPT_MAXFRAMES];