Implement associative array support

Associative arrays (global or TLS) are variables that are indexed with
a tuple (one or more values).  The underlying storage is provided by
the dvars (dynamic variables) BPF hash map.  Since dvars elements are
indexed using a unique 64-bit ID, and given that it provides storage
for both regular TLS variables and associate array elements, the
algorithms to calculate the ID are designed to provide orthogonal
value ranges.

1. Thread-local storage (TLS) variables: self->a
        dvar key = TLS key (highest bit = 0)
2. Global associative array elements: a[tuple]
        dvar key = &tuples[var id, tuple, 0] (highest bit = 1)
3. TLS associative array elements: self->a[tuple]
        dvar key = &tuples[var id, tuple, TLS key] (highest bit = 1)

Given that the TLS key can never be 0, uniqueness of the dvar key is
guaranteed in this scheme.

A new BPF hash map (tuples) associates the concatenation of the tuple
values as key with the address of the hash map element (which is
guaranteed to be unique).  Since BPF maps are allocated in kernel space,
these addresses will always have high order bit 1.

To ensure uniqueness between global associate array elements and TLS
associate array elements, and to ensure uniqueness between elements
with the same tuple index in different arrays, the key in the tuples
map is determined based on a rewritten tuple:

    [ variable ID, original tuple values, TLS variable key or 0 ]

While variable IDs are not unique between variable kinds, the final
component in the rewritten tuple is the TLS variable key (for TLS
associative arrays - never 0) or 0 (for global associative arrays).

Various new tests are added to exercise the new functionality.

Running out of dynamic variable space is not being reported as a drop
yet due to lack of drop counter support.  It generates an error
instead, and therefore the test for the drop reporting remains XFAIL.

Signed-off-by: Kris Van Hees <kris.van.hees@oracle.com>
Reviewed-by: Eugene Loh <eugene.loh@oracle.com>

bpf/get_dvar.c

@@ -11,8 +11,24 @@
 #endif
 
 extern struct bpf_map_def dvars;
+extern struct bpf_map_def tuples;
 extern uint64_t NCPUS;
 
+/*
+ * Dynamic variables are identified using a unique 64-bit key.  Three different
+ * categories of dynamic variables are supported in DTrace:
+ *
+ * 1. Thread-local storage (TLS) variables:
+ *	dvar key = TLS key (highest bit = 0)
+ * 2. Global associative array elements:
+ *	dvar key = &tuples[var id, tuple, (uint64_t)0] (highest bit = 1)
+ * 3. TLS associative array elements:
+ *	dvar key = &tuples[var id, tuple, TLS key] (highest bit = 1)
+ *
+ * Given that the TLS key can never be 0, uniqueness of the dvar key is
+ * guaranteed in this scheme.
+ */
+
 noinline uint64_t dt_tlskey(uint32_t id)
 {
 	uint64_t	key;
@@ -25,7 +41,7 @@ noinline uint64_t dt_tlskey(uint32_t id)
 		key += (uint32_t)(uint64_t)&NCPUS;
 
 	key++;
-	key = (key << 32) | id;
+	key = ((key & 0x7fffffff) << 32) | id;
 
 	return key;
 }
@@ -81,3 +97,59 @@ noinline void *dt_get_tvar(uint32_t id, uint64_t store, uint64_t nval)
 {
 	return dt_get_dvar(dt_tlskey(id), store, nval);
 }
+
+noinline void *dt_get_assoc(uint32_t id, const char *tuple,
+			    uint64_t store, uint64_t nval)
+{
+	uint64_t	*valp;
+	uint64_t	val;
+	uint64_t	dflt_val = 0;
+
+	/*
+	 * Place the variable ID at the beginning of the tuple.
+	 */
+	*(uint32_t *)tuple = id;
+
+	/*
+	 * Look for the tuple in the tuples map.
+	 */
+	valp = bpf_map_lookup_elem(&tuples, tuple);
+	if (valp == 0) {
+		/*
+		 * Not found.  If we are not storing a value (i.e. performing a
+		 * load), return the default value (0).  If we are trying to
+		 * delete an associative array element, we don't have to do
+		 * anything because it does not exist anyway.
+		 */
+		if (!store || !nval)
+			return 0;
+
+		/*
+		 * Create the tuple and use the address of the value as the
+		 * actual value.
+		 */
+		if (bpf_map_update_elem(&tuples, tuple, &dflt_val, BPF_ANY) < 0)
+			return 0;
+		valp = bpf_map_lookup_elem(&tuples, tuple);
+		if (valp == 0)
+			return 0;
+		*valp = (uint64_t)valp;
+		if (bpf_map_update_elem(&tuples, tuple, valp, BPF_ANY) < 0)
+			return 0;
+
+		val = *valp;
+	} else {
+		/*
+		 * Record the value (used as key into the dvars map), and if we
+		 * are storing a zero-value (deleting the element), delete the
+		 * tuple.  The associated dynamic variable will be deleted by
+		 * the dt_get_dvar() call.
+		 */
+		val = *valp;
+
+		if (store && !nval)
+			bpf_map_delete_elem(&tuples, tuple);
+	}
+
+	return dt_get_dvar(val, store, nval);
+}

libdtrace/dt_bpf.c

@@ -224,6 +224,11 @@ populate_probes_map(dtrace_hdl_t *dtp, int fd)
  *		tracing session.
  * - lvars:	Local variables map.  This is a per-CPU map with a singleton
  *		element (key 0) addressed by variable offset.
+ * - tuples:	Tuple-to-id map.  This is a global hash map indexed with a
+ *		tuple.  The value associated with the tuple key is an id that
+ *		is used to index the dvars map.  The key size is determined as
+ *		the largest tuple used across all programs in the tracing
+ *		session.
  */
 int
 dt_bpf_gmap_create(dtrace_hdl_t *dtp)
@@ -249,9 +254,10 @@ dt_bpf_gmap_create(dtrace_hdl_t *dtp)
 	/* Determine sizes for global, local, and TLS maps. */
 	gvarsz = P2ROUNDUP(dt_idhash_datasize(dtp->dt_globals), 8);
 	lvarsz = P2ROUNDUP(dtp->dt_maxlvaralloc, 8);
+
 	if (dtp->dt_maxdvarsize)
-		dvarc = (dtp->dt_options[DTRACEOPT_DYNVARSIZE] /
-			 dtp->dt_maxdvarsize) + 1;
+		dvarc = dtp->dt_options[DTRACEOPT_DYNVARSIZE] /
+			dtp->dt_maxdvarsize;
 
 	/* Create global maps as long as there are no errors. */
 	dtp->dt_stmap_fd = create_gmap(dtp, "state", BPF_MAP_TYPE_ARRAY,
@@ -356,8 +362,10 @@ dt_bpf_gmap_create(dtrace_hdl_t *dtp)
 		int	fd;
 		char	dflt[dtp->dt_maxdvarsize];
 
+		/* Allocate one extra element for the default value. */
 		fd = create_gmap(dtp, "dvars", BPF_MAP_TYPE_HASH,
-				 sizeof(uint64_t), dtp->dt_maxdvarsize, dvarc);
+				 sizeof(uint64_t), dtp->dt_maxdvarsize,
+				 dvarc + 1);
 		if (fd == -1)
 			return -1;	/* dt_errno is set for us */
 
@@ -366,6 +374,11 @@ dt_bpf_gmap_create(dtrace_hdl_t *dtp)
 		dt_bpf_map_update(fd, &key, &dflt);
 	}
 
+	if (dtp->dt_maxtuplesize > 0 &&
+	    create_gmap(dtp, "tuples", BPF_MAP_TYPE_HASH,
+			dtp->dt_maxtuplesize, sizeof(uint64_t), dvarc) == -1)
+		return -1;		/* dt_errno is set for us */
+
 	/* Populate the 'cpuinfo' map. */
 	dt_bpf_map_update(ci_mapfd, &key, dtp->dt_conf.cpus);
 

libdtrace/dt_bpf.h

@@ -1,6 +1,6 @@
 /*
  * Oracle Linux DTrace.
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * http://oss.oracle.com/licenses/upl.
  */
@@ -29,6 +29,7 @@ extern "C" {
 #define DT_CONST_NSPEC	9
 #define DT_CONST_NCPUS	10
 #define DT_CONST_PC	11
+#define DT_CONST_TUPSZ	12
 
 extern int perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu,
 			   int group_fd, unsigned long flags);

libdtrace/dt_cc.c

@@ -1,6 +1,6 @@
 /*
  * Oracle Linux DTrace.
- * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * http://oss.oracle.com/licenses/upl.
  */
@@ -2351,6 +2351,9 @@ dt_link_construct(dtrace_hdl_t *dtp, const dt_probe_t *prp, dtrace_difo_t *dp,
 				nrp->dofr_data =
 					dtp->dt_options[DTRACEOPT_STRSIZE];
 				continue;
+			case DT_CONST_TUPSZ:
+				nrp->dofr_data = DMEM_TUPLE_SZ(dtp);
+				continue;
 			case DT_CONST_NSPEC:
 				nrp->dofr_data = dtp->dt_options[DTRACEOPT_NSPEC];
 				continue;