Implement drop-counter support

DTrace provides a mechanism to report dropped events.  A drop occurs
when trace data cannot be recorded for a particular reason.  This
patch contains the full implementation of 4 categories of drops,
primarily because they are all closely related:

- principal buffer drops: reported when the producer failed to add
  trace data to the principal buffer
- aggregation buffer drops: reported when the producer failed to
  allocate an aggreggation and therefore failed to record data
- speculation drops: reported when something goes wrong with the
  recording of speculative tracing data
    + regular drops: reported when speculative data could not be
      written to a speculation buffer
    + busy drops: reported when a speculation could not be created
      because all buffers are busy being committed or discarded
    + unavailable drops: reported when no available speculation
      buffers were found
- dynamic variable drops: reported when a dynamic variable (or
  associative array element) could not be allocated

Two mechanisms for reporting drops are needed:

(1) Per-CPU reporting: used for principal and aggregation buffer
    drops (stored in the cpuinfo structures)
(2) Global reporting: used for speculation and dynamic variable
    drops (stored in the state BPF map)

Detection of drops (and subsequent reporting to the user) is done
through frequent retrieval of status data.  The handling of status
data (and the use of statusrate) is being re-introduced with this
patch.

The drop count for speculations is a bit more complex than the other
ones because drops can occur both in the producer (when data cannot
be written to the trace output buffer) *and* in the consumer (when
data cannot be recorded in a speculation buffer).  These separate
counts are combined whenever status processing takes place to ensure
the correct drop count is presented to the user.

Various tests have updated expected results because drops are now
being reported correctly.

Signed-off-by: Kris Van Hees <kris.van.hees@oracle.com>
Reviewed-by: Eugene Loh <eugene.loh@oracle.com>

bpf/get_agg.c

@@ -5,13 +5,32 @@
 #include <linux/bpf.h>
 #include <stdint.h>
 #include <bpf-helpers.h>
+#include <bpf-lib.h>
+#include <dt_bpf_maps.h>
 #include <dt_dctx.h>
 
 #ifndef noinline
 # define noinline	__attribute__((noinline))
 #endif
 
 extern struct bpf_map_def agggen;
+extern struct bpf_map_def cpuinfo;
+
+/*
+ * Register an aggregation drop.
+ */
+noinline uint64_t *dt_no_agg(void)
+{
+	uint32_t		key = 0;
+	dt_bpf_cpuinfo_t	*ci;
+
+	ci = bpf_map_lookup_elem(&cpuinfo, &key);
+	if (ci == 0)
+		return 0;
+
+	atomic_add(&ci->agg_drops, 1);
+	return 0;
+}
 
 /*
  * Get a pointer to the data storage for an aggregation.  Regular aggregations
@@ -29,7 +48,7 @@ noinline uint64_t *dt_get_agg(const dt_dctx_t *dctx, uint32_t id,
 	/* get the gen value */
 	genp = bpf_map_lookup_elem(&agggen, &id);
 	if (genp == 0)
-		return 0;
+		return dt_no_agg();
 
 	/* place the variable ID at the beginning of the key */
 	*(uint32_t *)key = id;
@@ -41,11 +60,11 @@ noinline uint64_t *dt_get_agg(const dt_dctx_t *dctx, uint32_t id,
 	if (valp == 0 || valp[0] < *genp) {
 		/* start with all zeroes */
 		if (bpf_map_update_elem(dctx->agg, key, dflt, BPF_ANY) < 0)
-			return 0;
+			return dt_no_agg();
 
 		valp = bpf_map_lookup_elem(dctx->agg, key);
 		if (valp == 0)
-			return 0;
+			return dt_no_agg();
 
 		/* ival is nonzero only for min() and max() */
 		if (ival)

bpf/get_dvar.c

@@ -1,19 +1,38 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  */
 #include <linux/bpf.h>
 #include <stdint.h>
 #include <bpf-helpers.h>
+#include <bpf-lib.h>
+#include <dt_dctx.h>
 
 #ifndef noinline
 # define noinline	__attribute__((noinline))
 #endif
 
 extern struct bpf_map_def dvars;
+extern struct bpf_map_def state;
 extern struct bpf_map_def tuples;
 extern uint64_t NCPUS;
 
+/*
+ * Register a adynamic variable drop.
+ */
+noinline void *dt_no_dvar(void)
+{
+	uint32_t	kind = DT_STATE_DYNVAR_DROPS;
+	uint32_t	*valp;
+
+	valp = bpf_map_lookup_elem(&state, &kind);
+	if (valp == 0)
+		return 0;
+
+	atomic_add32(valp, 1);
+	return 0;
+}
+
 /*
  * Dynamic variables are identified using a unique 64-bit key.  Three different
  * categories of dynamic variables are supported in DTrace:
@@ -80,13 +99,13 @@ noinline void *dt_get_dvar(uint64_t key, uint64_t store, uint64_t nval,
 	 * with the default value.
 	 */
 	if (bpf_map_update_elem(&dvars, &key, dflt, BPF_ANY) < 0)
-		return 0;
+		return dt_no_dvar();
 
 	val = bpf_map_lookup_elem(&dvars, &key);
 	if (val != 0)
 		return val;
 
-	return 0;
+	return dt_no_dvar();
 }
 
 noinline void *dt_get_tvar(uint32_t id, uint64_t store, uint64_t nval,
@@ -126,13 +145,13 @@ noinline void *dt_get_assoc(uint32_t id, const char *tuple, uint64_t store,
 		 * actual value.
 		 */
 		if (bpf_map_update_elem(&tuples, tuple, &dflt_val, BPF_ANY) < 0)
-			return 0;
+			return dt_no_dvar();
 		valp = bpf_map_lookup_elem(&tuples, tuple);
 		if (valp == 0)
-			return 0;
+			return dt_no_dvar();
 		*valp = (uint64_t)valp;
 		if (bpf_map_update_elem(&tuples, tuple, valp, BPF_ANY) < 0)
-			return 0;
+			return dt_no_dvar();
 
 		val = *valp;
 	} else {

bpf/speculation.c

@@ -19,18 +19,36 @@
 #endif
 
 extern struct bpf_map_def specs;
+extern struct bpf_map_def state;
 extern uint64_t NSPEC;
 
+/*
+ * Register a speculation drop.
+ */
+noinline uint32_t dt_no_spec(uint32_t kind)
+{
+	uint32_t	*valp;
+
+	valp = bpf_map_lookup_elem(&state, &kind);
+	if (valp == 0)
+		return 0;
+
+	atomic_add32(valp, 1);
+	return 0;
+}
+
 /*
  * Assign a speculation ID.
  */
 noinline uint32_t dt_speculation(void)
 {
-	uint32_t id;
-	dt_bpf_specs_t zero;
+	uint32_t	id, busy;
+	dt_bpf_specs_t	zero;
+	dt_bpf_specs_t	*spec;
 
 	__builtin_memset(&zero, 0, sizeof (dt_bpf_specs_t));
 
+	busy = 0;
 #if 1 /* Loops are broken in BPF right now */
 #define SEARCH(n)							\
 	do {								\
@@ -40,6 +58,11 @@ noinline uint32_t dt_speculation(void)
 		if (bpf_map_update_elem(&specs, &id, &zero,		\
 			BPF_NOEXIST) == 0)				\
 			return id;					\
+		spec = bpf_map_lookup_elem(&specs, &id);		\
+		if (spec != 0 && spec->draining > 0) {			\
+			busy++;						\
+			break;						\
+		}							\
 	} while (0);
 
 	SEARCH(1);
@@ -66,10 +89,16 @@ noinline uint32_t dt_speculation(void)
 		if (bpf_map_update_elem(&specs, &id, &zero,
 					BPF_NOEXIST) == 0)
 			return id;
+
+		spec = bpf_map_lookup_elem(&specs, &id);
+		if (spec != 0 && spec->draining > 0)  {
+			busy++;
+			break;
+		}
 	}
 #endif
 
-	return 0;
+	return dt_no_spec(busy ? DT_STATE_SPEC_BUSY : DT_STATE_SPEC_UNAVAIL);
 }
 
 /*

libdtrace/dt_aggregate.c

@@ -605,21 +605,24 @@ dt_aggregate_snap_one(dtrace_hdl_t *dtp, int aggid, int cpu, const char *key,
 static int
 dt_aggregate_snap_cpu(dtrace_hdl_t *dtp, processorid_t cpu, int fd)
 {
-	dt_aggregate_t	*agp = &dtp->dt_aggregate;
-	size_t		ksize = dtp->dt_maxtuplesize;
-	char		*key = agp->dtat_key;
-	char		*data = agp->dtat_buf;
-	char		*nxt = agp->dtat_nextkey;
-	uint32_t	*aggidp = (uint32_t *)key;
-	int		rval;
-
+	dt_aggregate_t		*agp = &dtp->dt_aggregate;
+	size_t			ksize = dtp->dt_maxtuplesize;
+	char			*key = agp->dtat_key;
+	char			*data = agp->dtat_buf;
+	char			*nxt = agp->dtat_nextkey;
+	uint32_t		*aggidp = (uint32_t *)key;
+	int			rval;
 
 	*aggidp = DTRACE_AGGIDNONE;
 	while (dt_bpf_map_next_key(fd, key, nxt) == 0) {
+		rval = dt_check_cpudrops(dtp, cpu, DTRACEDROP_AGGREGATION);
+		if (rval != 0)
+			return rval;
+
 		memcpy(key, nxt, ksize);
 
 		if (dt_bpf_map_lookup(fd, key, data) == -1)
-			return -1; /* FIXME: dt_set_errno() */
+			return dt_set_errno(dtp, EDT_BPF);
 
 		rval = dt_aggregate_snap_one(dtp, *aggidp, cpu, key, data);
 		if (rval != 0)

libdtrace/dt_bpf.c

@@ -604,26 +604,29 @@ gmap_create_buffers(dtrace_hdl_t *dtp)
 static int
 gmap_create_cpuinfo(dtrace_hdl_t *dtp)
 {
-	int			i, fd, rc;
+	int			i, rc;
 	uint32_t		key = 0;
 	dtrace_conf_t		*conf = &dtp->dt_conf;
 	size_t			ncpus = conf->max_cpuid + 1;
 	dt_bpf_cpuinfo_t	*data;
 	cpuinfo_t		*ci;
 
-	data = dt_zalloc(dtp, ncpus * sizeof(dt_bpf_cpuinfo_t));
+	data = dt_calloc(dtp, dtp->dt_conf.num_possible_cpus,
+			 sizeof(dt_bpf_cpuinfo_t));
 	if (data == NULL)
 		return dt_set_errno(dtp, EDT_NOMEM);
 
 	for (i = 0, ci = &conf->cpus[0]; i < ncpus; i++, ci++)
 		memcpy(&data[ci->cpu_id].ci, ci, sizeof(cpuinfo_t));
 
-	fd = create_gmap(dtp, "cpuinfo", BPF_MAP_TYPE_PERCPU_ARRAY,
-			 sizeof(uint32_t), sizeof(dt_bpf_cpuinfo_t), 1);
-	if (fd == -1)
+	dtp->dt_cpumap_fd = create_gmap(dtp, "cpuinfo",
+					BPF_MAP_TYPE_PERCPU_ARRAY,
+					sizeof(uint32_t),
+					sizeof(dt_bpf_cpuinfo_t), 1);
+	if (dtp->dt_cpumap_fd == -1)
 		return -1;
 
-	rc = dt_bpf_map_update(fd, &key, data);
+	rc = dt_bpf_map_update(dtp->dt_cpumap_fd, &key, data);
 	dt_free(dtp, data);
 	if (rc == -1)
 		return dt_bpf_error(dtp,