Add code for quantize() aggregation function

First, quantize the value into a 0-based bin number.  While we could
generate BPF code in dt_cg.c to accomplish this, it is easier to
maintain C code in the bpf/ subdirectory to be cross-compiled into
BPF when DTrace is built.

Then, the "implementation" function -- which needs to run twice, once
per aggregation copy -- merely updates the value in the designated
bin.  This "implementation" can be used for lquantize() and llquantize()
as well.  For that matter, it could be used for other aggregations such
as sum(), though that special case has only one bin.  The function tests
the bin number against 0 and some maxbin, both as a safety check and
because the BPF verifier needs such assurances in certain cases.

Signed-off-by: Eugene Loh <eugene.loh@oracle.com>
Reviewed-by: Kris Van Hees <kris.van.hees@oracle.com>

bpf/Build

@@ -22,6 +22,7 @@ bpf_dlib_TARGET = dlibs/bpf_dlib
 bpf_dlib_DIR := $(current-dir)
 bpf_dlib_SRCDEPS = $(objdir)/include/.dir.stamp
 bpf_dlib_SOURCES = \
+	agg_qbin.c \
 	get_bvar.c \
 	get_gvar.c set_gvar.c \
 	get_tvar.c set_tvar.c \

bpf/agg_qbin.c

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ */
+#include <linux/bpf.h>
+#include <stdint.h>
+#include <bpf-helpers.h>
+
+#ifndef noinline
+# define noinline	__attribute__((noinline))
+#endif
+
+noinline int16_t dt_agg_qbin(int64_t val)
+{
+	int16_t off;
+	uint64_t tmp;
+
+	if (val == 0)
+		return 63;
+	if (val == 0x8000000000000000)
+		return 0;
+
+	tmp = val;
+	if (val < 0)
+		tmp *= -1;
+
+	/* now, tmp has at least one 1, while the leading bit is 0 */
+	off = 1;
+	if (tmp & 0x7fffffff00000000) { off += 32; tmp >>= 32; }
+	if (tmp & 0x00000000ffff0000) { off += 16; tmp >>= 16; }
+	if (tmp & 0x000000000000ff00) { off +=  8; tmp >>=  8; }
+	if (tmp & 0x00000000000000f0) { off +=  4; tmp >>=  4; }
+	if (tmp & 0x000000000000000c) { off +=  2; tmp >>=  2; }
+	if (tmp & 0x0000000000000002) { off +=  1; }
+
+	if (val < 0)
+		off *= -1;
+	off += 63;
+	return off;
+}

libdtrace/dt_cg.c

@@ -3214,6 +3214,34 @@ dt_cg_agg_count(dt_pcb_t *pcb, dt_ident_t *aid, dt_node_t *dnp,
 	TRACE_REGSET("    AggCnt: End  ");
 }
 
+static void
+dt_cg_agg_quantize_impl(dt_irlist_t *dlp, dt_regset_t *drp, int dreg, int vreg, int ireg, int maxbin)
+{
+	uint_t		L = dt_irlist_label(dlp);
+	int		offreg;
+
+	TRACE_REGSET("            Impl: Begin");
+
+	if ((offreg = dt_regset_alloc(drp)) == -1)
+		longjmp(yypcb->pcb_jmpbuf, EDT_NOREG);
+
+	/* check bounds */
+	emit(dlp,  BPF_BRANCH_IMM(BPF_JGT, vreg, maxbin, L));
+	emit(dlp,  BPF_BRANCH_IMM(BPF_JLT, vreg, 0, L));
+
+	/* *(dest + 8 * off) += incr */
+	emit(dlp,  BPF_MOV_REG(offreg, vreg));
+	emit(dlp,  BPF_ALU64_IMM(BPF_MUL, offreg, 8));
+	emit(dlp,  BPF_ALU64_REG(BPF_ADD, offreg, dreg));
+	emit(dlp,  BPF_XADD_REG(BPF_DW, offreg, 0, ireg));
+
+	emitl(dlp, L,
+		   BPF_NOP());
+	dt_regset_free(drp, offreg);
+
+	TRACE_REGSET("            Impl: End  ");
+}
+
 static void
 dt_cg_agg_llquantize(dt_pcb_t *pcb, dt_ident_t *aid, dt_node_t *dnp,
 		     dt_irlist_t *dlp, dt_regset_t *drp)
@@ -3636,12 +3664,57 @@ static void
 dt_cg_agg_quantize(dt_pcb_t *pcb, dt_ident_t *aid, dt_node_t *dnp,
 		   dt_irlist_t *dlp, dt_regset_t *drp)
 {
+	dt_ident_t	*idp;
 	dt_node_t	*incr;
-	int		sz = DTRACE_QUANTIZE_NBUCKETS * sizeof(uint64_t);
+	int		ireg, sz = DTRACE_QUANTIZE_NBUCKETS * sizeof(uint64_t);
+
+	/*
+	 * The quantize() implementation is currently hardwired for
+	 *     DTRACE_QUANTIZE_NBUCKETS 127
+	 *     DTRACE_QUANTIZE_ZEROBUCKET 63
+	 * These values are defined in include/dtrace/actions_defines.h
+	 */
+	assert(DTRACE_QUANTIZE_NBUCKETS == 127 &&
+	    DTRACE_QUANTIZE_ZEROBUCKET == 63);
 
 	incr = dt_cg_agg_opt_incr(dnp, dnp->dn_aggfun->dn_args, "quantize", 2);
 
 	DT_CG_AGG_SET_STORAGE(aid, sz);
+
+	TRACE_REGSET("    AggQ  : Begin");
+
+	dt_cg_node(dnp->dn_aggfun->dn_args, dlp, drp);
+
+	/* quantize the value to a 0-based bin # using dt_agg_qbin() */
+	if (dt_regset_xalloc_args(drp) == -1)
+		longjmp(yypcb->pcb_jmpbuf, EDT_NOREG);
+	emit(dlp,  BPF_MOV_REG(BPF_REG_1, dnp->dn_aggfun->dn_args->dn_reg));
+	idp = dt_dlib_get_func(yypcb->pcb_hdl, "dt_agg_qbin");
+	assert(idp != NULL);
+	dt_regset_xalloc(drp, BPF_REG_0);
+	emite(dlp, BPF_CALL_FUNC(idp->di_id), idp);
+	dt_regset_free_args(drp);
+	emit(dlp,  BPF_MOV_REG(dnp->dn_aggfun->dn_args->dn_reg, BPF_REG_0));
+	dt_regset_free(drp, BPF_REG_0);
+
+	if (incr == NULL) {
+		if ((ireg = dt_regset_alloc(drp)) == -1)
+			longjmp(yypcb->pcb_jmpbuf, EDT_NOREG);
+
+		emit(dlp, BPF_MOV_IMM(ireg, 1));
+	} else {
+		dt_cg_node(incr, dlp, drp);
+		ireg = incr->dn_reg;
+	}
+
+	DT_CG_AGG_IMPL(aid, sz, dlp, drp, dt_cg_agg_quantize_impl,
+		       dnp->dn_aggfun->dn_args->dn_reg, ireg,
+		       DTRACE_QUANTIZE_NBUCKETS - 1);
+
+	dt_regset_free(drp, dnp->dn_aggfun->dn_args->dn_reg);
+	dt_regset_free(drp, ireg);
+
+	TRACE_REGSET("    AggQ  : End  ");
 }
 
 static void

libdtrace/dt_dlibs.c

@@ -52,6 +52,7 @@ static const dt_ident_t		dt_bpf_symbols[] = {
 	/* BPF built-in functions */
 	DT_BPF_SYMBOL(dt_program, DT_IDENT_FUNC),
 	/* BPF library (external) functions */
+	DT_BPF_SYMBOL(dt_agg_qbin, DT_IDENT_SYMBOL),
 	DT_BPF_SYMBOL(dt_get_bvar, DT_IDENT_SYMBOL),
 	DT_BPF_SYMBOL(dt_get_gvar, DT_IDENT_SYMBOL),
 	DT_BPF_SYMBOL(dt_get_string, DT_IDENT_SYMBOL),