hist: log2 histograms with finer granularity

Allow a second optional argument in hist(n, k) to map each power of 2 into 2^k buckets, thus creating a logarithmic scale with finer granularity and modest runtime overhead (a couple of shifts and add/mask in addition to the original algorithm). Allowed values of k are 0..5, with 0 as default for backward compatibility. The implementation follows my earlier code in https://github.com/luigirizzo/lr-cstats Example below: $ sudo src/bpftrace -e 'kfunc:tick_do_update_jiffies64 { @ = hist((nsecs & 0xff),2); }' Attaching 2 probes... @: [0] 4 |@ | [1] 1 | | [2] 3 |@ | [3] 2 | | [4] 3 |@ | [5] 0 | | [6] 3 |@ | [7] 2 | | [8, 10) 5 |@ | [10, 12) 7 |@@ | [12, 14) 5 |@ | [14, 16) 6 |@@ | [16, 20) 11 |@@@ | [20, 24) 14 |@@@@ | [24, 28) 20 |@@@@@@ | [28, 32) 13 |@@@@ | [32, 40) 40 |@@@@@@@@@@@@@ | [40, 48) 38 |@@@@@@@@@@@@@ | [48, 56) 35 |@@@@@@@@@@@ | [56, 64) 29 |@@@@@@@@@ | [64, 80) 72 |@@@@@@@@@@@@@@@@@@@@@@@@ | [80, 96) 64 |@@@@@@@@@@@@@@@@@@@@@ | [96, 112) 61 |@@@@@@@@@@@@@@@@@@@@ | [112, 128) 67 |@@@@@@@@@@@@@@@@@@@@@@ | [128, 160) 124 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | [160, 192) 130 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | [192, 224) 124 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ | [224, 256) 152 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
bpftrace · Nov 19, 2023 · 6d193bb · 6d193bb
1 parent baf41c9
commit 6d193bb
Show file tree

Hide file tree

Showing 11 changed files with 152 additions and 109 deletions.
diff --git a/docs/reference_guide.md b/docs/reference_guide.md
@@ -3389,7 +3389,7 @@ END
 - `min(int n)` - Record the minimum value seen
 - `max(int n)` - Record the maximum value seen
 - `stats(int n)` - Return the count, average, and total for this value
-- `hist(int n)` - Produce a log2 histogram of values of n
+- `hist(int n[, int k])` - Produce a log2 histogram of values of n with 2^k buckets per power of 2
 - `lhist(int n, int min, int max, int step)` - Produce a linear histogram of values of n
 - `delete(@x[key])` - Delete the map element passed in as an argument
 - `print(@x[, top [, div]])` - Print the map, optionally the top entries only and with a divisor
@@ -3577,11 +3577,15 @@ and the total of the argument value. This is similar to using count(), avg(), an
 Syntax:
 
 ```
-@histogram_name[optional_key] = hist(value)
+@histogram_name[optional_key] = hist(value[, k])
 ```
 
 This is implemented using a BPF map.
 
+Values are accumulated in 2^k buckets for each power of 2,
+with negative values in their own bucket.
+k can be 0..5, defaults to 0.
+
 Examples:
 
 ### 8.1. Power-Of-2:

diff --git a/man/adoc/bpftrace.adoc b/man/adoc/bpftrace.adoc
@@ -1237,9 +1237,10 @@ k:dummy {
 === hist
 
 .variants
-* `hist(int64 n)`
+* `hist(int64 n[, int k])`
 
-Create a log2 histogram of `n`.
+Create a log2 histogram of `n`C using $2^k$ buckets per power of 2,
+0 <= k <= 5, defaults to 0.
 
 ----
 kretprobe:vfs_read {

diff --git a/src/ast/passes/codegen_llvm.cpp b/src/ast/passes/codegen_llvm.cpp
@@ -543,12 +543,19 @@ void CodegenLLVM::visit(Call &call)
       log2_func_ = createLog2Function();
 
     Map &map = *call.map;
+    // There is only one log2_func_ so the second argument must be passed
+    // as an argument even though it is a constant 0..5
+    // Possible optimization is create one function per different value
+    // of the second argument.
+    auto scoped_del_arg2 = accept(call.vargs->at(1));
+    Value *k = b_.CreateIntCast(expr_, b_.getInt64Ty(), false);
+
     auto scoped_del = accept(call.vargs->front());
     // promote int to 64-bit
     expr_ = b_.CreateIntCast(expr_,
                              b_.getInt64Ty(),
                              call.vargs->front()->type.IsSigned());
-    Value *log2 = b_.CreateCall(log2_func_, expr_, "log2");
+    Value *log2 = b_.CreateCall(log2_func_, {expr_, k}, "log2");
     AllocaInst *key = getHistMapKey(map, log2);
     b_.CreateMapElemAdd(ctx_, map, key, b_.getInt64(1), call.loc);
     b_.CreateLifetimeEnd(key);
@@ -3054,81 +3061,99 @@ Value *CodegenLLVM::createLogicalOr(Binop &binop)
 Function *CodegenLLVM::createLog2Function()
 {
   auto ip = b_.saveIP();
-  // log2() returns a bucket index for the given value. Index 0 is for
-  // values less than 0, index 1 is for 0, and indexes 2 onwards is the
-  // power-of-2 histogram index.
+  // Arguments: N (int64), K (0..5)
+  // Maps each power of 2 into 2^K buckets, so we can build fine-grained
+  // histograms with low runtime cost.
+  //
+  // Returns 0 for N < 0, 1..2^K for 0 <= N < 2^K; otherwise
+  // N has the form ..01xxx... where the leftmost is at bit L >= K,
+  // xxx are the following K bits with a value 0 <= X < 2^K.
+  // In this case the function returns 1 + (L-K + 1) * 2^K + X
   //
-  // log2(int n)
+  // log2(int n, int k)
   // {
-  //   int result = 0;
-  //   int shift;
-  //   if (n < 0) return result;
-  //   result++;
-  //   if (n == 0) return result;
-  //   result++;
-  //   for (int i = 5; i >= 0; i--)
-  //   {
-  //     shift = (n >= (1<<(1<<i))) << i;
+  //   if (n < 0) return 0;
+  //   mask = (1ul << k) - 1;
+  //   if (n <= mask) return n + 1;
+  //   n0 = n;
+  //   // find leftmost 1
+  //   l = 0;
+  //   for (int i = 5; i >= 0; i--) {
+  //     threshold = 1ul << (1<<i)
+  //     shift = (n >= threshold) << i;
   //     n >>= shift;
-  //     result += shift;
+  //     l += shift;
   //   }
-  //   return result;
+  //   l -= k;
+  //   // mask K bits after leftmost 1
+  //   x = (n0 >> l) & mask;
+  //   return ((l + 1) << k) + x;
   // }
 
-  FunctionType *log2_func_type = FunctionType::get(b_.getInt64Ty(), {b_.getInt64Ty()}, false);
+  FunctionType *log2_func_type = FunctionType::get(b_.getInt64Ty(), {b_.getInt64Ty(), b_.getInt64Ty()}, false);
   Function *log2_func = Function::Create(log2_func_type, Function::InternalLinkage, "log2", module_.get());
   log2_func->addFnAttr(Attribute::AlwaysInline);
   log2_func->setSection("helpers");
   BasicBlock *entry = BasicBlock::Create(module_->getContext(), "entry", log2_func);
   b_.SetInsertPoint(entry);
 
-  // setup n and result registers
-  Value *arg = log2_func->arg_begin();
+  // storage for arguments
   Value *n_alloc = b_.CreateAllocaBPF(CreateUInt64());
-  b_.CreateStore(arg, n_alloc);
-  Value *result = b_.CreateAllocaBPF(CreateUInt64());
-  b_.CreateStore(b_.getInt64(0), result);
+  b_.CreateStore(log2_func->arg_begin(), n_alloc);
+  Value *k_alloc = b_.CreateAllocaBPF(CreateUInt64());
+  b_.CreateStore(log2_func->arg_begin() + 1, k_alloc);
 
   // test for less than zero
   BasicBlock *is_less_than_zero = BasicBlock::Create(module_->getContext(), "hist.is_less_than_zero", log2_func);
   BasicBlock *is_not_less_than_zero = BasicBlock::Create(module_->getContext(), "hist.is_not_less_than_zero", log2_func);
-  b_.CreateCondBr(b_.CreateICmpSLT(b_.CreateLoad(b_.getInt64Ty(), n_alloc),
-                                   b_.getInt64(0)),
-                  is_less_than_zero,
-                  is_not_less_than_zero);
+
+  Value *n =  b_.CreateLoad(b_.getInt64Ty(), n_alloc);
+  Value *zero = b_.getInt64(0);
+  b_.CreateCondBr(b_.CreateICmpSLT(n, zero), is_less_than_zero, is_not_less_than_zero);
+
   b_.SetInsertPoint(is_less_than_zero);
-  createRet(b_.CreateLoad(b_.getInt64Ty(), result));
+  createRet(zero);
+
   b_.SetInsertPoint(is_not_less_than_zero);
 
-  // test for equal to zero
+  // first set of buckets (<= mask)
+  Value *one = b_.getInt64(1);
+  Value *k =  b_.CreateLoad(b_.getInt64Ty(), k_alloc);
+  Value *mask = b_.CreateSub(b_.CreateShl(one, k), one);
+
   BasicBlock *is_zero = BasicBlock::Create(module_->getContext(), "hist.is_zero", log2_func);
   BasicBlock *is_not_zero = BasicBlock::Create(module_->getContext(), "hist.is_not_zero", log2_func);
-  b_.CreateCondBr(b_.CreateICmpEQ(b_.CreateLoad(b_.getInt64Ty(), n_alloc),
-                                  b_.getInt64(0)),
-                  is_zero,
-                  is_not_zero);
+  b_.CreateCondBr(b_.CreateICmpULE(n, mask), is_zero, is_not_zero);
+
   b_.SetInsertPoint(is_zero);
-  b_.CreateStore(b_.getInt64(1), result);
-  createRet(b_.CreateLoad(b_.getInt64Ty(), result));
+  createRet(b_.CreateAdd(n, one));
+
   b_.SetInsertPoint(is_not_zero);
 
-  // power-of-2 index, offset by +2
-  b_.CreateStore(b_.getInt64(2), result);
+  // index of first bit set in n, 1 means bit 0, guaranteed to be >= k
+  Value *l = zero;
   for (int i = 5; i >= 0; i--)
   {
-    Value *n = b_.CreateLoad(b_.getInt64Ty(), n_alloc);
-    Value *shift = b_.CreateShl(
-        b_.CreateIntCast(
-            b_.CreateICmpSGE(b_.CreateIntCast(n, b_.getInt64Ty(), false),
-                             b_.getInt64(1ULL << (1 << i))),
-            b_.getInt64Ty(),
-            false),
-        i);
-    b_.CreateStore(b_.CreateLShr(n, shift), n_alloc);
-    b_.CreateStore(b_.CreateAdd(b_.CreateLoad(b_.getInt64Ty(), result), shift),
-                   result);
-  }
-  createRet(b_.CreateLoad(b_.getInt64Ty(), result));
+    Value *threshold = b_.getInt64(1ul << (1ul << i));
+    Value *is_ge = b_.CreateICmpSGE(n, threshold);
+    is_ge = b_.CreateIntCast(is_ge, b_.getInt64Ty(), false); // cast is important.
+    Value *shift = b_.CreateShl(is_ge, i);
+    n = b_.CreateLShr(n, shift);
+    l = b_.CreateAdd(l, shift);
+  }
+
+  // see algorithm for next steps:
+  // subtract k, so we can move the next k bits of N to position 0
+  l = b_.CreateSub(l, k);
+  // now find the k bits in n after the first '1'
+  Value *x = b_.CreateAnd(b_.CreateLShr(b_.CreateLoad(b_.getInt64Ty(), n_alloc), l), mask);
+
+  Value *ret = b_.CreateAdd(l, one);
+  ret = b_.CreateShl(ret, k); // make room for the extra slots
+  ret = b_.CreateAdd(ret, x);
+  ret = b_.CreateAdd(ret, one);
+  createRet(ret);
+
   b_.restoreIP(ip);
   return module_->getFunction("log2");
 }

diff --git a/src/ast/passes/resource_analyser.cpp b/src/ast/passes/resource_analyser.cpp
@@ -132,6 +132,17 @@ void ResourceAnalyser::visit(Call &call)
     resources_.join_args.push_back(delim);
     resources_.needs_join_map = true;
   }
+  else if (call.func == "hist")
+  {
+    auto &r = resources_.hist_bits_arg;
+    int old, bits = static_cast<Integer *>(call.vargs->at(1))->n;
+    if (r.find(call.map->ident) != r.end() && (old = r[call.map->ident]) != bits) {
+      LOG(ERROR, call.loc, err_)
+          << "Different bits in a single hist, had " << old << " now " << bits;
+    } else {
+       r[call.map->ident] = bits;
+    }
+  }
   else if (call.func == "lhist")
   {
     Expression &min_arg = *call.vargs->at(1);

diff --git a/src/ast/passes/semantic_analyser.cpp b/src/ast/passes/semantic_analyser.cpp
@@ -554,7 +554,16 @@ void SemanticAnalyser::visit(Call &call)
 
   if (call.func == "hist") {
     check_assignment(call, true, false, false);
-    check_nargs(call, 1);
+    check_varargs(call, 1, 2);
+    if (call.vargs->size() == 1) {
+      call.vargs->push_back(new Integer(0, call.loc)); // default bits is 0
+    } else {
+      check_arg(call, Type::integer, 1);
+      const auto bits = bpftrace_.get_int_literal(call.vargs->at(1));
+      if (bits < 0 || bits > 5) {
+        LOG(ERROR, call.loc, err_) << call.func << ": bits " << *bits << " must be 0..5";
+      }
+    }
     check_arg(call, Type::integer, 0);
 
     call.type = CreateHist();

diff --git a/src/bpftrace.cpp b/src/bpftrace.cpp
@@ -1779,7 +1779,7 @@ int BPFtrace::print_map_hist(IMap &map, uint32_t top, uint32_t div)
     {
       // New key - create a list of buckets for it
       if (map.type_.IsHistTy())
-        values_by_key[key_prefix] = std::vector<uint64_t>(65);
+        values_by_key[key_prefix] = std::vector<uint64_t>(65 * 32);  // space for up to 5 bits
       else
         values_by_key[key_prefix] = std::vector<uint64_t>(1002);
     }

diff --git a/src/imap.h b/src/imap.h
@@ -55,6 +55,7 @@ class IMap
   int lqmax = 0;
   int lqstep = 0;
 
+  int bits() const { return lqstep; } // used in "hist()"
   bool is_per_cpu_type()
   {
     return map_type_ == libbpf::BPF_MAP_TYPE_PERCPU_HASH ||

diff --git a/src/output.cpp b/src/output.cpp
@@ -35,32 +35,19 @@ std::ostream& operator<<(std::ostream& out, MessageType type) {
   return out;
 }
 
-std::string TextOutput::hist_index_label(int power)
+// power must be >= 2<<bits ?
+std::string TextOutput::hist_index_label(int power, int bits)
 {
-  char suffix = '\0';
-  if (power >= 40)
-  {
-    suffix = 'T';
-    power -= 40;
-  }
-  else if (power >= 30)
-  {
-    suffix = 'G';
-    power -= 30;
-  }
-  else if (power >= 20)
-  {
-    suffix = 'M';
-    power -= 20;
-  }
-  else if (power >= 10)
-  {
-    suffix = 'K';
-    power -= 10;
-  }
+  uint32_t n = (1 << bits), frac = power & (n - 1);
+
+  power = (power >> bits) - 1;
+
+  int decade = power / 10;
+  char suffix = "\0KMGTPE"[decade];
+  power -= 10 * decade;
 
   std::ostringstream label;
-  label << (1<<power);
+  label << (1<<power) * (n + frac);
   if (suffix)
     label << suffix;
   return label.str();
@@ -359,7 +346,7 @@ std::string Output::map_hist_to_str(
 
     auto key_str = map_key_to_str(bpftrace, map, key);
     auto val_str = map.type_.IsHistTy()
-                       ? hist_to_str(value, div)
+                       ? hist_to_str(value, div, map.bits())
                        : lhist_to_str(value, map.lqmin, map.lqmax, map.lqstep);
 
     elems.push_back(map_keyval_to_str(map, key_str, val_str));
@@ -426,7 +413,8 @@ void TextOutput::map(
 }
 
 std::string TextOutput::hist_to_str(const std::vector<uint64_t> &values,
-                                    uint32_t div) const
+                                    uint32_t div,
+                                    int bits) const
 {
   int min_index, max_index, max_value;
   hist_prepare(values, min_index, max_index, max_value);
@@ -441,18 +429,14 @@ std::string TextOutput::hist_to_str(const std::vector<uint64_t> &values,
     {
       header << "(..., 0)";
     }
-    else if (i == 1)
-    {
-      header << "[0]";
-    }
-    else if (i == 2)
+    else if (i <= (2 << bits))
     {
-      header << "[1]";
+      header << "[" << (i - 1) << "]";
     }
     else
     {
-      header << "[" << hist_index_label(i-2);
-      header << ", " << hist_index_label(i-2+1) << ")";
+      header << "[" << hist_index_label(i - 1, bits);
+      header << ", " << hist_index_label(i, bits) << ")";
     }
 
     int max_width = 52;
@@ -683,7 +667,8 @@ void JsonOutput::map(
 }
 
 std::string JsonOutput::hist_to_str(const std::vector<uint64_t> &values,
-                                    uint32_t div) const
+                                    uint32_t div,
+                                    int bits) const
 {
   int min_index, max_index, max_value;
   hist_prepare(values, min_index, max_index, max_value);
@@ -698,24 +683,19 @@ std::string JsonOutput::hist_to_str(const std::vector<uint64_t> &values,
       res << ", ";
 
     res << "{";
-    if (i == 0)
-    {
-      res << "\"max\": -1, ";
-    }
-    else if (i == 1)
-    {
-      res << "\"min\": 0, \"max\": 0, ";
-    }
-    else if (i == 2)
+    long low = i - 1, high = i - 1;  // covers negative and first 2<<bits slots
+    if (i > (2 << bits))
     {
-      res << "\"min\": 1, \"max\": 1, ";
-    }
-    else
-    {
-      long low = 1ULL << (i - 2);
-      long high = (1ULL << (i - 2 + 1)) - 1;
-      res << "\"min\": " << low << ", \"max\": " << high << ", ";
+      uint32_t power = (i - 1) >> bits, n = 1<<bits, frac = (i - 1) & (n - 1);
+      low = (1ul << (power - 1) ) * (n + frac);
+      power = i >> bits;
+      frac = i & (n - 1);
+      high = (1ul << (power - 1)) * (n + frac) - 1;
     }
+    if (low != -1)
+      res << "\"min\": " << low << ", ";
+    res << "\"max\": " << high << ", ";
+
     res << "\"count\": " << values.at(i) / div;
     res << "}";
   }