Render Pixi flamegraph using Elasticsearch as data source (#44)

* Add comments

* Add stricter typing

* Collate stack frame metadata

* Split grouping frames and creating a hash

defaultGroupBy previously did both and now we have separate methods.

* Decode file ID from base64 URL

* Rename creation methods

We use creation methods to construct instances of certain types.
Sometimes these instances allow for no arguments, thus, the created
instance is an instance with sensible defaults.

To make the intent clearer for readers and also to adhere to the
conventions used throughout the Kibana codebase, I renamed the creation
methods to use 'create*' instead of 'build*'.

src/plugins/profiling/common/callercallee.test.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import {
+  createCallerCalleeIntermediateNode,
+  fromCallerCalleeIntermediateNode,
+} from './callercallee';
+import { createStackFrameMetadata, hashFrameGroup } from './profiling';
+
+describe('Caller-callee operations', () => {
+  test('1', () => {
+    const parentFrame = createStackFrameMetadata({
+      FileID: '6bc50d345244d5956f93a1b88f41874d',
+      FrameType: 3,
+      AddressOrLine: 971740,
+      FunctionName: 'epoll_wait',
+      SourceID: 'd670b496cafcaea431a23710fb5e4f58',
+      SourceLine: 30,
+      ExeFileName: 'libc-2.26.so',
+      Index: 1,
+    });
+    const parent = createCallerCalleeIntermediateNode(parentFrame, 10);
+
+    const childFrame = createStackFrameMetadata({
+      FileID: '8d8696a4fd51fa88da70d3fde138247d',
+      FrameType: 3,
+      AddressOrLine: 67000,
+      FunctionName: 'epoll_poll',
+      SourceID: 'f0a7901dcefed6cc8992a324b9df733c',
+      SourceLine: 150,
+      ExeFileName: 'auditd',
+      Index: 0,
+    });
+    const child = createCallerCalleeIntermediateNode(childFrame, 10);
+
+    const root = createCallerCalleeIntermediateNode(createStackFrameMetadata(), 10);
+    root.callees.set(hashFrameGroup(child.frameGroup), child);
+    root.callees.set(hashFrameGroup(parent.frameGroup), parent);
+
+    const graph = fromCallerCalleeIntermediateNode(root);
+
+    // Modify original frames to verify graph does not contain references
+    parent.samples = 30;
+    child.samples = 20;
+
+    expect(graph.Callees[0].Samples).toEqual(10);
+    expect(graph.Callees[1].Samples).toEqual(10);
+  });
+});

src/plugins/profiling/common/callercallee.ts

@@ -6,44 +6,43 @@
  * Side Public License, v 1.
  */
 
-import { override } from '.';
+import { clone } from 'lodash';
+
 import {
-  buildFrameGroup,
   compareFrameGroup,
   defaultGroupBy,
   FrameGroup,
   FrameGroupID,
+  hashFrameGroup,
   StackFrameMetadata,
   StackTraceID,
 } from './profiling';
 
-export interface CallerCalleeIntermediateNode {
-  nodeID: FrameGroup;
+export type CallerCalleeIntermediateNode = {
+  frameGroup: FrameGroup;
   callers: Map<FrameGroupID, CallerCalleeIntermediateNode>;
   callees: Map<FrameGroupID, CallerCalleeIntermediateNode>;
   frameMetadata: Set<StackFrameMetadata>;
   samples: number;
-}
+};
 
-export function buildCallerCalleeIntermediateNode(
-  frame: StackFrameMetadata,
+export function createCallerCalleeIntermediateNode(
+  frameMetadata: StackFrameMetadata,
   samples: number
 ): CallerCalleeIntermediateNode {
-  let node: CallerCalleeIntermediateNode = {
-    nodeID: buildFrameGroup(),
+  return {
+    frameGroup: defaultGroupBy(frameMetadata),
     callers: new Map<FrameGroupID, CallerCalleeIntermediateNode>(),
     callees: new Map<FrameGroupID, CallerCalleeIntermediateNode>(),
-    frameMetadata: new Set<StackFrameMetadata>(),
+    frameMetadata: new Set<StackFrameMetadata>([frameMetadata]),
     samples: samples,
   };
-  node.frameMetadata.add(frame);
-  return node;
 }
 
-interface relevantTrace {
+type relevantTrace = {
   frames: StackFrameMetadata[];
   index: number;
-}
+};
 
 // selectRelevantTraces searches through a map that maps trace hashes to their
 // frames and only returns those traces that have a frame that are equivalent
@@ -59,7 +58,7 @@ function selectRelevantTraces(
   frames: Map<StackTraceID, StackFrameMetadata[]>
 ): Map<StackTraceID, relevantTrace> {
   const result = new Map<StackTraceID, relevantTrace>();
-  const rootString = defaultGroupBy(rootFrame);
+  const rootString = hashFrameGroup(defaultGroupBy(rootFrame));
   for (const [stackTraceID, frameMetadata] of frames) {
     if (rootFrame.FileID === '' && rootFrame.AddressOrLine === 0) {
       // If the root frame is empty, every trace is relevant, and all elements
@@ -74,7 +73,7 @@ function selectRelevantTraces(
       // Search for the right index of the root frame in the frameMetadata, and
       // set it in the result.
       for (let i = 0; i < frameMetadata.length; i++) {
-        if (rootString === defaultGroupBy(frameMetadata[i])) {
+        if (rootString === hashFrameGroup(defaultGroupBy(frameMetadata[i]))) {
           result.set(stackTraceID, {
             frames: frameMetadata,
             index: i,
@@ -87,7 +86,7 @@ function selectRelevantTraces(
 }
 
 function sortRelevantTraces(relevantTraces: Map<StackTraceID, relevantTrace>): StackTraceID[] {
-  const sortedRelevantTraces: StackTraceID[] = new Array(relevantTraces.size);
+  const sortedRelevantTraces = new Array<StackTraceID>();
   for (const trace of relevantTraces.keys()) {
     sortedRelevantTraces.push(trace);
   }
@@ -98,107 +97,145 @@ function sortRelevantTraces(relevantTraces: Map<StackTraceID, relevantTrace>): S
   });
 }
 
-export function buildCallerCalleeIntermediateRoot(
+// createCallerCalleeIntermediateRoot creates a graph in the internal
+// representation from a StackFrameMetadata that identifies the "centered"
+// function and the trace results that provide traces and the number of times
+// that the trace has been seen.
+//
+// The resulting data structure contains all of the data, but is not yet in the
+// form most easily digestible by others.
+export function createCallerCalleeIntermediateRoot(
   rootFrame: StackFrameMetadata,
   traces: Map<StackTraceID, number>,
   frames: Map<StackTraceID, StackFrameMetadata[]>
 ): CallerCalleeIntermediateNode {
-  const root = buildCallerCalleeIntermediateNode(rootFrame, 0);
+  // Create a node for the centered frame
+  const root = createCallerCalleeIntermediateNode(rootFrame, 0);
+
+  // Obtain only the relevant frames (e.g. frames that contain the root frame
+  // somewhere). If the root frame is "empty" (e.g. fileID is zero and line
+  // number is zero), all frames are deemed relevant.
   const relevantTraces = selectRelevantTraces(rootFrame, frames);
+
+  // For a deterministic result we have to walk the traces in a deterministic
+  // order. A deterministic result allows for deterministic UI views, something
+  // that users expect.
   const relevantTracesSorted = sortRelevantTraces(relevantTraces);
+
+  // Walk through all traces that contain the root. Increment the count of the
+  // root by the count of that trace. Walk "up" the trace (through the callers)
+  // and add the count of the trace to each caller. Then walk "down" the trace
+  // (through the callees) and add the count of the trace to each callee.
   for (const traceHash of relevantTracesSorted) {
     const trace = relevantTraces.get(traceHash)!;
+
+    // The slice of frames is ordered so that the leaf function is at index 0.
+    // This means that the "second part" of the slice are the callers, and the
+    // "first part" are the callees.
+    //
+    // We currently assume there are no callers.
     const callees = trace.frames;
     const samples = traces.get(traceHash)!;
 
     // Go through the callees, reverse iteration
-    let currentNode = root;
+    let currentNode = clone(root);
+    root.samples += samples;
     for (let i = callees.length - 1; i >= 0; i--) {
       const callee = callees[i];
-      const calleeName = defaultGroupBy(callee);
+      const calleeName = hashFrameGroup(defaultGroupBy(callee));
       let node = currentNode.callees.get(calleeName);
       if (node === undefined) {
-        node = buildCallerCalleeIntermediateNode(callee, samples);
+        node = createCallerCalleeIntermediateNode(callee, samples);
         currentNode.callees.set(calleeName, node);
       } else {
-        node.samples = samples;
+        node.samples += samples;
       }
       currentNode = node;
     }
   }
   return root;
 }
 
-export interface CallerCalleeNode {
+export type CallerCalleeNode = {
   Callers: CallerCalleeNode[];
   Callees: CallerCalleeNode[];
-
   FileID: string;
   FrameType: number;
   ExeFileName: string;
   FunctionID: string;
   FunctionName: string;
   AddressOrLine: number;
   FunctionSourceLine: number;
-
-  // symbolization fields - currently unused
   FunctionSourceID: string;
   FunctionSourceURL: string;
   SourceFilename: string;
   SourceLine: number;
-
   Samples: number;
-}
-
-const defaultCallerCalleeNode: CallerCalleeNode = {
-  Callers: [],
-  Callees: [],
-  FileID: '',
-  FrameType: 0,
-  ExeFileName: '',
-  FunctionID: '',
-  FunctionName: '',
-  AddressOrLine: 0,
-  FunctionSourceLine: 0,
-  FunctionSourceID: '',
-  FunctionSourceURL: '',
-  SourceFilename: '',
-  SourceLine: 0,
-  Samples: 0,
 };
 
-export function buildCallerCalleeNode(node: Partial<CallerCalleeNode> = {}): CallerCalleeNode {
-  return override(defaultCallerCalleeNode, node);
+export function createCallerCalleeNode(options: Partial<CallerCalleeNode> = {}): CallerCalleeNode {
+  const node = {} as CallerCalleeNode;
+
+  node.Callers = clone(options.Callers ?? []);
+  node.Callees = clone(options.Callees ?? []);
+  node.FileID = options.FileID ?? '';
+  node.FrameType = options.FrameType ?? 0;
+  node.ExeFileName = options.ExeFileName ?? '';
+  node.FunctionID = options.FunctionID ?? '';
+  node.FunctionName = options.FunctionName ?? '';
+  node.AddressOrLine = options.AddressOrLine ?? 0;
+  node.FunctionSourceLine = options.FunctionSourceLine ?? 0;
+  node.FunctionSourceID = options.FunctionSourceID ?? '';
+  node.FunctionSourceURL = options.FunctionSourceURL ?? '';
+  node.SourceFilename = options.SourceFilename ?? '';
+  node.SourceLine = options.SourceLine ?? 0;
+  node.Samples = options.Samples ?? 0;
+
+  return node;
 }
 
 // selectCallerCalleeData is the "standard" way of merging multiple frames into
 // one node. It simply takes the data from the first frame.
 function selectCallerCalleeData(frameMetadata: Set<StackFrameMetadata>, node: CallerCalleeNode) {
   for (const metadata of frameMetadata) {
+    node.FileID = metadata.FileID;
+    node.FrameType = metadata.FrameType;
     node.ExeFileName = metadata.ExeFileName;
     node.FunctionID = metadata.FunctionName;
     node.FunctionName = metadata.FunctionName;
+    node.AddressOrLine = metadata.AddressOrLine;
+
+    // Unknown/invalid offsets are currently set to 0.
+    //
+    // In this case we leave FunctionSourceLine=0 as a flag for the UI that the
+    // FunctionSourceLine should not be displayed.
+    //
+    // As FunctionOffset=0 could also be a legit value, this work-around needs
+    // a real fix. The idea for after GA is to change FunctionOffset=-1 to
+    // indicate unknown/invalid.
+    if (metadata.FunctionOffset > 0) {
+      node.FunctionSourceLine = metadata.SourceLine - metadata.FunctionOffset;
+    } else {
+      node.FunctionSourceLine = 0;
+    }
+
     node.FunctionSourceID = metadata.SourceID;
     node.FunctionSourceURL = metadata.SourceCodeURL;
-    node.FunctionSourceLine = metadata.FunctionLine;
-    node.SourceLine = metadata.SourceLine;
-    node.FrameType = metadata.FrameType;
     node.SourceFilename = metadata.SourceFilename;
-    node.FileID = metadata.FileID;
-    node.AddressOrLine = metadata.AddressOrLine;
+    node.SourceLine = metadata.SourceLine;
     break;
   }
 }
 
 function sortNodes(
   nodes: Map<FrameGroupID, CallerCalleeIntermediateNode>
 ): CallerCalleeIntermediateNode[] {
-  const sortedNodes: CallerCalleeIntermediateNode[] = new Array(nodes.size);
+  const sortedNodes = new Array<CallerCalleeIntermediateNode>();
   for (const node of nodes.values()) {
     sortedNodes.push(node);
   }
   return sortedNodes.sort((n1, n2) => {
-    return compareFrameGroup(n1.nodeID, n2.nodeID);
+    return compareFrameGroup(n1.frameGroup, n2.frameGroup);
   });
 }
 
@@ -208,7 +245,7 @@ function sortNodes(
 export function fromCallerCalleeIntermediateNode(
   root: CallerCalleeIntermediateNode
 ): CallerCalleeNode {
-  const node = buildCallerCalleeNode({ Samples: root.samples });
+  const node = createCallerCalleeNode({ Samples: root.samples });
 
   // Populate the other fields with data from the root node. Selectors are not supposed
   // to be able to fail.