Service Map Data API at Runtime (elastic#54027)

* [APM] Runtime service maps

* Make nodes interactive

* Don't use smaller range query on initial request

* Address feedback from Ron

* Get all services separately

* Get single service as well

* Query both transactions/spans for initial request

* Optimize 'top' query for service maps

* Use agent.name from scripted metric

* adds basic loading overlay

* filter out service map node self reference edges from being rendered

* Make service map initial load time range configurable with
`xpack.apm.serviceMapInitialTimeRange` default to last 1 hour in
milliseconds

* ensure destination.address is not missing in the composite agg when
fetching sample trace ids

* wip: added incremental data fetch & progress bar

* implement progressive loading design while blocking service map interaction during loading

* adds filter that destination.address exists before fetching sample trace ids

* reduce pairs of connections to 1 bi-directional connection with arrows on both ends of the edge

* Optimize query; add update button

* Allow user interaction after 5s, auto update in that time, otherwise
show toast for user to update the map with button

* Correctly reduce nodes/connections

* - remove non-interactive state while loading
- use cytoscape element definition types

* - readability improvements to the ServiceMap component
- only show the update map button toast after last request loads

* addresses feedback for changes to the Cytoscape component

* Add span.type/span.subtype do external nodes

* PR feedback

Co-authored-by: Dario Gieselaar <d.gieselaar@gmail.com>

x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts

@@ -14,6 +14,8 @@ export const HTTP_REQUEST_METHOD = 'http.request.method';
 export const USER_ID = 'user.id';
 export const USER_AGENT_NAME = 'user_agent.name';
 
+export const DESTINATION_ADDRESS = 'destination.address';
+
 export const OBSERVER_VERSION_MAJOR = 'observer.version_major';
 export const OBSERVER_LISTENING = 'observer.listening';
 export const PROCESSOR_EVENT = 'processor.event';

x-pack/legacy/plugins/apm/common/service_map.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface ServiceConnectionNode {
+  'service.name': string;
+  'service.environment': string | null;
+  'agent.name': string;
+}
+export interface ExternalConnectionNode {
+  'destination.address': string;
+  'span.type': string;
+  'span.subtype': string;
+}
+
+export type ConnectionNode = ServiceConnectionNode | ExternalConnectionNode;
+
+export interface Connection {
+  source: ConnectionNode;
+  destination: ConnectionNode;
+}

x-pack/legacy/plugins/apm/index.ts

@@ -71,7 +71,8 @@ export const apm: LegacyPluginInitializer = kibana => {
         autocreateApmIndexPattern: Joi.boolean().default(true),
 
         // service map
-        serviceMapEnabled: Joi.boolean().default(false)
+        serviceMapEnabled: Joi.boolean().default(false),
+        serviceMapInitialTimeRange: Joi.number().default(60 * 1000 * 60) // last 1 hour
       }).default();
     },
 

x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx

@@ -73,6 +73,7 @@ export function Cytoscape({
       cy.on('data', event => {
         // Add the "primary" class to the node if its id matches the serviceName.
         if (cy.nodes().length > 0 && serviceName) {
+          cy.nodes().removeClass('primary');
           cy.getElementById(serviceName).addClass('primary');
         }
 

x-pack/legacy/plugins/apm/public/components/app/ServiceMap/LoadingOverlay.tsx

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import theme from '@elastic/eui/dist/eui_theme_light.json';
+import React from 'react';
+import { EuiProgress, EuiText, EuiSpacer } from '@elastic/eui';
+import styled from 'styled-components';
+import { i18n } from '@kbn/i18n';
+
+const Container = styled.div`
+  position: relative;
+`;
+
+const Overlay = styled.div`
+  position: absolute;
+  top: 0;
+  z-index: 1;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  width: 100%;
+  padding: ${theme.gutterTypes.gutterMedium};
+`;
+
+const ProgressBarContainer = styled.div`
+  width: 50%;
+  max-width: 600px;
+`;
+
+interface Props {
+  children: React.ReactNode;
+  isLoading: boolean;
+  percentageLoaded: number;
+}
+
+export const LoadingOverlay = ({
+  children,
+  isLoading,
+  percentageLoaded
+}: Props) => (
+  <Container>
+    {isLoading && (
+      <Overlay>
+        <ProgressBarContainer>
+          <EuiProgress
+            value={percentageLoaded}
+            max={100}
+            color="primary"
+            size="m"
+          />
+        </ProgressBarContainer>
+        <EuiSpacer size="s" />
+        <EuiText size="s" textAlign="center">
+          {i18n.translate('xpack.apm.loadingServiceMap', {
+            defaultMessage:
+              'Loading service map... This might take a short while.'
+          })}
+        </EuiText>
+      </Overlay>
+    )}
+    {children}
+  </Container>
+);

x-pack/legacy/plugins/apm/public/components/app/ServiceMap/cytoscapeOptions.ts

@@ -8,17 +8,13 @@ import theme from '@elastic/eui/dist/eui_theme_light.json';
 import { icons, defaultIcon } from './icons';
 
 const layout = {
-  animate: true,
-  animationEasing: theme.euiAnimSlightBounce as cytoscape.Css.TransitionTimingFunction,
-  animationDuration: parseInt(theme.euiAnimSpeedFast, 10),
   name: 'dagre',
   nodeDimensionsIncludeLabels: true,
-  rankDir: 'LR',
-  spacingFactor: 2
+  rankDir: 'LR'
 };
 
 function isDatabaseOrExternal(agentName: string) {
-  return agentName === 'database' || agentName === 'external';
+  return !agentName;
 }
 
 const style: cytoscape.Stylesheet[] = [
@@ -47,7 +43,7 @@ const style: cytoscape.Stylesheet[] = [
       'font-family': 'Inter UI, Segoe UI, Helvetica, Arial, sans-serif',
       'font-size': theme.euiFontSizeXS,
       height: theme.avatarSizing.l.size,
-      label: 'data(id)',
+      label: 'data(label)',
       'min-zoomed-font-size': theme.euiSizeL,
       'overlay-opacity': 0,
       shape: (el: cytoscape.NodeSingular) =>
@@ -76,7 +72,18 @@ const style: cytoscape.Stylesheet[] = [
       //
       // @ts-ignore
       'target-distance-from-node': theme.paddingSizes.xs,
-      width: 2
+      width: 1,
+      'source-arrow-shape': 'none'
+    }
+  },
+  {
+    selector: 'edge[bidirectional]',
+    style: {
+      'source-arrow-shape': 'triangle',
+      'target-arrow-shape': 'triangle',
+      // @ts-ignore
+      'source-distance-from-node': theme.paddingSizes.xs,
+      'target-distance-from-node': theme.paddingSizes.xs
     }
   }
 ];

x-pack/legacy/plugins/apm/public/components/app/ServiceMap/get_cytoscape_elements.ts

@@ -0,0 +1,158 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { ValuesType } from 'utility-types';
+import { sortBy, isEqual } from 'lodash';
+import { Connection, ConnectionNode } from '../../../../common/service_map';
+import { ServiceMapAPIResponse } from '../../../../server/lib/service_map/get_service_map';
+import { getAPMHref } from '../../shared/Links/apm/APMLink';
+
+function getConnectionNodeId(node: ConnectionNode): string {
+  if ('destination.address' in node) {
+    // use a prefix to distinguish exernal destination ids from services
+    return `>${node['destination.address']}`;
+  }
+  return node['service.name'];
+}
+
+function getConnectionId(connection: Connection) {
+  return `${getConnectionNodeId(connection.source)}~${getConnectionNodeId(
+    connection.destination
+  )}`;
+}
+export function getCytoscapeElements(
+  responses: ServiceMapAPIResponse[],
+  search: string
+) {
+  const discoveredServices = responses.flatMap(
+    response => response.discoveredServices
+  );
+
+  const serviceNodes = responses
+    .flatMap(response => response.services)
+    .map(service => ({
+      ...service,
+      id: service['service.name']
+    }));
+
+  // maps destination.address to service.name if possible
+  function getConnectionNode(node: ConnectionNode) {
+    let mappedNode: ConnectionNode | undefined;
+
+    if ('destination.address' in node) {
+      mappedNode = discoveredServices.find(map => isEqual(map.from, node))?.to;
+    }
+
+    if (!mappedNode) {
+      mappedNode = node;
+    }
+
+    return {
+      ...mappedNode,
+      id: getConnectionNodeId(mappedNode)
+    };
+  }
+
+  // build connections with mapped nodes
+  const connections = responses
+    .flatMap(response => response.connections)
+    .map(connection => {
+      const source = getConnectionNode(connection.source);
+      const destination = getConnectionNode(connection.destination);
+
+      return {
+        source,
+        destination,
+        id: getConnectionId({ source, destination })
+      };
+    })
+    .filter(connection => connection.source.id !== connection.destination.id);
+
+  const nodes = connections
+    .flatMap(connection => [connection.source, connection.destination])
+    .concat(serviceNodes);
+
+  type ConnectionWithId = ValuesType<typeof connections>;
+  type ConnectionNodeWithId = ValuesType<typeof nodes>;
+
+  const connectionsById = connections.reduce((connectionMap, connection) => {
+    return {
+      ...connectionMap,
+      [connection.id]: connection
+    };
+  }, {} as Record<string, ConnectionWithId>);
+
+  const nodesById = nodes.reduce((nodeMap, node) => {
+    return {
+      ...nodeMap,
+      [node.id]: node
+    };
+  }, {} as Record<string, ConnectionNodeWithId>);
+
+  const cyNodes = (Object.values(nodesById) as ConnectionNodeWithId[]).map(
+    node => {
+      let data = {};
+
+      if ('service.name' in node) {
+        data = {
+          href: getAPMHref(
+            `/services/${node['service.name']}/service-map`,
+            search
+          ),
+          agentName: node['agent.name'] || node['agent.name']
+        };
+      }
+
+      return {
+        group: 'nodes' as const,
+        data: {
+          id: node.id,
+          label:
+            'service.name' in node
+              ? node['service.name']
+              : node['destination.address'],
+          ...data
+        }
+      };
+    }
+  );
+
+  // instead of adding connections in two directions,
+  // we add a `bidirectional` flag to use in styling
+  const dedupedConnections = (sortBy(
+    Object.values(connectionsById),
+    // make sure that order is stable
+    'id'
+  ) as ConnectionWithId[]).reduce<
+    Array<ConnectionWithId & { bidirectional?: boolean }>
+  >((prev, connection) => {
+    const reversedConnection = prev.find(
+      c =>
+        c.destination.id === connection.source.id &&
+        c.source.id === connection.destination.id
+    );
+
+    if (reversedConnection) {
+      reversedConnection.bidirectional = true;
+      return prev;
+    }
+
+    return prev.concat(connection);
+  }, []);
+
+  const cyEdges = dedupedConnections.map(connection => {
+    return {
+      group: 'edges' as const,
+      data: {
+        id: connection.id,
+        source: connection.source.id,
+        target: connection.destination.id,
+        bidirectional: connection.bidirectional ? true : undefined
+      }
+    };
+  }, []);
+
+  return [...cyNodes, ...cyEdges];
+}