[ML] Explain log rate spikes: Move API stream demos to Kibana example…

…s. (#132590)

This creates a response_stream plugin in the Kibana /examples section. The plugin demonstrates API endpoints that can stream data chunks with a single request with gzip/compression support. gzip-streams get decompressed natively by browsers. The plugin demonstrates two use cases to get started: Streaming a raw string as well as a more complex example that streams Redux-like actions to the client which update React state via useReducer().

.github/CODEOWNERS

@@ -187,7 +187,7 @@
 /x-pack/test/screenshot_creation/apps/ml_docs @elastic/ml-ui
 /x-pack/test/screenshot_creation/services/ml_screenshots.ts @elastic/ml-ui
 
-# Additional plugins maintained by the ML team.
+# Additional plugins and packages maintained by the ML team.
 /x-pack/plugins/aiops/ @elastic/ml-ui
 /x-pack/plugins/data_visualizer/ @elastic/ml-ui
 /x-pack/plugins/file_upload/ @elastic/ml-ui
@@ -198,6 +198,8 @@
 /x-pack/test/functional/apps/transform/ @elastic/ml-ui
 /x-pack/test/functional/services/transform/ @elastic/ml-ui
 /x-pack/test/functional_basic/apps/transform/ @elastic/ml-ui
+/packages/kbn-aiops-utils @elastic/ml-ui
+/examples/response_stream/ @elastic/ml-ui
 
 # Maps
 #CC# /x-pack/plugins/maps/ @elastic/kibana-gis

examples/response_stream/README.md

@@ -0,0 +1,28 @@
+## response stream
+
+This plugin demonstrates how to stream chunks of data to the client with just a single request.
+
+To run Kibana with the described examples, use `yarn start --run-examples`.
+
+The `response_stream` plugin demonstrates API endpoints that can stream data chunks with a single request with gzip/compression support. gzip-streams get decompressed natively by browsers. The plugin demonstrates two use cases to get started: Streaming a raw string as well as a more complex example that streams Redux-like actions to the client which update React state via `useReducer()`.
+
+Code in `@kbn/aiops-utils` contains helpers to set up a stream on the server side (`streamFactory()`) and consume it on the client side via a custom hook (`useFetchStream()`). The utilities make use of TS generics in a way that allows to have type safety for both request related options as well as the returned data.
+
+No additional third party libraries are used in the helpers to make it work. On the server, they integrate with `Hapi` and use node's own `gzip`. On the client, the custom hook abstracts away the necessary logic to consume the stream, internally it makes use of a generator function and `useReducer()` to update React state.
+
+On the server, the simpler stream to send a string is set up like this:
+
+```ts
+const { end, push, responseWithHeaders } = streamFactory(request.headers);
+```
+
+The request's headers get passed on to automatically identify if compression is supported by the client.
+
+On the client, the custom hook is used like this:
+
+```ts
+const { error, start, cancel, data, isRunning } = useFetchStream<
+    ApiSimpleStringStream, typeof basePath
+>(`${basePath}/internal/response_stream/simple_string_stream`);
+```
+

examples/response_stream/common/README.md

@@ -0,0 +1 @@
+The `./api` folder contains shared code used to support working with the same type specifications on server and client.

examples/response_stream/common/api/index.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type {
+  UseFetchStreamCustomReducerParams,
+  UseFetchStreamParamsDefault,
+} from '@kbn/aiops-utils';
+
+import {
+  reducerStreamReducer,
+  ReducerStreamRequestBodySchema,
+  ReducerStreamApiAction,
+} from './reducer_stream';
+import { SimpleStringStreamRequestBodySchema } from './simple_string_stream';
+
+export const API_ENDPOINT = {
+  REDUCER_STREAM: '/internal/response_stream/reducer_stream',
+  SIMPLE_STRING_STREAM: '/internal/response_stream/simple_string_stream',
+} as const;
+
+export interface ApiReducerStream extends UseFetchStreamCustomReducerParams {
+  endpoint: typeof API_ENDPOINT.REDUCER_STREAM;
+  reducer: typeof reducerStreamReducer;
+  body: ReducerStreamRequestBodySchema;
+  actions: ReducerStreamApiAction;
+}
+
+export interface ApiSimpleStringStream extends UseFetchStreamParamsDefault {
+  endpoint: typeof API_ENDPOINT.SIMPLE_STRING_STREAM;
+  body: SimpleStringStreamRequestBodySchema;
+}

x-pack/plugins/aiops/common/api/example_stream.ts → examples/response_stream/common/api/reducer_stream/index.ts

@@ -1,20 +1,14 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-import { schema, TypeOf } from '@kbn/config-schema';
-
-export const aiopsExampleStreamSchema = schema.object({
-  /** Boolean flag to enable/disabling simulation of response errors. */
-  simulateErrors: schema.maybe(schema.boolean()),
-  /** Maximum timeout between streaming messages. */
-  timeout: schema.maybe(schema.number()),
-});
-
-export type AiopsExampleStreamSchema = TypeOf<typeof aiopsExampleStreamSchema>;
+export { reducerStreamReducer } from './reducer';
+export { reducerStreamRequestBodySchema } from './request_body_schema';
+export type { ReducerStreamRequestBodySchema } from './request_body_schema';
 
 export const API_ACTION_NAME = {
   UPDATE_PROGRESS: 'update_progress',
@@ -65,7 +59,7 @@ export function deleteEntityAction(payload: string): ApiActionDeleteEntity {
   };
 }
 
-export type AiopsExampleStreamApiAction =
+export type ReducerStreamApiAction =
   | ApiActionUpdateProgress
   | ApiActionAddToEntity
   | ApiActionDeleteEntity;

x-pack/plugins/aiops/public/components/single_endpoint_streaming_demo/stream_reducer.ts → examples/response_stream/common/api/reducer_stream/reducer.ts

@@ -1,33 +1,27 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-import { AiopsExampleStreamApiAction, API_ACTION_NAME } from '../../../common/api/example_stream';
+import { ReducerStreamApiAction, API_ACTION_NAME } from '.';
 
 export const UI_ACTION_NAME = {
-  ERROR: 'error',
   RESET: 'reset',
 } as const;
 export type UiActionName = typeof UI_ACTION_NAME[keyof typeof UI_ACTION_NAME];
 
 export interface StreamState {
-  errors: string[];
   progress: number;
   entities: Record<string, number>;
 }
 export const initialState: StreamState = {
-  errors: [],
   progress: 0,
   entities: {},
 };
 
-interface UiActionError {
-  type: typeof UI_ACTION_NAME.ERROR;
-  payload: string;
-}
 interface UiActionResetStream {
   type: typeof UI_ACTION_NAME.RESET;
 }
@@ -36,14 +30,14 @@ export function resetStream(): UiActionResetStream {
   return { type: UI_ACTION_NAME.RESET };
 }
 
-type UiAction = UiActionResetStream | UiActionError;
-export type ReducerAction = AiopsExampleStreamApiAction | UiAction;
-export function streamReducer(
+type UiAction = UiActionResetStream;
+export type ReducerAction = ReducerStreamApiAction | UiAction;
+export function reducerStreamReducer(
   state: StreamState,
   action: ReducerAction | ReducerAction[]
 ): StreamState {
   if (Array.isArray(action)) {
-    return action.reduce(streamReducer, state);
+    return action.reduce(reducerStreamReducer, state);
   }
 
   switch (action.type) {
@@ -72,15 +66,7 @@ export function streamReducer(
       };
     case UI_ACTION_NAME.RESET:
       return initialState;
-    case UI_ACTION_NAME.ERROR:
-      return {
-        ...state,
-        errors: [...state.errors, action.payload],
-      };
     default:
-      return {
-        ...state,
-        errors: [...state.errors, 'UNKNOWN_ACTION_ERROR'],
-      };
+      return state;
   }
 }

examples/response_stream/common/api/reducer_stream/request_body_schema.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+export const reducerStreamRequestBodySchema = schema.object({
+  /** Boolean flag to enable/disabling simulation of response errors. */
+  simulateErrors: schema.maybe(schema.boolean()),
+  /** Maximum timeout between streaming messages. */
+  timeout: schema.maybe(schema.number()),
+});
+export type ReducerStreamRequestBodySchema = TypeOf<typeof reducerStreamRequestBodySchema>;

examples/response_stream/common/api/simple_string_stream/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { simpleStringStreamRequestBodySchema } from './request_body_schema';
+export type { SimpleStringStreamRequestBodySchema } from './request_body_schema';

examples/response_stream/common/api/simple_string_stream/request_body_schema.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+export const simpleStringStreamRequestBodySchema = schema.object({
+  /** Maximum timeout between streaming messages. */
+  timeout: schema.number(),
+});
+export type SimpleStringStreamRequestBodySchema = TypeOf<
+  typeof simpleStringStreamRequestBodySchema
+>;

examples/response_stream/kibana.json

@@ -0,0 +1,14 @@
+{
+  "id": "responseStream",
+  "kibanaVersion": "kibana",
+  "version": "0.0.1",
+  "server": true,
+  "ui": true,
+  "owner": {
+    "name": "ML UI",
+    "githubTeam": "ml-ui"
+  },
+  "requiredPlugins": ["developerExamples"],
+  "optionalPlugins": [],
+  "requiredBundles": ["kibanaReact"]
+}

examples/response_stream/public/components/page.tsx

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as React from 'react';
+
+import {
+  EuiPageBody,
+  EuiPageContent,
+  EuiPageContentBody,
+  EuiPageHeader,
+  EuiPageHeaderSection,
+  EuiTitle,
+} from '@elastic/eui';
+
+export interface PageProps {
+  title?: React.ReactNode;
+}
+
+export const Page: React.FC<PageProps> = ({ title = 'Untitled', children }) => {
+  return (
+    <EuiPageBody>
+      <EuiPageHeader>
+        <EuiPageHeaderSection>
+          <EuiTitle size="l">
+            <h1>{title}</h1>
+          </EuiTitle>
+        </EuiPageHeaderSection>
+      </EuiPageHeader>
+      <EuiPageContent>
+        <EuiPageContentBody style={{ maxWidth: 800, margin: '0 auto' }}>
+          {children}
+        </EuiPageContentBody>
+      </EuiPageContent>
+    </EuiPageBody>
+  );
+};

examples/response_stream/public/containers/app/index.tsx

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import React from 'react';
+import { BrowserRouter as Router, Route, Redirect, Switch } from 'react-router-dom';
+import { EuiPage } from '@elastic/eui';
+import { useDeps } from '../../hooks/use_deps';
+import { Sidebar } from './sidebar';
+import { routes } from '../../routes';
+
+export const App: React.FC = () => {
+  const { appBasePath } = useDeps();
+
+  const routeElements: React.ReactElement[] = [];
+  for (const { items } of routes) {
+    for (const { id, component } of items) {
+      routeElements.push(<Route key={id} path={`/${id}`} render={(props) => component} />);
+    }
+  }
+
+  return (
+    <Router basename={appBasePath}>
+      <EuiPage>
+        <Sidebar />
+        <Switch>
+          {routeElements}
+          <Redirect to="/simple-string-stream" />
+        </Switch>
+      </EuiPage>
+    </Router>
+  );
+};

x-pack/plugins/aiops/public/components/single_endpoint_streaming_demo/get_status_message.tsx → examples/response_stream/public/containers/app/pages/page_reducer_stream/get_status_message.tsx

@@ -1,8 +1,9 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
 export function getStatusMessage(isRunning: boolean, isCancelled: boolean, progress: number) {
@@ -13,7 +14,7 @@ export function getStatusMessage(isRunning: boolean, isCancelled: boolean, progr
   } else if (!isRunning && isCancelled) {
     return 'Oh no, development got cancelled!';
   } else if (!isRunning && progress === 100) {
-    return 'Development clompeted, the release got out the door!';
+    return 'Development completed, the release got out the door!';
   }
 
   // When the process stops but wasn't cancelled by the user and progress is not yet at 100%,