New asset manager plugin (tech preview, off by default) (#152456)

## Summary

This plugin will contain the asset inventory and topology API in Kibana,
giving Kibana projects access to inventory and topology data via an HTTP
and/or JS API on the server and client.

[Currently proposed API
docs](https://github.com/elastic/o11y-topology-playground/tree/main/docs/api)
will be moved to this repo as well, contained inside this plugin folder,
as a part of this PR.

## Enabling the plugin

This plugin is entirely in "technical preview" and because of this, must
be specifically enabled via config for it to do anything besides being
run by the core plugin framework. To enable the server API layer, as
well as the index template management, put the following line in your
kibana.yml file:

```yml
xpack.assetManager.alphaEnabled: true
```

## Running the API integration tests

Run the functional test server with the asset manager config in place:

```shell
$ node scripts/functional_tests_server --config x-pack/test/api_integration/apis/asset_manager/config.ts
```

Then run the functional test runner with the same config, to target just
these tests:

```shell
$ node scripts/functional_test_runner --config=x-pack/test/api_integration/apis/asset_manager/config.
ts
```

_Note:_ The config file added in this folder enables the tech preview
plugin ([see file
here](https://github.com/elastic/kibana/pull/152456/files#diff-bc00de6c34c9bc131cfbdf3570c487fe9ee947e9a88a84c59d6b139b79d7708eR20)).

### Running the integration tests for verifying that the plugin is
"disabled" by default

There is a small set of tests that confirm that the endpoints return 404
and there is no index template installed if the config value is not set
in the kibana.yml file. To run this suite, use the following config:

```shell
$ node scripts/functional_tests_server --config x-pack/test/api_integration/apis/asset_manager/config_when_disabled.ts
$ node scripts/functional_test_runner --config=x-pack/test/api_integration/apis/asset_manager/config_when_disabled.
ts
```

## Testing this PR with sample data

There are some sample data mechanisms in place inside this PR to allow
us to build out the endpoints.

### View sample docs
```http
GET /api/asset-manager/assets/sample
```

This will return a list of the assets that are included if you elect to
write assets. This is a good endpoint to use to find EAN (Elastic Asset
Name) values that you may want to exclude from writing for a given time
period, to simulate assets appearing/disappearing over time.

### Write sample docs
```http
POST /api/asset-manager/assets/sample
{
  "baseDateTime": "2023-02-28T12:00:00.000Z",
  "excludeEans": ["k8s.cluster:cluster-002"]
}
```

This posts all of the sample asset documents to Elasticsearch using the
`baseDateTime` value as the timestamp. Any valid string or number that
is accepted by `new Date()` should work for `baseDateTime`.

The `excludeEans` value is an array of EAN ("Elastic Asset Name") values
that you don't want to write on this particular run. This way you can
have assets appear (exclude them in the past, don't exclude them during
a later run) or disappear (vice versa) and see how that shows up in
other endpoints.

**Note:** *Remember that when you curl a Kibana server API with a POST
request, you must include a `kbn-xsrf` header with any string value you
want.*

### Get asset docs from ES
```http
GET /api/asset-manager/assets?type=k8s.cluster&from=now-10m
```

This is the primary "real" endpoint available right now. It should
retrieve a list of assets based on the type/from/to/ean filter values
you specify. Once you load the sample data, this endpoint should return
results.

## Debug logging

There are some extra debug logs for ES queries that are running in the
code in this PR. To print those logs to the Kibana server console, run
Kibana using `DEBUG_LOGGER=true`

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/ftr_configs.yml

@@ -71,6 +71,9 @@ disabled:
   # Scalability testing config that we run in its own pipeline
   - x-pack/test/scalability/config.ts
 
+  # Asset Manager configs, in tech preview, will move to enabled after more stability introduced
+  - x-pack/test/api_integration/apis/asset_manager/config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - test/accessibility/config.ts
@@ -149,6 +152,7 @@ enabled:
   - x-pack/test/api_integration/config_security_basic.ts
   - x-pack/test/api_integration/config_security_trial.ts
   - x-pack/test/api_integration/apis/aiops/config.ts
+  - x-pack/test/api_integration/apis/asset_manager/config_when_disabled.ts
   - x-pack/test/api_integration/apis/cases/config.ts
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts
   - x-pack/test/api_integration/apis/console/config.ts

.github/CODEOWNERS

@@ -42,6 +42,7 @@ packages/kbn-apm-synthtrace-client @elastic/apm-ui
 packages/kbn-apm-utils @elastic/apm-ui
 test/plugin_functional/plugins/app_link_test @elastic/kibana-core
 x-pack/test/usage_collection/plugins/application_usage_test @elastic/kibana-core
+x-pack/plugins/asset_manager @jasonrhodes
 x-pack/test/security_api_integration/plugins/audit_log @elastic/kibana-security
 packages/kbn-axe-config @elastic/kibana-qa
 packages/kbn-babel-preset @elastic/kibana-operations

docs/developer/plugin-list.asciidoc

@@ -434,6 +434,11 @@ The plugin exposes the static DefaultEditorController class to consume.
 |undefined
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/asset_manager/README.md[assetManager]
+|This plugin provides access to the asset data stored in assets-* indices, primarily
+for inventory and topology purposes.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/banners/README.md[banners]
 |Allow to add a header banner that will be displayed on every page of the Kibana application
 

package.json

@@ -151,6 +151,7 @@
     "@kbn/apm-utils": "link:packages/kbn-apm-utils",
     "@kbn/app-link-test-plugin": "link:test/plugin_functional/plugins/app_link_test",
     "@kbn/application-usage-test-plugin": "link:x-pack/test/usage_collection/plugins/application_usage_test",
+    "@kbn/assetManager-plugin": "link:x-pack/plugins/asset_manager",
     "@kbn/audit-log-plugin": "link:x-pack/test/security_api_integration/plugins/audit_log",
     "@kbn/banners-plugin": "link:x-pack/plugins/banners",
     "@kbn/bfetch-explorer-plugin": "link:examples/bfetch_explorer",

tsconfig.base.json

@@ -78,6 +78,8 @@
       "@kbn/app-link-test-plugin/*": ["test/plugin_functional/plugins/app_link_test/*"],
       "@kbn/application-usage-test-plugin": ["x-pack/test/usage_collection/plugins/application_usage_test"],
       "@kbn/application-usage-test-plugin/*": ["x-pack/test/usage_collection/plugins/application_usage_test/*"],
+      "@kbn/assetManager-plugin": ["x-pack/plugins/asset_manager"],
+      "@kbn/assetManager-plugin/*": ["x-pack/plugins/asset_manager/*"],
       "@kbn/audit-log-plugin": ["x-pack/test/security_api_integration/plugins/audit_log"],
       "@kbn/audit-log-plugin/*": ["x-pack/test/security_api_integration/plugins/audit_log/*"],
       "@kbn/axe-config": ["packages/kbn-axe-config"],

x-pack/plugins/asset_manager/README.md

@@ -0,0 +1,39 @@
+# Asset Manager Plugin
+
+This plugin provides access to the asset data stored in assets-\* indices, primarily
+for inventory and topology purposes.
+
+## Documentation
+
+See [docs for the provided APIs in the docs folder](./docs/index.md).
+
+## Running Tests
+
+There are integration tests for the endpoints implemented thus far as well as for
+the sample data tests. There is also a small set of tests meant to ensure that the
+plugin is not doing anything without the proper config value in place to enable
+the plugin fully. For more on enabling the plugin, see [the docs page](./docs/index.md).
+
+The "not enabled" tests are run by default in CI. To run them manually, do the following:
+
+```shell
+$ node scripts/functional_tests_server --config x-pack/test/api_integration/apis/asset_manager/config_when_disabled.ts
+$ node scripts/functional_test_runner --config=x-pack/test/api_integration/apis/asset_manager/config_when_disabled.ts
+```
+
+The "enabled" tests are NOT run by CI yet, to prevent blocking Kibana development for a
+test failure in this alpha, tech preview plugin. They will be moved into the right place
+to make them run for CI before the plugin is enabled by default. To run them manually:
+
+```shell
+$ node scripts/functional_tests_server --config x-pack/test/api_integration/apis/asset_manager/config.ts
+$ node scripts/functional_test_runner --config=x-pack/test/api_integration/apis/asset_manager/config.ts
+```
+
+## Using Sample Data
+
+This plugin comes with a full "working set" of sample asset documents, meant
+to provide enough data in the correct schema format so that all of the API
+endpoints return expected values.
+
+To create the sample data, follow [the instructions in the REST API docs](./docs/index.md#sample-data).

x-pack/plugins/asset_manager/common/debug_log.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export function debug(...args: any[]) {
+  if (process.env.NODE_ENV === 'production' || !process.env.DEBUG_LOGGER) {
+    return;
+  }
+  // eslint-disable-next-line no-console
+  console.log('[DEBUG LOG]', ...args);
+}

x-pack/plugins/asset_manager/common/types_api.ts

@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export type AssetKind = 'unknown' | 'node';
+export type AssetType = 'k8s.pod' | 'k8s.cluster' | 'k8s.node';
+export type AssetStatus =
+  | 'CREATING'
+  | 'ACTIVE'
+  | 'DELETING'
+  | 'FAILED'
+  | 'UPDATING'
+  | 'PENDING'
+  | 'UNKNOWN';
+export type CloudProviderName = 'aws' | 'gcp' | 'azure' | 'other' | 'unknown' | 'none';
+
+interface WithTimestamp {
+  '@timestamp': string;
+}
+export interface ECSDocument extends WithTimestamp {
+  'kubernetes.namespace'?: string;
+  'kubernetes.pod.name'?: string;
+  'kubernetes.pod.uid'?: string;
+  'kubernetes.pod.start_time'?: Date;
+  'kubernetes.node.name'?: string;
+  'kubernetes.node.start_time'?: Date;
+
+  'orchestrator.api_version'?: string;
+  'orchestrator.namespace'?: string;
+  'orchestrator.organization'?: string;
+  'orchestrator.type'?: string;
+  'orchestrator.cluster.id'?: string;
+  'orchestrator.cluster.name'?: string;
+  'orchestrator.cluster.url'?: string;
+  'orchestrator.cluster.version'?: string;
+
+  'cloud.provider'?: CloudProviderName;
+  'cloud.region'?: string;
+  'cloud.service.name'?: string;
+}
+
+export interface Asset extends ECSDocument {
+  'asset.collection_version'?: string;
+  'asset.ean': string;
+  'asset.id': string;
+  'asset.kind'?: AssetKind;
+  'asset.name'?: string;
+  'asset.type': AssetType;
+  'asset.status'?: AssetStatus;
+  'asset.parents'?: string | string[];
+  'asset.children'?: string | string[];
+  'asset.namespace'?: string;
+}
+
+export type AssetWithoutTimestamp = Omit<Asset, '@timestamp'>;
+
+export interface K8sPod extends WithTimestamp {
+  id: string;
+  name?: string;
+  ean: string;
+  node?: string;
+  cloud?: {
+    provider?: CloudProviderName;
+    region?: string;
+  };
+}
+
+export interface K8sNodeMetricBucket {
+  timestamp: number;
+  date?: string;
+  averageMemoryAvailable: number | null;
+  averageMemoryUsage: number | null;
+  maxMemoryUsage: number | null;
+  averageCpuCoreNs: number | null;
+  maxCpuCoreNs: number | null;
+}
+
+export interface K8sNodeLog {
+  timestamp: number;
+  message: string;
+}
+
+export interface K8sNode extends WithTimestamp {
+  id: string;
+  name?: string;
+  ean: string;
+  pods?: K8sPod[];
+  cluster?: K8sCluster;
+  cloud?: {
+    provider?: CloudProviderName;
+    region?: string;
+  };
+  metrics?: K8sNodeMetricBucket[];
+  logs?: K8sNodeLog[];
+}
+
+export interface K8sCluster extends WithTimestamp {
+  name?: string;
+  nodes?: K8sNode[];
+  ean: string;
+  status?: AssetStatus;
+  version?: string;
+  cloud?: {
+    provider?: CloudProviderName;
+    region?: string;
+  };
+}
+
+export interface AssetFilters {
+  type?: AssetType | AssetType[];
+  kind?: AssetKind;
+  ean?: string;
+  id?: string;
+  typeLike?: string;
+  eanLike?: string;
+  collectionVersion?: number | 'latest' | 'all';
+  from?: string;
+  to?: string;
+}