Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate server-side ES domain to packages #136297

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
6550e9f
create es types package
pgayvallet Jul 13, 2022
9d272fa
start moving client types to @kbn/core-elasticsearch-server
pgayvallet Jul 13, 2022
5a7f812
move ElasticsearchClientConfig to package
pgayvallet Jul 13, 2022
7d3620b
start adapting usages
pgayvallet Jul 13, 2022
21610bf
start fixing imports
pgayvallet Jul 13, 2022
0f4953b
fix more imports
pgayvallet Jul 13, 2022
a07f24d
just a bit more
pgayvallet Jul 13, 2022
add4b0e
move service types
pgayvallet Jul 14, 2022
fa83d21
[CI] Auto-commit changed files from 'node scripts/precommit_hook.js -…
kibanamachine Jul 14, 2022
b03557b
fix more internal types
pgayvallet Jul 14, 2022
64c3bea
Merge remote-tracking branch 'upstream/main' into kbn-136283-server-s…
pgayvallet Jul 14, 2022
d108f7a
move retry call cluster helpers outside of client package
pgayvallet Jul 14, 2022
7892891
move client code and mocks to packages
pgayvallet Jul 14, 2022
76a8c84
fix imports
pgayvallet Jul 14, 2022
73d9556
adapt external usages
pgayvallet Jul 14, 2022
675f1eb
adapt more external usages
pgayvallet Jul 14, 2022
d9a632f
Merge remote-tracking branch 'upstream/main' into kbn-136283-server-s…
pgayvallet Jul 14, 2022
2a3d8f2
adapt more external usages 2
pgayvallet Jul 14, 2022
edc8434
fix mocked module
pgayvallet Jul 14, 2022
4b722cf
create empty domain packages
pgayvallet Jul 14, 2022
cf50cba
more external usages fix
pgayvallet Jul 14, 2022
22b5cbc
move ALL the things (again)
pgayvallet Jul 14, 2022
e640dfe
mock external import fix
pgayvallet Jul 14, 2022
a4ee98f
fix tests, add test dependencies
pgayvallet Jul 14, 2022
e7e3e8c
fix some internal usages
pgayvallet Jul 14, 2022
77aa97e
fix more internal usages
pgayvallet Jul 14, 2022
4133c84
[CI] Auto-commit changed files from 'node scripts/precommit_hook.js -…
kibanamachine Jul 14, 2022
70b2de0
fix cli_setup usage
pgayvallet Jul 14, 2022
01ba9b7
desperate times force desperate decisions
pgayvallet Jul 14, 2022
1f4dd07
fix misc stuff
pgayvallet Jul 14, 2022
366daa8
update snapshots (?!)
pgayvallet Jul 14, 2022
1988d60
fix mocked package
pgayvallet Jul 14, 2022
5e0eddb
self review
pgayvallet Jul 15, 2022
15e3b29
Merge remote-tracking branch 'upstream/main' into kbn-136283-server-s…
pgayvallet Jul 15, 2022
195b75b
Merge remote-tracking branch 'upstream/main' into kbn-136283-server-s…
pgayvallet Jul 19, 2022
491ecc1
Merge remote-tracking branch 'upstream/main' into kbn-136283-server-s…
pgayvallet Jul 20, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,12 @@
"@kbn/core-doc-links-server": "link:bazel-bin/packages/core/doc-links/core-doc-links-server",
"@kbn/core-doc-links-server-internal": "link:bazel-bin/packages/core/doc-links/core-doc-links-server-internal",
"@kbn/core-doc-links-server-mocks": "link:bazel-bin/packages/core/doc-links/core-doc-links-server-mocks",
"@kbn/core-elasticsearch-client-server": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-client-server",
"@kbn/core-elasticsearch-client-server-internal": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-client-server-internal",
"@kbn/core-elasticsearch-client-server-mocks": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-client-server-mocks",
"@kbn/core-elasticsearch-server": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-server",
"@kbn/core-elasticsearch-server-internal": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-server-internal",
"@kbn/core-elasticsearch-server-mocks": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-server-mocks",
"@kbn/core-environment-server-internal": "link:bazel-bin/packages/core/environment/core-environment-server-internal",
"@kbn/core-environment-server-mocks": "link:bazel-bin/packages/core/environment/core-environment-server-mocks",
"@kbn/core-execution-context-browser": "link:bazel-bin/packages/core/execution-context/core-execution-context-browser",
Expand Down Expand Up @@ -757,6 +763,12 @@
"@types/kbn__core-doc-links-server": "link:bazel-bin/packages/core/doc-links/core-doc-links-server/npm_module_types",
"@types/kbn__core-doc-links-server-internal": "link:bazel-bin/packages/core/doc-links/core-doc-links-server-internal/npm_module_types",
"@types/kbn__core-doc-links-server-mocks": "link:bazel-bin/packages/core/doc-links/core-doc-links-server-mocks/npm_module_types",
"@types/kbn__core-elasticsearch-client-server": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-client-server/npm_module_types",
"@types/kbn__core-elasticsearch-client-server-internal": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-client-server-internal/npm_module_types",
"@types/kbn__core-elasticsearch-client-server-mocks": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-client-server-mocks/npm_module_types",
"@types/kbn__core-elasticsearch-server": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-server/npm_module_types",
"@types/kbn__core-elasticsearch-server-internal": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-server-internal/npm_module_types",
"@types/kbn__core-elasticsearch-server-mocks": "link:bazel-bin/packages/core/elasticsearch/core-elasticsearch-server-mocks/npm_module_types",
"@types/kbn__core-environment-server-internal": "link:bazel-bin/packages/core/environment/core-environment-server-internal/npm_module_types",
"@types/kbn__core-environment-server-mocks": "link:bazel-bin/packages/core/environment/core-environment-server-mocks/npm_module_types",
"@types/kbn__core-execution-context-browser": "link:bazel-bin/packages/core/execution-context/core-execution-context-browser/npm_module_types",
Expand Down
10 changes: 10 additions & 0 deletions packages/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ filegroup(
"//packages/core/doc-links/core-doc-links-server-internal:build",
"//packages/core/doc-links/core-doc-links-server-mocks:build",
"//packages/core/doc-links/core-doc-links-server:build",
"//packages/core/elasticsearch/core-elasticsearch-client-server-internal:build",
"//packages/core/elasticsearch/core-elasticsearch-client-server-mocks:build",
"//packages/core/elasticsearch/core-elasticsearch-server-internal:build",
"//packages/core/elasticsearch/core-elasticsearch-server-mocks:build",
"//packages/core/elasticsearch/core-elasticsearch-server:build",
"//packages/core/environment/core-environment-server-internal:build",
"//packages/core/environment/core-environment-server-mocks:build",
"//packages/core/execution-context/core-execution-context-browser-internal:build",
Expand Down Expand Up @@ -248,6 +253,11 @@ filegroup(
"//packages/core/doc-links/core-doc-links-server-internal:build_types",
"//packages/core/doc-links/core-doc-links-server-mocks:build_types",
"//packages/core/doc-links/core-doc-links-server:build_types",
"//packages/core/elasticsearch/core-elasticsearch-client-server-internal:build_types",
"//packages/core/elasticsearch/core-elasticsearch-client-server-mocks:build_types",
"//packages/core/elasticsearch/core-elasticsearch-server-internal:build_types",
"//packages/core/elasticsearch/core-elasticsearch-server-mocks:build_types",
"//packages/core/elasticsearch/core-elasticsearch-server:build_types",
"//packages/core/environment/core-environment-server-internal:build_types",
"//packages/core/environment/core-environment-server-mocks:build_types",
"//packages/core/execution-context/core-execution-context-browser-internal:build_types",
Expand Down
2 changes: 2 additions & 0 deletions packages/core/base/core-base-common/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@ NPM_MODULE_EXTRA_FILES = [
]

RUNTIME_DEPS = [
"//packages/kbn-std",
]

TYPES_DEPS = [
"@npm//@types/node",
"@npm//@types/jest",
"//packages/kbn-std:npm_module_types",
"//packages/kbn-config:npm_module_types",
]

Expand Down
2 changes: 2 additions & 0 deletions packages/core/base/core-base-common/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@
export type { PluginOpaqueId, PluginName, DiscoveredPlugin } from './plugins';
export { PluginType } from './plugins';
export { EUI_STYLES_GLOBAL } from './eui';
export { ServiceStatusLevels } from './service_status';
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

core-base-common seems to be morphing into a "catch-all" I-don't-know-where-to-put-this-thing package. It's fine for now as the package is still very small. During the cleanup phase or slightly prior to that, we should audit everything in core-base-* packages and see if we could organize the components into a domain-like structure.

Copy link
Contributor Author

@pgayvallet pgayvallet Jul 19, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

core-base-common seems to be morphing into a "catch-all" I-don't-know-where-to-put-this-thing package

Yup, I totally agree. The intent here is to avoid the 'lets create 99 another one-file packages before the end of the migration' effect. We move all the 'misc' stuff here, and we'll revisit once everything has been moved.

During the cleanup phase or slightly prior to that, we should audit everything in core-base-* packages and see if we could organize the components into a domain-like structure

Fully agree too.

Copy link
Contributor

@TinaHeiligers TinaHeiligers Jul 19, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From a quick search through core, it's elasticsearch, status, and savedObjects that all use the types from status. Am I missing any other core domains in this list?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, AFAIK for now only the ES and SO services declare their statuses.

export type { ServiceStatus, ServiceStatusLevel } from './service_status';
101 changes: 101 additions & 0 deletions packages/core/base/core-base-common/src/service_status.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { deepFreeze } from '@kbn/std';

/**
* The current status of a service at a point in time.
*
* @typeParam Meta - JSON-serializable object. Plugins should export this type to allow other plugins to read the `meta`
* field in a type-safe way.
* @public
*/
export interface ServiceStatus<Meta extends Record<string, any> | unknown = unknown> {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The status calculation logic of the ES services relies on those types, however the status service also depends on the elasticsearch service.

To break the dependency I moved those things into @kbn/core-base-common. common instead of server because these types are also used as base for some types on the client-side.

/**
* The current availability level of the service.
*/
level: ServiceStatusLevel;
/**
* A high-level summary of the service status.
*/
summary: string;
/**
* A more detailed description of the service status.
*/
detail?: string;
/**
* A URL to open in a new tab about how to resolve or troubleshoot the problem.
*/
documentationUrl?: string;
/**
* Any JSON-serializable data to be included in the HTTP API response. Useful for providing more fine-grained,
* machine-readable information about the service status. May include status information for underlying features.
*/
meta?: Meta;
}

/**
* The current "level" of availability of a service.
*
* @remarks
* The values implement `valueOf` to allow for easy comparisons between status levels with <, >, etc. Higher values
* represent higher severities. Note that the default `Array.prototype.sort` implementation does not correctly sort
* these values.
*
* A snapshot serializer is available in `src/core/server/test_utils` to ease testing of these values with Jest.
*
* @public
*/
export const ServiceStatusLevels = deepFreeze({
/**
* Everything is working!
*/
available: {
toString: () => 'available',
valueOf: () => 0,
toJSON() {
return this.toString();
},
},
/**
* Some features may not be working.
*/
degraded: {
toString: () => 'degraded',
valueOf: () => 1,
toJSON() {
return this.toString();
},
},
/**
* The service is unavailable, but other functions that do not depend on this service should work.
*/
unavailable: {
toString: () => 'unavailable',
valueOf: () => 2,
toJSON() {
return this.toString();
},
},
/**
* Block all user functions and display the status page, reserved for Core services only.
*/
critical: {
toString: () => 'critical',
valueOf: () => 3,
toJSON() {
return this.toString();
},
},
});

/**
* A convenience type that represents the union of each value in {@link ServiceStatusLevels}.
* @public
*/
export type ServiceStatusLevel = typeof ServiceStatusLevels[keyof typeof ServiceStatusLevels];
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
load("@npm//@bazel/typescript:index.bzl", "ts_config")
load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")

PKG_DIRNAME = "core-elasticsearch-client-server-internal"
PKG_REQUIRE_NAME = "@kbn/core-elasticsearch-client-server-internal"

SOURCE_FILES = glob(
[
"src/**/*.ts",
],
exclude = [
"**/*.test.*",
"**/*.stories.*",
],
)

SRCS = SOURCE_FILES

filegroup(
name = "srcs",
srcs = SRCS,
)

NPM_MODULE_EXTRA_FILES = [
"package.json",
]

RUNTIME_DEPS = [
"@npm//moment",
"@npm//@elastic/elasticsearch",
"@npm//@elastic/numeral",
"//packages/kbn-std",
"//packages/kbn-es-errors",
"//packages/core/http/core-http-router-server-internal",
### test dependencies
"//packages/core/logging/core-logging-server-mocks",
"//packages/core/http/core-http-server-mocks",
]

TYPES_DEPS = [
"@npm//@types/node",
"@npm//@types/jest",
"@npm//moment",
"@npm//@elastic/elasticsearch",
"@npm//@elastic/numeral",
"//packages/kbn-utility-types:npm_module_types",
"//packages/kbn-std:npm_module_types",
"//packages/kbn-es-errors:npm_module_types",
"//packages/kbn-logging:npm_module_types",
"//packages/core/http/core-http-server:npm_module_types",
"//packages/core/http/core-http-router-server-internal:npm_module_types",
"//packages/core/elasticsearch/core-elasticsearch-server:npm_module_types",
]

jsts_transpiler(
name = "target_node",
srcs = SRCS,
build_pkg_name = package_name(),
)

ts_config(
name = "tsconfig",
src = "tsconfig.json",
deps = [
"//:tsconfig.base.json",
"//:tsconfig.bazel.json",
],
)

ts_project(
name = "tsc_types",
args = ['--pretty'],
srcs = SRCS,
deps = TYPES_DEPS,
declaration = True,
declaration_map = True,
emit_declaration_only = True,
out_dir = "target_types",
root_dir = "src",
tsconfig = ":tsconfig",
)

js_library(
name = PKG_DIRNAME,
srcs = NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS + [":target_node"],
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],
)

pkg_npm(
name = "npm_module",
deps = [":" + PKG_DIRNAME],
)

filegroup(
name = "build",
srcs = [":npm_module"],
visibility = ["//visibility:public"],
)

pkg_npm_types(
name = "npm_module_types",
srcs = SRCS,
deps = [":tsc_types"],
package_name = PKG_REQUIRE_NAME,
tsconfig = ":tsconfig",
visibility = ["//visibility:public"],
)

filegroup(
name = "build_types",
srcs = [":npm_module_types"],
visibility = ["//visibility:public"],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# @kbn/core-elasticsearch-client-server-internal

This package contains the internal implementation for Core's server-side elasticsearch client.
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

module.exports = {
preset: '@kbn/test/jest_node',
rootDir: '../../../..',
roots: ['<rootDir>/packages/core/elasticsearch/core-elasticsearch-client-server-internal'],
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "@kbn/core-elasticsearch-client-server-internal",
"private": true,
"version": "1.0.0",
"main": "./target_node/index.js",
"license": "SSPL-1.0 OR Elastic License 2.0"
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@
*/

import { duration } from 'moment';
import { ElasticsearchClientConfig, parseClientOptions } from './client_config';
import { DEFAULT_HEADERS } from '../default_headers';
import type { ElasticsearchClientConfig } from '@kbn/core-elasticsearch-server';
import { parseClientOptions } from './client_config';
import { DEFAULT_HEADERS } from './headers';

const createConfig = (
parts: Partial<ElasticsearchClientConfig> = {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,35 +10,8 @@ import { ConnectionOptions as TlsConnectionOptions } from 'tls';
import { URL } from 'url';
import { Duration } from 'moment';
import type { ClientOptions } from '@elastic/elasticsearch/lib/client';
import { ElasticsearchConfig } from '../elasticsearch_config';
import { DEFAULT_HEADERS } from '../default_headers';

/**
* Configuration options to be used to create a {@link IClusterClient | cluster client} using the
* {@link ElasticsearchServiceStart.createClient | createClient API}
*
* @public
*/
export type ElasticsearchClientConfig = Pick<
ElasticsearchConfig,
| 'customHeaders'
| 'maxSockets'
| 'compression'
| 'sniffOnStart'
| 'sniffOnConnectionFault'
| 'requestHeadersWhitelist'
| 'sniffInterval'
| 'hosts'
| 'username'
| 'password'
| 'serviceAccountToken'
> & {
pingTimeout?: ElasticsearchConfig['pingTimeout'] | ClientOptions['pingTimeout'];
requestTimeout?: ElasticsearchConfig['requestTimeout'] | ClientOptions['requestTimeout'];
ssl?: Partial<ElasticsearchConfig['ssl']>;
keepAlive?: boolean;
caFingerprint?: ClientOptions['caFingerprint'];
};
import type { ElasticsearchClientConfig } from '@kbn/core-elasticsearch-server';
import { DEFAULT_HEADERS } from './headers';

/**
* Parse the client options from given client config and `scoped` flag.
Expand Down
Loading