diff --git a/packages/kbn-test/BUILD.bazel b/packages/kbn-test/BUILD.bazel index 6732b08d8bc72a..4dc8d684941f33 100644 --- a/packages/kbn-test/BUILD.bazel +++ b/packages/kbn-test/BUILD.bazel @@ -53,6 +53,7 @@ RUNTIME_DEPS = [ "@npm//execa", "@npm//exit-hook", "@npm//form-data", + "@npm//get-port", "@npm//getopts", "@npm//globby", "@npm//he", @@ -90,6 +91,7 @@ TYPES_DEPS = [ "@npm//del", "@npm//exit-hook", "@npm//form-data", + "@npm//get-port", "@npm//getopts", "@npm//jest", "@npm//jest-cli", diff --git a/packages/kbn-test/src/es/es_client_for_testing.ts b/packages/kbn-test/src/es/es_client_for_testing.ts index 084cb8d77eac57..3eeccffcc2186a 100644 --- a/packages/kbn-test/src/es/es_client_for_testing.ts +++ b/packages/kbn-test/src/es/es_client_for_testing.ts @@ -27,6 +27,22 @@ export interface EsClientForTestingOptions extends Omit +) { + const ccsConfig = config.get('esTestCluster.ccs'); + if (!ccsConfig) { + throw new Error('FTR config is missing esTestCluster.ccs'); + } + + return createEsClientForTesting({ + esUrl: ccsConfig.remoteClusterUrl, + requestTimeout: config.get('timeouts.esRequestTimeout'), + ...overrides, + }); +} + export function createEsClientForFtrConfig( config: Config, overrides?: Omit diff --git a/packages/kbn-test/src/es/index.ts b/packages/kbn-test/src/es/index.ts index 641253acc3647e..bdc33889458279 100644 --- a/packages/kbn-test/src/es/index.ts +++ b/packages/kbn-test/src/es/index.ts @@ -9,5 +9,9 @@ export { createTestEsCluster } from './test_es_cluster'; export type { CreateTestEsClusterOptions, EsTestCluster, ICluster } from './test_es_cluster'; export { esTestConfig } from './es_test_config'; -export { createEsClientForTesting, createEsClientForFtrConfig } from './es_client_for_testing'; +export { + createEsClientForTesting, + createEsClientForFtrConfig, + createRemoteEsClientForFtrConfig, +} from './es_client_for_testing'; export type { EsClientForTestingOptions } from './es_client_for_testing'; diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 6e4fc2fb14628f..27f29ce6995a7d 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -136,7 +136,15 @@ export interface CreateTestEsClusterOptions { * } */ port?: number; + /** + * Should this ES cluster use SSL? + */ ssl?: boolean; + /** + * Explicit transport port for a single node to run on, or a string port range to use eg. '9300-9400' + * defaults to the transport port from `packages/kbn-test/src/es/es_test_config.ts` + */ + transportPort?: number | string; } export function createTestEsCluster< @@ -155,13 +163,14 @@ export function createTestEsCluster< esJavaOpts, clusterName: customClusterName = 'es-test-cluster', ssl, + transportPort, } = options; const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`; const defaultEsArgs = [ `cluster.name=${clusterName}`, - `transport.port=${esTestConfig.getTransportPort()}`, + `transport.port=${transportPort ?? esTestConfig.getTransportPort()}`, // For multi-node clusters, we make all nodes master-eligible by default. ...(nodes.length > 1 ? ['discovery.type=zen', `cluster.initial_master_nodes=${nodes.map((n) => n.name).join(',')}`] diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 42a77b85ddc6c3..cf1afbb810c713 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -192,12 +192,17 @@ export const schema = Joi.object() esTestCluster: Joi.object() .keys({ - license: Joi.string().default('basic'), + license: Joi.valid('basic', 'trial', 'gold').default('basic'), from: Joi.string().default('snapshot'), - serverArgs: Joi.array(), + serverArgs: Joi.array().items(Joi.string()), esJavaOpts: Joi.string(), dataArchive: Joi.string(), ssl: Joi.boolean().default(false), + ccs: Joi.object().keys({ + remoteClusterUrl: Joi.string().uri({ + scheme: /https?/, + }), + }), }) .default(), @@ -290,6 +295,7 @@ export const schema = Joi.object() security: Joi.object() .keys({ roles: Joi.object().default(), + remoteEsRoles: Joi.object(), defaultRoles: Joi.array() .items(Joi.string()) .when('$primary', { diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts index 68e7a4992fcfc8..ba314e8325a657 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts @@ -8,6 +8,7 @@ import { resolve } from 'path'; import type { ToolingLog } from '@kbn/dev-utils'; +import getPort from 'get-port'; import { KIBANA_ROOT } from './paths'; import type { Config } from '../../functional_test_runner/'; import { createTestEsCluster } from '../../es'; @@ -15,32 +16,102 @@ import { createTestEsCluster } from '../../es'; interface RunElasticsearchOptions { log: ToolingLog; esFrom?: string; + config: Config; +} + +interface CcsConfig { + remoteClusterUrl: string; } -export async function runElasticsearch({ + +type EsConfig = ReturnType; + +function getEsConfig({ config, - options, -}: { - config: Config; - options: RunElasticsearchOptions; -}) { - const { log, esFrom } = options; - const ssl = config.get('esTestCluster.ssl'); - const license = config.get('esTestCluster.license'); - const esArgs = config.get('esTestCluster.serverArgs'); - const esJavaOpts = config.get('esTestCluster.esJavaOpts'); + esFrom = config.get('esTestCluster.from'), +}: RunElasticsearchOptions) { + const ssl = !!config.get('esTestCluster.ssl'); + const license: 'basic' | 'trial' | 'gold' = config.get('esTestCluster.license'); + const esArgs: string[] = config.get('esTestCluster.serverArgs') ?? []; + const esJavaOpts: string | undefined = config.get('esTestCluster.esJavaOpts'); const isSecurityEnabled = esArgs.includes('xpack.security.enabled=true'); - const cluster = createTestEsCluster({ - port: config.get('servers.elasticsearch.port'), - password: isSecurityEnabled ? 'changeme' : config.get('servers.elasticsearch.password'), + const port: number | undefined = config.get('servers.elasticsearch.port'); + const ccsConfig: CcsConfig | undefined = config.get('esTestCluster.ccs'); + + const password: string | undefined = isSecurityEnabled + ? 'changeme' + : config.get('servers.elasticsearch.password'); + + const dataArchive: string | undefined = config.get('esTestCluster.dataArchive'); + + return { + ssl, license, - log, - basePath: resolve(KIBANA_ROOT, '.es'), - esFrom: esFrom || config.get('esTestCluster.from'), - dataArchive: config.get('esTestCluster.dataArchive'), esArgs, esJavaOpts, - ssl, + isSecurityEnabled, + esFrom, + port, + password, + dataArchive, + ccsConfig, + }; +} + +export async function runElasticsearch( + options: RunElasticsearchOptions +): Promise<() => Promise> { + const { log } = options; + const config = getEsConfig(options); + + if (!config.ccsConfig) { + const node = await startEsNode(log, 'ftr', config); + return async () => { + await node.cleanup(); + }; + } + + const remotePort = await getPort(); + const remoteNode = await startEsNode(log, 'ftr-remote', { + ...config, + port: parseInt(new URL(config.ccsConfig.remoteClusterUrl).port, 10), + transportPort: remotePort, + }); + + const localNode = await startEsNode(log, 'ftr-local', { + ...config, + esArgs: [...config.esArgs, `cluster.remote.ftr-remote.seeds=localhost:${remotePort}`], + }); + + return async () => { + await localNode.cleanup(); + await remoteNode.cleanup(); + }; +} + +async function startEsNode( + log: ToolingLog, + name: string, + config: EsConfig & { transportPort?: number } +) { + const cluster = createTestEsCluster({ + clusterName: `cluster-${name}`, + esArgs: config.esArgs, + esFrom: config.esFrom, + esJavaOpts: config.esJavaOpts, + license: config.license, + password: config.password, + port: config.port, + ssl: config.ssl, + log, + basePath: resolve(KIBANA_ROOT, '.es'), + nodes: [ + { + name, + dataArchive: config.dataArchive, + }, + ], + transportPort: config.transportPort, }); await cluster.start(); diff --git a/packages/kbn-test/src/functional_tests/tasks.ts b/packages/kbn-test/src/functional_tests/tasks.ts index 5906193ca145c7..b1213ceef2905e 100644 --- a/packages/kbn-test/src/functional_tests/tasks.ts +++ b/packages/kbn-test/src/functional_tests/tasks.ts @@ -108,10 +108,10 @@ export async function runTests(options: RunTestsParams) { await withProcRunner(log, async (procs) => { const config = await readConfigFile(log, options.esVersion, configPath); - let es; + let shutdownEs; try { if (process.env.TEST_ES_DISABLE_STARTUP !== 'true') { - es = await runElasticsearch({ config, options: { ...options, log } }); + shutdownEs = await runElasticsearch({ ...options, log, config }); } await runKibanaServer({ procs, config, options }); await runFtr({ configPath, options: { ...options, log } }); @@ -125,8 +125,8 @@ export async function runTests(options: RunTestsParams) { await procs.stop('kibana'); } finally { - if (es) { - await es.cleanup(); + if (shutdownEs) { + await shutdownEs(); } } } @@ -166,7 +166,7 @@ export async function startServers({ ...options }: StartServerOptions) { await withProcRunner(log, async (procs) => { const config = await readConfigFile(log, options.esVersion, options.config); - const es = await runElasticsearch({ config, options: opts }); + const shutdownEs = await runElasticsearch({ ...opts, config }); await runKibanaServer({ procs, config, @@ -190,7 +190,7 @@ export async function startServers({ ...options }: StartServerOptions) { log.success(makeSuccessMessage(options)); await procs.waitForAllToStop(); - await es.cleanup(); + await shutdownEs(); }); } diff --git a/packages/kbn-test/src/index.ts b/packages/kbn-test/src/index.ts index 26d40f70edb780..c9f0e67c558f19 100644 --- a/packages/kbn-test/src/index.ts +++ b/packages/kbn-test/src/index.ts @@ -36,6 +36,7 @@ export { createTestEsCluster, createEsClientForTesting, createEsClientForFtrConfig, + createRemoteEsClientForFtrConfig, } from './es'; export { diff --git a/scripts/functional_tests.js b/scripts/functional_tests.js index 972c682ab0d9f8..b185bcb0ea5d0c 100644 --- a/scripts/functional_tests.js +++ b/scripts/functional_tests.js @@ -9,7 +9,7 @@ require('../src/setup_node_env'); require('@kbn/test').runTestsCli([ require.resolve('../test/functional/config.js'), - require.resolve('../test/functional_ccs/config.js'), + require.resolve('../test/functional_ccs/config.ts'), require.resolve('../test/plugin_functional/config.ts'), require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'), require.resolve('../test/new_visualize_flow/config.ts'), diff --git a/test/common/services/es_archiver.ts b/test/common/services/es_archiver.ts index 2ea4b6ce3a4342..3212e809286860 100644 --- a/test/common/services/es_archiver.ts +++ b/test/common/services/es_archiver.ts @@ -8,11 +8,12 @@ import { EsArchiver } from '@kbn/es-archiver'; import { FtrProviderContext } from '../ftr_provider_context'; -import * as KibanaServer from './kibana_server'; +import * as KibanaServer from '../../common/services/kibana_server'; export function EsArchiverProvider({ getService }: FtrProviderContext): EsArchiver { const config = getService('config'); const client = getService('es'); + const log = getService('log'); const kibanaServer = getService('kibanaServer'); const retry = getService('retry'); diff --git a/test/common/services/security/system_indices_user.ts b/test/common/services/security/system_indices_user.ts index 2546fbeafffa71..091621207a6710 100644 --- a/test/common/services/security/system_indices_user.ts +++ b/test/common/services/security/system_indices_user.ts @@ -6,25 +6,18 @@ * Side Public License, v 1. */ -import { systemIndicesSuperuser, createEsClientForFtrConfig } from '@kbn/test'; +import { Client } from '@elastic/elasticsearch'; +import { ToolingLog } from '@kbn/dev-utils'; +import { + systemIndicesSuperuser, + createEsClientForFtrConfig, + createRemoteEsClientForFtrConfig, +} from '@kbn/test'; import { FtrProviderContext } from '../../ftr_provider_context'; const SYSTEM_INDICES_SUPERUSER_ROLE = 'system_indices_superuser'; -export async function createSystemIndicesUser(ctx: FtrProviderContext) { - const log = ctx.getService('log'); - const config = ctx.getService('config'); - - const enabled = !config - .get('esTestCluster.serverArgs') - .some((arg: string) => arg === 'xpack.security.enabled=false'); - - if (!enabled) { - return; - } - - const es = createEsClientForFtrConfig(config); - +async function ensureSystemIndicesUser(es: Client, log: ToolingLog) { // There are cases where the test config file doesn't have security disabled // but tests are still executed on ES without security. Checking this case // by trying to fetch the users list. @@ -67,3 +60,24 @@ export async function createSystemIndicesUser(ctx: FtrProviderContext) { await es.close(); } + +export async function createSystemIndicesUser(ctx: FtrProviderContext) { + const log = ctx.getService('log'); + const config = ctx.getService('config'); + + const enabled = !config + .get('esTestCluster.serverArgs') + .some((arg: string) => arg === 'xpack.security.enabled=false'); + + if (!enabled) { + return; + } + + const localEs = createEsClientForFtrConfig(config); + await ensureSystemIndicesUser(localEs, log); + + if (config.get('esTestCluster.ccs')) { + const remoteEs = createRemoteEsClientForFtrConfig(config); + await ensureSystemIndicesUser(remoteEs, log); + } +} diff --git a/test/common/services/security/test_user.ts b/test/common/services/security/test_user.ts index 7c4751220fa1f1..bc5dbf68698bc0 100644 --- a/test/common/services/security/test_user.ts +++ b/test/common/services/security/test_user.ts @@ -90,6 +90,28 @@ export async function createTestUserService(ctx: FtrProviderContext, role: Role, await role.create(name, definition); } + // when configured to setup remote roles, load the remote es service and set them up directly via es + const remoteEsRoles: undefined | Record = config.get('security.remoteEsRoles'); + if (remoteEsRoles) { + let remoteEs; + try { + remoteEs = ctx.getService('remoteEs' as 'es'); + } catch (error) { + throw new Error( + 'unable to load `remoteEs` cluster, which should provide an ES client configured to talk to the remote cluster. Include that service from another FTR config or fix the error it is throwing on creation: ' + + error.message + ); + } + + for (const [name, body] of Object.entries(remoteEsRoles)) { + log.info(`creating ${name} role on remote cluster`); + await remoteEs.security.putRole({ + name, + ...body, + }); + } + } + // delete the test_user if present (will it error if the user doesn't exist?) try { await user.delete(TEST_USER_NAME); diff --git a/test/functional/fixtures/kbn_archiver/date_nested_ccs.json b/test/functional/fixtures/kbn_archiver/date_nested_ccs.json index 933b21d920c00b..9a411ba96705a9 100644 --- a/test/functional/fixtures/kbn_archiver/date_nested_ccs.json +++ b/test/functional/fixtures/kbn_archiver/date_nested_ccs.json @@ -2,10 +2,10 @@ "attributes": { "fields": "[]", "timeFieldName": "nested.timestamp", - "title": "remote:date-nested" + "title": "ftr-remote:date-nested" }, "coreMigrationVersion": "8.2.0", - "id": "remote:date-nested", + "id": "ftr-remote:date-nested", "migrationVersion": { "index-pattern": "8.0.0" }, diff --git a/test/functional/fixtures/kbn_archiver/discover_ccs.json b/test/functional/fixtures/kbn_archiver/discover_ccs.json index d53aa1bc759afb..4c1143ed4e798f 100644 --- a/test/functional/fixtures/kbn_archiver/discover_ccs.json +++ b/test/functional/fixtures/kbn_archiver/discover_ccs.json @@ -3,10 +3,10 @@ "fieldAttrs": "{\"referer\":{\"customLabel\":\"Referer custom\"}}", "fields": "[{\"name\":\"@message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@message\"}}},{\"name\":\"@tags\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@tags\"}}},{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"agent\"}}},{\"name\":\"bytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"extension\"}}},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"headings\"}}},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"host\"}}},{\"name\":\"id\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"index\"}}},{\"name\":\"ip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"links\"}}},{\"name\":\"machine.os\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"machine.os\"}}},{\"name\":\"machine.ram\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"esTypes\":[\"double\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"nestedField.child\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"nested\":{\"path\":\"nestedField\"}}},{\"name\":\"phpmemory\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:section\"}}},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:tag\"}}},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:description\"}}},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image\"}}},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:height\"}}},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:width\"}}},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:site_name\"}}},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:title\"}}},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:type\"}}},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:url\"}}},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:card\"}}},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:description\"}}},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:image\"}}},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:site\"}}},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:title\"}}},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.url\"}}},{\"name\":\"request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"request\"}}},{\"name\":\"response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"response\"}}},{\"name\":\"spaces\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"spaces\"}}},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"url\"}}},{\"name\":\"utc_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"xss\"}}}]", "timeFieldName": "@timestamp", - "title": "remote:logstash-*" + "title": "ftr-remote:logstash-*" }, "coreMigrationVersion": "8.0.0", - "id": "remote:logstash-*", + "id": "ftr-remote:logstash-*", "migrationVersion": { "index-pattern": "7.11.0" }, @@ -41,7 +41,7 @@ }, "references": [ { - "id": "remote:logstash-*", + "id": "ftr-remote:logstash-*", "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern" } diff --git a/test/functional_ccs/apps/discover/_data_view_ccs.ts b/test/functional_ccs/apps/discover/data_view_ccs.ts similarity index 85% rename from test/functional_ccs/apps/discover/_data_view_ccs.ts rename to test/functional_ccs/apps/discover/data_view_ccs.ts index 5a04e0e2a7532b..44258b9cbadd63 100644 --- a/test/functional_ccs/apps/discover/_data_view_ccs.ts +++ b/test/functional_ccs/apps/discover/data_view_ccs.ts @@ -6,13 +6,13 @@ * Side Public License, v 1. */ -import { FtrProviderContext } from './ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); const testSubjects = getService('testSubjects'); const kibanaServer = getService('kibanaServer'); - const esArchiver = getService('esArchiver'); + const remoteEsArchiver = getService('remoteEsArchiver'); const security = getService('security'); const PageObjects = getPageObjects(['common', 'discover', 'header', 'timePicker']); @@ -30,8 +30,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // FLAKY: https://github.com/elastic/kibana/issues/126658 describe.skip('discover integration with data view editor', function describeIndexTests() { before(async function () { - await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']); - await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); + await security.testUser.setRoles([ + 'kibana_admin', + 'test_logstash_reader', + 'ccs_remote_search', + ]); + await remoteEsArchiver.loadIfNeeded( + 'test/functional/fixtures/es_archiver/logstash_functional' + ); await kibanaServer.savedObjects.clean({ types: ['saved-search', 'index-pattern'] }); await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); @@ -45,7 +51,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('use ccs to create a new data view', async function () { - const dataViewToCreate = 'remote:logstash'; + const dataViewToCreate = 'ftr-remote:logstash'; await createDataView(dataViewToCreate); await PageObjects.header.waitUntilLoadingHasFinished(); await retry.waitForWithTimeout( diff --git a/test/functional_ccs/apps/discover/index.ts b/test/functional_ccs/apps/discover/index.ts index 629423b1b75aa4..2e9d428f44c601 100644 --- a/test/functional_ccs/apps/discover/index.ts +++ b/test/functional_ccs/apps/discover/index.ts @@ -6,38 +6,24 @@ * Side Public License, v 1. */ -import { FtrProviderContext } from './ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, loadTestFile }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const browser = getService('browser'); - const esClient = getService('es'); describe('discover app css', function () { this.tags('ciGroup6'); - before(async function () { - await esClient.cluster.putSettings({ - persistent: { - cluster: { - remote: { - remote: { - skip_unavailable: 'true', - seeds: ['localhost:9300'], - }, - }, - }, - }, - }); - return browser.setWindowSize(1300, 800); + before(async () => { + await browser.setWindowSize(1300, 800); }); - after(function unloadMakelogs() { - // Make sure to clean up the cluster setting from the before above. - return esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); - }); + loadTestFile(require.resolve('./data_view_ccs')); + loadTestFile(require.resolve('./saved_queries_ccs')); - loadTestFile(require.resolve('./_data_view_ccs')); - loadTestFile(require.resolve('./_saved_queries_ccs')); + after(async () => { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); }); } diff --git a/test/functional_ccs/apps/discover/_saved_queries_ccs.ts b/test/functional_ccs/apps/discover/saved_queries_ccs.ts similarity index 93% rename from test/functional_ccs/apps/discover/_saved_queries_ccs.ts rename to test/functional_ccs/apps/discover/saved_queries_ccs.ts index 325f279ff28ab7..08b6d61368f5d4 100644 --- a/test/functional_ccs/apps/discover/_saved_queries_ccs.ts +++ b/test/functional_ccs/apps/discover/saved_queries_ccs.ts @@ -8,12 +8,12 @@ import expect from '@kbn/expect'; -import { FtrProviderContext } from './ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); const log = getService('log'); - const esArchiver = getService('esArchiver'); + const remoteEsArchiver = getService('remoteEsArchiver'); const kibanaServer = getService('kibanaServer'); const PageObjects = getPageObjects(['common', 'discover', 'timePicker']); const browser = getService('browser'); @@ -47,8 +47,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await kibanaServer.importExport.load( 'test/functional/fixtures/kbn_archiver/date_nested_ccs.json' ); - await esArchiver.load('test/functional/fixtures/es_archiver/date_nested'); - await esArchiver.load('test/functional/fixtures/es_archiver/logstash_functional'); + await remoteEsArchiver.load('test/functional/fixtures/es_archiver/date_nested'); + await remoteEsArchiver.load('test/functional/fixtures/es_archiver/logstash_functional'); await kibanaServer.uiSettings.replace(defaultSettings); log.debug('discover'); @@ -61,8 +61,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await kibanaServer.importExport.unload( 'test/functional/fixtures/kbn_archiver/date_nested_ccs' ); - await esArchiver.unload('test/functional/fixtures/es_archiver/date_nested'); - await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + await remoteEsArchiver.unload('test/functional/fixtures/es_archiver/date_nested'); + await remoteEsArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); await PageObjects.common.unsetTime(); }); @@ -87,12 +87,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await filterBar.hasFilter('extension.raw', 'jpg')).to.be(false); expect(await queryBar.getQueryString()).to.eql(''); - await PageObjects.discover.selectIndexPattern('remote:date-nested'); + await PageObjects.discover.selectIndexPattern('ftr-remote:date-nested'); expect(await filterBar.hasFilter('extension.raw', 'jpg')).to.be(false); expect(await queryBar.getQueryString()).to.eql(''); - await PageObjects.discover.selectIndexPattern('remote:logstash-*'); + await PageObjects.discover.selectIndexPattern('ftr-remote:logstash-*'); expect(await filterBar.hasFilter('extension.raw', 'jpg')).to.be(false); expect(await queryBar.getQueryString()).to.eql(''); diff --git a/test/functional_ccs/config.js b/test/functional_ccs/config.js deleted file mode 100644 index 4cd88757983720..00000000000000 --- a/test/functional_ccs/config.js +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { services } from '../functional/services'; - -export default async function ({ readConfigFile }) { - const functionalConfig = await readConfigFile(require.resolve('../functional/config')); - - return { - ...functionalConfig.getAll(), - - testFiles: [require.resolve('./apps/discover')], - - services, - - junit: { - reportName: 'Kibana CCS Tests', - }, - }; -} diff --git a/test/functional_ccs/config.ts b/test/functional_ccs/config.ts new file mode 100644 index 00000000000000..e99a5310453d93 --- /dev/null +++ b/test/functional_ccs/config.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import { services } from './services'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../functional/config')); + + return { + ...functionalConfig.getAll(), + + testFiles: [require.resolve('./apps/discover')], + + services, + + junit: { + reportName: 'Kibana CCS Tests', + }, + + security: { + ...functionalConfig.get('security'), + remoteEsRoles: { + ccs_remote_search: { + indices: [ + { + names: ['*'], + privileges: ['read', 'view_index_metadata', 'read_cross_cluster'], + }, + ], + }, + }, + defaultRoles: [...(functionalConfig.get('security.defaultRoles') ?? []), 'ccs_remote_search'], + }, + + esTestCluster: { + ...functionalConfig.get('esTestCluster'), + ccs: { + remoteClusterUrl: + process.env.REMOTE_CLUSTER_URL ?? + `http://elastic:changeme@localhost:${ + functionalConfig.get('servers.elasticsearch.port') + 1 + }`, + }, + }, + }; +} diff --git a/test/functional_ccs/apps/discover/ftr_provider_context.d.ts b/test/functional_ccs/ftr_provider_context.ts similarity index 80% rename from test/functional_ccs/apps/discover/ftr_provider_context.d.ts rename to test/functional_ccs/ftr_provider_context.ts index ea232d23463e65..8fa82b46ac4063 100644 --- a/test/functional_ccs/apps/discover/ftr_provider_context.d.ts +++ b/test/functional_ccs/ftr_provider_context.ts @@ -7,7 +7,7 @@ */ import { GenericFtrProviderContext } from '@kbn/test'; -import { services } from '../../../functional/services'; -import { pageObjects } from '../../../functional/page_objects'; +import { services } from './services'; +import { pageObjects } from '../functional/page_objects'; export type FtrProviderContext = GenericFtrProviderContext; diff --git a/test/functional_ccs/services/index.ts b/test/functional_ccs/services/index.ts new file mode 100644 index 00000000000000..dcdffa077fe083 --- /dev/null +++ b/test/functional_ccs/services/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { services as functionalServices } from '../../functional/services'; +import { RemoteEsProvider } from './remote_es'; +import { RemoteEsArchiverProvider } from './remote_es_archiver'; + +export const services = { + ...functionalServices, + remoteEs: RemoteEsProvider, + remoteEsArchiver: RemoteEsArchiverProvider, +}; diff --git a/test/functional_ccs/services/remote_es.ts b/test/functional_ccs/services/remote_es.ts new file mode 100644 index 00000000000000..05a10d9e068f03 --- /dev/null +++ b/test/functional_ccs/services/remote_es.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Client } from '@elastic/elasticsearch'; + +import { systemIndicesSuperuser, createRemoteEsClientForFtrConfig } from '@kbn/test'; +import { FtrProviderContext } from '../ftr_provider_context'; + +/** + * Kibana-specific @elastic/elasticsearch client instance. + */ +export function RemoteEsProvider({ getService }: FtrProviderContext): Client { + const config = getService('config'); + + return createRemoteEsClientForFtrConfig(config, { + // Use system indices user so tests can write to system indices + authOverride: systemIndicesSuperuser, + }); +} diff --git a/test/functional_ccs/services/remote_es_archiver.ts b/test/functional_ccs/services/remote_es_archiver.ts new file mode 100644 index 00000000000000..569792d050a4d5 --- /dev/null +++ b/test/functional_ccs/services/remote_es_archiver.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EsArchiver } from '@kbn/es-archiver'; +import { FtrProviderContext } from '../ftr_provider_context'; + +export function RemoteEsArchiverProvider({ getService }: FtrProviderContext): EsArchiver { + const remoteEs = getService('remoteEs'); + const log = getService('log'); + const kibanaServer = getService('kibanaServer'); + + return new EsArchiver({ + client: remoteEs, + log, + kbnClient: kibanaServer, + }); +}