From 680ef034834d9c55ec1a9abb789f0fb0677db698 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 16 Feb 2022 10:00:39 -0600 Subject: [PATCH 01/15] Remove ts refs build from bootstrap (#125314) Co-authored-by: Tyler Smalley Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- packages/kbn-pm/dist/index.js | 7 ------- packages/kbn-pm/src/commands/bootstrap.ts | 10 ---------- 2 files changed, 17 deletions(-) diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index 607afa266da831..d390966a6a52f6 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -8987,13 +8987,6 @@ const BootstrapCommand = { }, { prefix: '[vscode]', debug: false - }); - await Object(_utils_child_process__WEBPACK_IMPORTED_MODULE_3__["spawnStreaming"])(process.execPath, ['scripts/build_ts_refs', '--ignore-type-failures'], { - cwd: kbn.getAbsolute(), - env: process.env - }, { - prefix: '[ts refs]', - debug: false }); // send timings await reporter.timings({ diff --git a/packages/kbn-pm/src/commands/bootstrap.ts b/packages/kbn-pm/src/commands/bootstrap.ts index e7b8cad7ebc161..0b3141ab9a5a98 100644 --- a/packages/kbn-pm/src/commands/bootstrap.ts +++ b/packages/kbn-pm/src/commands/bootstrap.ts @@ -137,16 +137,6 @@ export const BootstrapCommand: ICommand = { { prefix: '[vscode]', debug: false } ); - await spawnStreaming( - process.execPath, - ['scripts/build_ts_refs', '--ignore-type-failures'], - { - cwd: kbn.getAbsolute(), - env: process.env, - }, - { prefix: '[ts refs]', debug: false } - ); - // send timings await reporter.timings({ upstreamBranch: kbn.kibanaProject.json.branch, From 4319ea80a430625213b1eff468b20da6ecb5a732 Mon Sep 17 00:00:00 2001 From: Lee Drengenberg Date: Wed, 16 Feb 2022 10:02:44 -0600 Subject: [PATCH 02/15] [Archive Migration] replace es_archive with kbn_archive invalid_scripted_field.json (#125321) * remove es_archive with kbn_archive invalid_scripted_field.json * remove unused es_archive files * replace another usage of es_archives/invalid_scripted_field * clear any existing index patterns before loading our new one * remove .only Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- test/functional/apps/discover/_errors.ts | 8 +- .../invalid_scripted_field/data.json.gz | Bin 956 -> 0 bytes .../invalid_scripted_field/mappings.json | 212 ------------------ .../kbn_archiver/invalid_scripted_field.json | 16 ++ .../apps/discover/error_handling.ts | 7 +- .../invalid_scripted_field/data.json.gz | Bin 956 -> 0 bytes .../invalid_scripted_field/mappings.json | 212 ------------------ 7 files changed, 27 insertions(+), 428 deletions(-) delete mode 100644 test/functional/fixtures/es_archiver/invalid_scripted_field/data.json.gz delete mode 100644 test/functional/fixtures/es_archiver/invalid_scripted_field/mappings.json create mode 100644 test/functional/fixtures/kbn_archiver/invalid_scripted_field.json delete mode 100644 x-pack/test/functional/es_archives/invalid_scripted_field/data.json.gz delete mode 100644 x-pack/test/functional/es_archives/invalid_scripted_field/mappings.json diff --git a/test/functional/apps/discover/_errors.ts b/test/functional/apps/discover/_errors.ts index b252cbf5f0824b..327f39ee0dec47 100644 --- a/test/functional/apps/discover/_errors.ts +++ b/test/functional/apps/discover/_errors.ts @@ -11,6 +11,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); const toasts = getService('toasts'); const testSubjects = getService('testSubjects'); const PageObjects = getPageObjects(['common', 'header', 'discover', 'timePicker']); @@ -18,13 +19,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('errors', function describeIndexTests() { before(async function () { await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); - await esArchiver.load('test/functional/fixtures/es_archiver/invalid_scripted_field'); + await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); + await kibanaServer.importExport.load( + 'test/functional/fixtures/kbn_archiver/invalid_scripted_field' + ); await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); await PageObjects.common.navigateToApp('discover'); }); after(async function () { - await esArchiver.load('test/functional/fixtures/es_archiver/empty_kibana'); + await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); }); describe('invalid scripted field error', () => { diff --git a/test/functional/fixtures/es_archiver/invalid_scripted_field/data.json.gz b/test/functional/fixtures/es_archiver/invalid_scripted_field/data.json.gz deleted file mode 100644 index 380dd6049179a59865b91aef55ba5836c37dcea7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 956 zcmV;t14H~DiwFP!000026V;nRPunmQ$M1ZKD#wXNP0MJ>Y0@U7T{ulUY)sLs{gPNX zc4j*Rg!t~W(}XsxASz&eJyfcn9s8HRpP!vLzB?L?B6nM3l#U{e?T_Da_191 zDR{_bTIHfh=kX+-j6v2VByH(XKykbYW#TKb1O>g_>c+5jMWeow4U4xe8zK2cQ+ey+ zNf6h3i$+1^NNway4tv^J(2<-Bp?|6686 z?KSer9HhlyLNKg^r3@E3(*zq7g~0+`XYGp`1b%LG^+mJqK_J`b-5+lwiVa-9BUNYJ z1{Ja%F*ztKA&CMGEq#RVH3wBaBnl&HA3T}h`;lf#XJzf6q=RFbHjjq8YN{ogopE~O z>zp}Tg|Q*grg0BQD-I$(=3`-Kk4o@qaVQ0!<*h^eCJy}uFiZ+Bk9ZUZyYzo%!5fFl z3a#ik&jS)A2P#mO5Y&?r%1G@1^UG9o6EH>M0WQycFt+P{3(%DD_eWtP~1kO-Q6Q6Gf#A zR8&Gxg_Sn9bZ8vJzCuH$#{!T+32ifEygN`e7EPQBW8KP54^F~1w*7fmUDmYR-C&Le z$vAuG=byE5;T;4RCz!;v(p=;Mc^2M6(s%*+1ZmwYrLg58pju>()T@K%IVsJePmnq& z$gAlY2`j0K)IaOD%~E#FAtr4-MMY}!?+L&I&@e4g6s03CWyg64ie3xuO8SDfn6&k@ z!eK^h75)q)Z7rQ}lC+L3AaM=fcYpQu5_&A!3N}Vk7SQ7n*3Vsw@P$;$z;LoEZbBY_ zmWQw~-@!)l zYjS1Tthc>LLb$iEe+xoM?0VxZz61XiZ5)3J4tE~tFZ|7ggFNd~7sFzHxf8ObE&C|b diff --git a/test/functional/fixtures/es_archiver/invalid_scripted_field/mappings.json b/test/functional/fixtures/es_archiver/invalid_scripted_field/mappings.json deleted file mode 100644 index 0d41e0ce86c146..00000000000000 --- a/test/functional/fixtures/es_archiver/invalid_scripted_field/mappings.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "type": "index", - "value": { - "aliases": { - ".kibana": {} - }, - "index": ".kibana_1", - "mappings": { - "dynamic": "strict", - "properties": { - "config": { - "dynamic": "true", - "properties": { - "buildNum": { - "type": "keyword" - }, - "defaultIndex": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "dashboard": { - "properties": { - "description": { - "type": "text" - }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "optionsJSON": { - "type": "text" - }, - "panelsJSON": { - "type": "text" - }, - "refreshInterval": { - "properties": { - "display": { - "type": "keyword" - }, - "pause": { - "type": "boolean" - }, - "section": { - "type": "integer" - }, - "value": { - "type": "integer" - } - } - }, - "timeFrom": { - "type": "keyword" - }, - "timeRestore": { - "type": "boolean" - }, - "timeTo": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "uiStateJSON": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "index-pattern": { - "properties": { - "fieldFormatMap": { - "type": "text" - }, - "fields": { - "type": "text" - }, - "intervalName": { - "type": "keyword" - }, - "notExpandable": { - "type": "boolean" - }, - "sourceFilters": { - "type": "text" - }, - "timeFieldName": { - "type": "keyword" - }, - "title": { - "type": "text" - } - } - }, - "search": { - "properties": { - "columns": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "sort": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "server": { - "properties": { - "uuid": { - "type": "keyword" - } - } - }, - "type": { - "type": "keyword" - }, - "updated_at": { - "type": "date" - }, - "url": { - "properties": { - "accessCount": { - "type": "long" - }, - "accessDate": { - "type": "date" - }, - "createDate": { - "type": "date" - }, - "url": { - "fields": { - "keyword": { - "ignore_above": 2048, - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "visualization": { - "properties": { - "description": { - "type": "text" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "savedSearchId": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "uiStateJSON": { - "type": "text" - }, - "version": { - "type": "integer" - }, - "visState": { - "type": "text" - } - } - } - } - }, - "settings": { - "index": { - "number_of_replicas": "0", - "number_of_shards": "1" - } - } - } -} \ No newline at end of file diff --git a/test/functional/fixtures/kbn_archiver/invalid_scripted_field.json b/test/functional/fixtures/kbn_archiver/invalid_scripted_field.json new file mode 100644 index 00000000000000..23abe20855e3e6 --- /dev/null +++ b/test/functional/fixtures/kbn_archiver/invalid_scripted_field.json @@ -0,0 +1,16 @@ +{ + "attributes": { + "fields": "[{\"name\":\"@message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"machine.os\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"machine.ram\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"phpmemory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"spaces\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"utc_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"invalid_scripted_field\",\"type\":\"number\",\"count\":0,\"scripted\":true,\"script\":\"invalid\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]", + "timeFieldName": "@timestamp", + "title": "log*" + }, + "coreMigrationVersion": "7.17.1", + "id": "18ea0c30-2d77-11e8-93f2-6f8a391f33ef", + "migrationVersion": { + "index-pattern": "7.11.0" + }, + "references": [], + "type": "index-pattern", + "updated_at": "2018-03-22T02:17:11.427Z", + "version": "WzMsMl0=" +} \ No newline at end of file diff --git a/x-pack/test/functional/apps/discover/error_handling.ts b/x-pack/test/functional/apps/discover/error_handling.ts index 19cedc8ecb5551..194a9a592c1c2f 100644 --- a/x-pack/test/functional/apps/discover/error_handling.ts +++ b/x-pack/test/functional/apps/discover/error_handling.ts @@ -10,19 +10,22 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); const toasts = getService('toasts'); const PageObjects = getPageObjects(['common', 'discover', 'timePicker']); describe('errors', function describeIndexTests() { before(async function () { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional'); - await esArchiver.load('x-pack/test/functional/es_archives/invalid_scripted_field'); + await kibanaServer.importExport.load( + 'test/functional/fixtures/kbn_archiver/invalid_scripted_field' + ); await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); await PageObjects.common.navigateToApp('discover'); }); after(async function () { - await esArchiver.unload('x-pack/test/functional/es_archives/invalid_scripted_field'); + await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); }); // this is the same test as in OSS but it catches different error message issue in different licences diff --git a/x-pack/test/functional/es_archives/invalid_scripted_field/data.json.gz b/x-pack/test/functional/es_archives/invalid_scripted_field/data.json.gz deleted file mode 100644 index 380dd6049179a59865b91aef55ba5836c37dcea7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 956 zcmV;t14H~DiwFP!000026V;nRPunmQ$M1ZKD#wXNP0MJ>Y0@U7T{ulUY)sLs{gPNX zc4j*Rg!t~W(}XsxASz&eJyfcn9s8HRpP!vLzB?L?B6nM3l#U{e?T_Da_191 zDR{_bTIHfh=kX+-j6v2VByH(XKykbYW#TKb1O>g_>c+5jMWeow4U4xe8zK2cQ+ey+ zNf6h3i$+1^NNway4tv^J(2<-Bp?|6686 z?KSer9HhlyLNKg^r3@E3(*zq7g~0+`XYGp`1b%LG^+mJqK_J`b-5+lwiVa-9BUNYJ z1{Ja%F*ztKA&CMGEq#RVH3wBaBnl&HA3T}h`;lf#XJzf6q=RFbHjjq8YN{ogopE~O z>zp}Tg|Q*grg0BQD-I$(=3`-Kk4o@qaVQ0!<*h^eCJy}uFiZ+Bk9ZUZyYzo%!5fFl z3a#ik&jS)A2P#mO5Y&?r%1G@1^UG9o6EH>M0WQycFt+P{3(%DD_eWtP~1kO-Q6Q6Gf#A zR8&Gxg_Sn9bZ8vJzCuH$#{!T+32ifEygN`e7EPQBW8KP54^F~1w*7fmUDmYR-C&Le z$vAuG=byE5;T;4RCz!;v(p=;Mc^2M6(s%*+1ZmwYrLg58pju>()T@K%IVsJePmnq& z$gAlY2`j0K)IaOD%~E#FAtr4-MMY}!?+L&I&@e4g6s03CWyg64ie3xuO8SDfn6&k@ z!eK^h75)q)Z7rQ}lC+L3AaM=fcYpQu5_&A!3N}Vk7SQ7n*3Vsw@P$;$z;LoEZbBY_ zmWQw~-@!)l zYjS1Tthc>LLb$iEe+xoM?0VxZz61XiZ5)3J4tE~tFZ|7ggFNd~7sFzHxf8ObE&C|b diff --git a/x-pack/test/functional/es_archives/invalid_scripted_field/mappings.json b/x-pack/test/functional/es_archives/invalid_scripted_field/mappings.json deleted file mode 100644 index 0d41e0ce86c146..00000000000000 --- a/x-pack/test/functional/es_archives/invalid_scripted_field/mappings.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "type": "index", - "value": { - "aliases": { - ".kibana": {} - }, - "index": ".kibana_1", - "mappings": { - "dynamic": "strict", - "properties": { - "config": { - "dynamic": "true", - "properties": { - "buildNum": { - "type": "keyword" - }, - "defaultIndex": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "dashboard": { - "properties": { - "description": { - "type": "text" - }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "optionsJSON": { - "type": "text" - }, - "panelsJSON": { - "type": "text" - }, - "refreshInterval": { - "properties": { - "display": { - "type": "keyword" - }, - "pause": { - "type": "boolean" - }, - "section": { - "type": "integer" - }, - "value": { - "type": "integer" - } - } - }, - "timeFrom": { - "type": "keyword" - }, - "timeRestore": { - "type": "boolean" - }, - "timeTo": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "uiStateJSON": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "index-pattern": { - "properties": { - "fieldFormatMap": { - "type": "text" - }, - "fields": { - "type": "text" - }, - "intervalName": { - "type": "keyword" - }, - "notExpandable": { - "type": "boolean" - }, - "sourceFilters": { - "type": "text" - }, - "timeFieldName": { - "type": "keyword" - }, - "title": { - "type": "text" - } - } - }, - "search": { - "properties": { - "columns": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "sort": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "version": { - "type": "integer" - } - } - }, - "server": { - "properties": { - "uuid": { - "type": "keyword" - } - } - }, - "type": { - "type": "keyword" - }, - "updated_at": { - "type": "date" - }, - "url": { - "properties": { - "accessCount": { - "type": "long" - }, - "accessDate": { - "type": "date" - }, - "createDate": { - "type": "date" - }, - "url": { - "fields": { - "keyword": { - "ignore_above": 2048, - "type": "keyword" - } - }, - "type": "text" - } - } - }, - "visualization": { - "properties": { - "description": { - "type": "text" - }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } - }, - "savedSearchId": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "uiStateJSON": { - "type": "text" - }, - "version": { - "type": "integer" - }, - "visState": { - "type": "text" - } - } - } - } - }, - "settings": { - "index": { - "number_of_replicas": "0", - "number_of_shards": "1" - } - } - } -} \ No newline at end of file From 68f475f0edfd75a58cdf37c539a5ff08124986a9 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 16 Feb 2022 17:08:59 +0100 Subject: [PATCH 03/15] retry clicking (#125768) --- test/functional/services/inspector.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/functional/services/inspector.ts b/test/functional/services/inspector.ts index 753d9b7b0b85ea..b8dec2a2092fb3 100644 --- a/test/functional/services/inspector.ts +++ b/test/functional/services/inspector.ts @@ -195,8 +195,12 @@ export class InspectorService extends FtrService { */ public async openInspectorView(viewId: string): Promise { this.log.debug(`Open Inspector view ${viewId}`); - await this.testSubjects.click('inspectorViewChooser'); - await this.testSubjects.click(viewId); + await this.retry.try(async () => { + await this.testSubjects.click('inspectorViewChooser'); + // check whether popover menu opens, if not, fail and retry opening + await this.testSubjects.existOrFail(viewId, { timeout: 2000 }); + await this.testSubjects.click(viewId); + }); } /** From 8757e965fcc0e5cee96fa8c1be8627b9d3c21966 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 16 Feb 2022 10:14:42 -0600 Subject: [PATCH 04/15] Revert "Remove ts refs build from bootstrap (#125314)" This reverts commit 680ef034834d9c55ec1a9abb789f0fb0677db698. --- packages/kbn-pm/dist/index.js | 7 +++++++ packages/kbn-pm/src/commands/bootstrap.ts | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index d390966a6a52f6..607afa266da831 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -8987,6 +8987,13 @@ const BootstrapCommand = { }, { prefix: '[vscode]', debug: false + }); + await Object(_utils_child_process__WEBPACK_IMPORTED_MODULE_3__["spawnStreaming"])(process.execPath, ['scripts/build_ts_refs', '--ignore-type-failures'], { + cwd: kbn.getAbsolute(), + env: process.env + }, { + prefix: '[ts refs]', + debug: false }); // send timings await reporter.timings({ diff --git a/packages/kbn-pm/src/commands/bootstrap.ts b/packages/kbn-pm/src/commands/bootstrap.ts index 0b3141ab9a5a98..e7b8cad7ebc161 100644 --- a/packages/kbn-pm/src/commands/bootstrap.ts +++ b/packages/kbn-pm/src/commands/bootstrap.ts @@ -137,6 +137,16 @@ export const BootstrapCommand: ICommand = { { prefix: '[vscode]', debug: false } ); + await spawnStreaming( + process.execPath, + ['scripts/build_ts_refs', '--ignore-type-failures'], + { + cwd: kbn.getAbsolute(), + env: process.env, + }, + { prefix: '[ts refs]', debug: false } + ); + // send timings await reporter.timings({ upstreamBranch: kbn.kibanaProject.json.branch, From 2e3257c90cac9ed60b302a64689620e25eacd3dc Mon Sep 17 00:00:00 2001 From: Kevin Logan <56395104+kevinlog@users.noreply.github.com> Date: Wed, 16 Feb 2022 11:25:34 -0500 Subject: [PATCH 05/15] [Security Solution] Change manifest check in policy test to improve test (#125784) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../security_solution_endpoint/apps/endpoint/policy_details.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index 72b64f78319ef9..3f057d198a25c1 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -493,7 +493,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { version: policyInfo.packageInfo.version, }, }, - artifact_manifest: agentFullPolicy.inputs[0].artifact_manifest, + artifact_manifest: agentFullPolicyUpdated.inputs[0].artifact_manifest, }), ]); }); From 37dfc22c1a52931bd7bc0426c9fc13cd43250d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yulia=20=C4=8Cech?= <6585477+yuliacech@users.noreply.github.com> Date: Wed, 16 Feb 2022 17:45:32 +0100 Subject: [PATCH 06/15] [Snapshot & Restore] Fix repository types on Cloud (#125656) * [Snapshot & Restore] Fix repository types on Cloud * [Snapshot & Restore] Move repo types tests into `on-prem` describe block Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../snapshot_restore/common/constants.ts | 9 +- .../server/routes/api/repositories.test.ts | 101 ++++++++++-------- .../server/routes/api/repositories.ts | 13 ++- .../apis/management/snapshot_restore/index.ts | 1 + .../snapshot_restore/repositories.ts | 34 ++++++ 5 files changed, 107 insertions(+), 51 deletions(-) create mode 100644 x-pack/test/api_integration/apis/management/snapshot_restore/repositories.ts diff --git a/x-pack/plugins/snapshot_restore/common/constants.ts b/x-pack/plugins/snapshot_restore/common/constants.ts index c4b64bb9395f8e..d8b159fd01e962 100644 --- a/x-pack/plugins/snapshot_restore/common/constants.ts +++ b/x-pack/plugins/snapshot_restore/common/constants.ts @@ -35,12 +35,15 @@ export enum REPOSITORY_TYPES { } // Deliberately do not include `source` as a default repository since we treat it as a flag -export const DEFAULT_REPOSITORY_TYPES: RepositoryType[] = [ +export const ON_PREM_REPOSITORY_TYPES: RepositoryType[] = [ + REPOSITORY_TYPES.fs, + REPOSITORY_TYPES.url, +]; + +export const MODULE_REPOSITORY_TYPES: RepositoryType[] = [ REPOSITORY_TYPES.azure, REPOSITORY_TYPES.gcs, REPOSITORY_TYPES.s3, - REPOSITORY_TYPES.fs, - REPOSITORY_TYPES.url, ]; export const PLUGIN_REPOSITORY_TYPES: RepositoryType[] = [REPOSITORY_TYPES.hdfs]; diff --git a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts index 844be90f9b8423..f29a84beaecf89 100644 --- a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts +++ b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { DEFAULT_REPOSITORY_TYPES, REPOSITORY_PLUGINS_MAP } from '../../../common'; +import { + ON_PREM_REPOSITORY_TYPES, + MODULE_REPOSITORY_TYPES, + REPOSITORY_PLUGINS_MAP, +} from '../../../common'; import { addBasePath } from '../helpers'; import { registerRepositoriesRoutes } from './repositories'; import { RouterMock, routeDependencies, RequestMock } from '../../test/helpers'; @@ -253,52 +257,59 @@ describe('[Snapshot and Restore API Routes] Repositories', () => { path: addBasePath('repository_types'), }; - it('returns default types if no repository plugins returned from ES', async () => { - nodesInfoFn.mockResolvedValue({ nodes: { testNodeId: { plugins: [] } } }); - - const expectedResponse = [...DEFAULT_REPOSITORY_TYPES]; - await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); - }); - - it('returns default types with any repository plugins returned from ES', async () => { - const pluginNames = Object.keys(REPOSITORY_PLUGINS_MAP); - const pluginTypes = Object.entries(REPOSITORY_PLUGINS_MAP).map(([key, value]) => value); - - const mockEsResponse = { - nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } }, - }; - nodesInfoFn.mockResolvedValue(mockEsResponse); - - const expectedResponse = [...DEFAULT_REPOSITORY_TYPES, ...pluginTypes]; - await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); - }); - - it(`doesn't return non-repository plugins returned from ES`, async () => { - const pluginNames = ['foo-plugin', 'bar-plugin']; - const mockEsResponse = { - nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } }, - }; - nodesInfoFn.mockResolvedValue(mockEsResponse); - - const expectedResponse = [...DEFAULT_REPOSITORY_TYPES]; - - await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); - }); - - it(`doesn't return repository plugins that are not installed on all nodes`, async () => { - const dataNodePlugins = ['repository-hdfs']; - const masterNodePlugins: string[] = []; - const mockEsResponse = { - nodes: { - dataNode: { plugins: [...dataNodePlugins.map((key) => ({ name: key }))] }, - masterNode: { plugins: [...masterNodePlugins.map((key) => ({ name: key }))] }, - }, - }; - nodesInfoFn.mockResolvedValue(mockEsResponse); + // TODO add Cloud specific tests for repo types + describe('on prem', () => { + it('returns module types and on-prem types if no repository plugins returned from ES', async () => { + nodesInfoFn.mockResolvedValue({ nodes: { testNodeId: { plugins: [] } } }); + + const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; + await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + }); + + it('returns module types and on-prem types with any repository plugins returned from ES', async () => { + const pluginNames = Object.keys(REPOSITORY_PLUGINS_MAP); + const pluginTypes = Object.entries(REPOSITORY_PLUGINS_MAP).map(([key, value]) => value); + + const mockEsResponse = { + nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } }, + }; + nodesInfoFn.mockResolvedValue(mockEsResponse); + + const expectedResponse = [ + ...MODULE_REPOSITORY_TYPES, + ...ON_PREM_REPOSITORY_TYPES, + ...pluginTypes, + ]; + await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + }); + + it(`doesn't return non-repository plugins returned from ES`, async () => { + const pluginNames = ['foo-plugin', 'bar-plugin']; + const mockEsResponse = { + nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } }, + }; + nodesInfoFn.mockResolvedValue(mockEsResponse); + + const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; + + await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + }); + + it(`doesn't return repository plugins that are not installed on all nodes`, async () => { + const dataNodePlugins = ['repository-hdfs']; + const masterNodePlugins: string[] = []; + const mockEsResponse = { + nodes: { + dataNode: { plugins: [...dataNodePlugins.map((key) => ({ name: key }))] }, + masterNode: { plugins: [...masterNodePlugins.map((key) => ({ name: key }))] }, + }, + }; + nodesInfoFn.mockResolvedValue(mockEsResponse); - const expectedResponse = [...DEFAULT_REPOSITORY_TYPES]; + const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; - await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + }); }); it('should throw if ES error', async () => { diff --git a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.ts b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.ts index 22c4d4a2b3e91f..4666870133f1f5 100644 --- a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.ts +++ b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.ts @@ -12,7 +12,11 @@ import type { PluginStats, } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { DEFAULT_REPOSITORY_TYPES, REPOSITORY_PLUGINS_MAP } from '../../../common'; +import { + ON_PREM_REPOSITORY_TYPES, + REPOSITORY_PLUGINS_MAP, + MODULE_REPOSITORY_TYPES, +} from '../../../common'; import { Repository, RepositoryType } from '../../../common/types'; import { RouteDependencies } from '../../types'; import { addBasePath } from '../helpers'; @@ -154,8 +158,11 @@ export function registerRepositoriesRoutes({ { path: addBasePath('repository_types'), validate: false }, license.guardApiRoute(async (ctx, req, res) => { const { client: clusterClient } = ctx.core.elasticsearch; - // In ECE/ESS, do not enable the default types - const types: RepositoryType[] = isCloudEnabled ? [] : [...DEFAULT_REPOSITORY_TYPES]; + // module repo types are available everywhere out of the box + // on-prem repo types are not available on Cloud + const types: RepositoryType[] = isCloudEnabled + ? [...MODULE_REPOSITORY_TYPES] + : [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; try { const { nodes } = await clusterClient.asCurrentUser.nodes.info({ diff --git a/x-pack/test/api_integration/apis/management/snapshot_restore/index.ts b/x-pack/test/api_integration/apis/management/snapshot_restore/index.ts index db5dbc9735e66f..debfb683cd8833 100644 --- a/x-pack/test/api_integration/apis/management/snapshot_restore/index.ts +++ b/x-pack/test/api_integration/apis/management/snapshot_restore/index.ts @@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { describe('Snapshot and Restore', () => { loadTestFile(require.resolve('./policies')); loadTestFile(require.resolve('./snapshots')); + loadTestFile(require.resolve('./repositories')); }); } diff --git a/x-pack/test/api_integration/apis/management/snapshot_restore/repositories.ts b/x-pack/test/api_integration/apis/management/snapshot_restore/repositories.ts new file mode 100644 index 00000000000000..982f32faf73ce3 --- /dev/null +++ b/x-pack/test/api_integration/apis/management/snapshot_restore/repositories.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +const API_BASE_PATH = '/api/snapshot_restore'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const deployment = getService('deployment'); + + describe('snapshot repositories', function () { + describe('repository types', () => { + it('returns a list of default repository types', async () => { + const { body } = await supertest.get(`${API_BASE_PATH}/repository_types`).expect(200); + + const isCloud = await deployment.isCloud(); + if (isCloud) { + // on Cloud there are only module repo types + expect(body).to.eql(['azure', 'gcs', 's3']); + } else { + // on prem there are module repo types and file system and url repo types + expect(body).to.eql(['azure', 'gcs', 's3', 'fs', 'url']); + } + }); + }); + }); +} From 3dd2d75bce2939b39e72b7ef60e32392b3fad876 Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Wed, 16 Feb 2022 17:50:17 +0100 Subject: [PATCH 07/15] [Fleet] Use package policy version to determine managed policy upgrades (#125788) --- .../services/managed_package_policies.test.ts | 76 ++++--------------- .../services/managed_package_policies.ts | 22 ++---- 2 files changed, 19 insertions(+), 79 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts b/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts index 7ccfeb4fe76412..af3b3ef4dfccda 100644 --- a/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts +++ b/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts @@ -7,11 +7,11 @@ import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks'; -import type { Installation, PackageInfo } from '../../common'; +import type { Installation } from '../../common'; import { shouldUpgradePolicies, upgradeManagedPackagePolicies } from './managed_package_policies'; import { packagePolicyService } from './package_policy'; -import { getPackageInfo, getInstallation } from './epm/packages'; +import { getInstallation } from './epm/packages'; jest.mock('./package_policy'); jest.mock('./epm/packages'); @@ -30,7 +30,6 @@ describe('upgradeManagedPackagePolicies', () => { afterEach(() => { (packagePolicyService.get as jest.Mock).mockReset(); (packagePolicyService.getUpgradeDryRunDiff as jest.Mock).mockReset(); - (getPackageInfo as jest.Mock).mockReset(); (getInstallation as jest.Mock).mockReset(); (packagePolicyService.upgrade as jest.Mock).mockReset(); }); @@ -69,17 +68,10 @@ describe('upgradeManagedPackagePolicies', () => { } ); - (getPackageInfo as jest.Mock).mockImplementationOnce( - ({ savedObjectsClient, pkgName, pkgVersion }) => ({ - name: pkgName, - version: pkgVersion, - keepPoliciesUpToDate: false, - }) - ); - (getInstallation as jest.Mock).mockResolvedValueOnce({ id: 'test-installation', version: '0.0.1', + keep_policies_up_to_date: false, }); await upgradeManagedPackagePolicies(soClient, esClient, ['non-managed-package-id']); @@ -121,17 +113,10 @@ describe('upgradeManagedPackagePolicies', () => { } ); - (getPackageInfo as jest.Mock).mockImplementationOnce( - ({ savedObjectsClient, pkgName, pkgVersion }) => ({ - name: pkgName, - version: pkgVersion, - keepPoliciesUpToDate: true, - }) - ); - (getInstallation as jest.Mock).mockResolvedValueOnce({ id: 'test-installation', version: '1.0.0', + keep_policies_up_to_date: true, }); await upgradeManagedPackagePolicies(soClient, esClient, ['managed-package-id']); @@ -177,17 +162,10 @@ describe('upgradeManagedPackagePolicies', () => { } ); - (getPackageInfo as jest.Mock).mockImplementationOnce( - ({ savedObjectsClient, pkgName, pkgVersion }) => ({ - name: pkgName, - version: pkgVersion, - keepPoliciesUpToDate: true, - }) - ); - (getInstallation as jest.Mock).mockResolvedValueOnce({ id: 'test-installation', version: '1.0.0', + keep_policies_up_to_date: true, }); const result = await upgradeManagedPackagePolicies(soClient, esClient, [ @@ -229,19 +207,12 @@ describe('shouldUpgradePolicies', () => { describe('package policy is up-to-date', () => { describe('keep_policies_up_to_date is true', () => { it('returns false', () => { - const packageInfo = { - version: '1.0.0', - keepPoliciesUpToDate: true, - }; - const installedPackage = { version: '1.0.0', + keep_policies_up_to_date: true, }; - const result = shouldUpgradePolicies( - packageInfo as PackageInfo, - installedPackage as Installation - ); + const result = shouldUpgradePolicies('1.0.0', installedPackage as Installation); expect(result).toBe(false); }); @@ -249,19 +220,12 @@ describe('shouldUpgradePolicies', () => { describe('keep_policies_up_to_date is false', () => { it('returns false', () => { - const packageInfo = { - version: '1.0.0', - keepPoliciesUpToDate: false, - }; - const installedPackage = { version: '1.0.0', + keep_policies_up_to_date: false, }; - const result = shouldUpgradePolicies( - packageInfo as PackageInfo, - installedPackage as Installation - ); + const result = shouldUpgradePolicies('1.0.0', installedPackage as Installation); expect(result).toBe(false); }); @@ -271,19 +235,12 @@ describe('shouldUpgradePolicies', () => { describe('package policy is out-of-date', () => { describe('keep_policies_up_to_date is true', () => { it('returns true', () => { - const packageInfo = { - version: '1.0.0', - keepPoliciesUpToDate: true, - }; - const installedPackage = { version: '1.1.0', + keep_policies_up_to_date: true, }; - const result = shouldUpgradePolicies( - packageInfo as PackageInfo, - installedPackage as Installation - ); + const result = shouldUpgradePolicies('1.0.0', installedPackage as Installation); expect(result).toBe(true); }); @@ -291,19 +248,12 @@ describe('shouldUpgradePolicies', () => { describe('keep_policies_up_to_date is false', () => { it('returns false', () => { - const packageInfo = { - version: '1.0.0', - keepPoliciesUpToDate: false, - }; - const installedPackage = { version: '1.1.0', + keep_policies_up_to_date: false, }; - const result = shouldUpgradePolicies( - packageInfo as PackageInfo, - installedPackage as Installation - ); + const result = shouldUpgradePolicies('1.0.0', installedPackage as Installation); expect(result).toBe(false); }); diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.ts b/x-pack/plugins/fleet/server/services/managed_package_policies.ts index c702cfe96d9860..77715ad488feb6 100644 --- a/x-pack/plugins/fleet/server/services/managed_package_policies.ts +++ b/x-pack/plugins/fleet/server/services/managed_package_policies.ts @@ -8,14 +8,10 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import semverGte from 'semver/functions/gte'; -import type { - Installation, - PackageInfo, - UpgradePackagePolicyDryRunResponseItem, -} from '../../common'; +import type { Installation, UpgradePackagePolicyDryRunResponseItem } from '../../common'; import { appContextService } from './app_context'; -import { getInstallation, getPackageInfo } from './epm/packages'; +import { getInstallation } from './epm/packages'; import { packagePolicyService } from './package_policy'; export interface UpgradeManagedPackagePoliciesResult { @@ -42,12 +38,6 @@ export const upgradeManagedPackagePolicies = async ( continue; } - const packageInfo = await getPackageInfo({ - savedObjectsClient: soClient, - pkgName: packagePolicy.package.name, - pkgVersion: packagePolicy.package.version, - }); - const installedPackage = await getInstallation({ savedObjectsClient: soClient, pkgName: packagePolicy.package.name, @@ -62,7 +52,7 @@ export const upgradeManagedPackagePolicies = async ( continue; } - if (shouldUpgradePolicies(packageInfo, installedPackage)) { + if (shouldUpgradePolicies(packagePolicy.package.version, installedPackage)) { // Since upgrades don't report diffs/errors, we need to perform a dry run first in order // to notify the user of any granular policy upgrade errors that occur during Fleet's // preconfiguration check @@ -101,13 +91,13 @@ export const upgradeManagedPackagePolicies = async ( }; export function shouldUpgradePolicies( - packageInfo: PackageInfo, + packagePolicyPackageVersion: string, installedPackage: Installation ): boolean { const isPolicyVersionGteInstalledVersion = semverGte( - packageInfo.version, + packagePolicyPackageVersion, installedPackage.version ); - return !isPolicyVersionGteInstalledVersion && !!packageInfo.keepPoliciesUpToDate; + return !isPolicyVersionGteInstalledVersion && !!installedPackage.keep_policies_up_to_date; } From 82c21484b38f044ad6d2d389abca590671169766 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Wed, 16 Feb 2022 08:57:42 -0800 Subject: [PATCH 08/15] Add tooltip for Surrounding documents (#123890) Co-authored-by: Andrea Del Rio --- .../discover_grid/discover_grid_flyout.tsx | 53 ++++++++++++++----- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.tsx index 371eb014eab8fb..bdf9268c73060e 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid_flyout.tsx @@ -15,6 +15,7 @@ import { EuiFlyout, EuiFlyoutBody, EuiFlyoutHeader, + EuiIconTip, EuiTitle, EuiButtonEmpty, EuiText, @@ -147,7 +148,8 @@ export function DiscoverGridFlyout({ {indexPattern.isTimeBased() && indexPattern.id && ( - - - {i18n.translate('discover.grid.tableRow.viewSurroundingDocumentsLinkTextSimple', { - defaultMessage: 'Surrounding documents', - })} - - + + + + {i18n.translate( + 'discover.grid.tableRow.viewSurroundingDocumentsLinkTextSimple', + { + defaultMessage: 'Surrounding documents', + } + )} + + + + + + )} {activePage !== -1 && ( From d68a0d390a34ce712815b78a97ca82d220c151b8 Mon Sep 17 00:00:00 2001 From: Tobias Stadler Date: Wed, 16 Feb 2022 18:03:29 +0100 Subject: [PATCH 09/15] [APM] Improve open transaction/span/error in Discover (#125431) --- docs/apm/transactions.asciidoc | 2 +- .../__snapshots__/index.test.tsx.snap | 6 +----- .../error_group_details/detail_view/index.tsx | 19 ++++++++----------- .../waterfall/span_flyout/index.tsx | 15 ++++++--------- .../transaction_action_menu/sections.test.ts | 6 +++--- .../transaction_action_menu/sections.ts | 2 +- .../transaction_action_menu.test.tsx | 2 +- .../columns/series_actions.test.tsx | 4 ++-- .../series_editor/columns/series_actions.tsx | 2 +- 9 files changed, 24 insertions(+), 34 deletions(-) diff --git a/docs/apm/transactions.asciidoc b/docs/apm/transactions.asciidoc index e7555a6c3e3d6e..e8c5d6600c0bd0 100644 --- a/docs/apm/transactions.asciidoc +++ b/docs/apm/transactions.asciidoc @@ -151,7 +151,7 @@ Learn more about a trace sample in the *Metadata* tab: * User - Requires additional configuration, but allows you to see which user experienced the current transaction. TIP: All of this data is stored in documents in Elasticsearch. -This means you can select "Actions - View sample document" to see the actual Elasticsearch document under the discover tab. +This means you can select "Actions - View transaction in Discover" to see the actual Elasticsearch document under the discover tab. *Trace sample logs* diff --git a/x-pack/plugins/apm/public/components/app/error_group_details/detail_view/__snapshots__/index.test.tsx.snap b/x-pack/plugins/apm/public/components/app/error_group_details/detail_view/__snapshots__/index.test.tsx.snap index de13bf910ce0f3..5f300b45de80ac 100644 --- a/x-pack/plugins/apm/public/components/app/error_group_details/detail_view/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/apm/public/components/app/error_group_details/detail_view/__snapshots__/index.test.tsx.snap @@ -34,11 +34,7 @@ exports[`DetailView should render Discover button 1`] = ` } kuery="" > - - View 10 occurrences in Discover. - + View 10 occurrences in Discover. `; diff --git a/x-pack/plugins/apm/public/components/app/error_group_details/detail_view/index.tsx b/x-pack/plugins/apm/public/components/app/error_group_details/detail_view/index.tsx index 97118bf763d43e..0a6b134275121f 100644 --- a/x-pack/plugins/apm/public/components/app/error_group_details/detail_view/index.tsx +++ b/x-pack/plugins/apm/public/components/app/error_group_details/detail_view/index.tsx @@ -6,7 +6,6 @@ */ import { - EuiButtonEmpty, EuiIcon, EuiPanel, EuiSpacer, @@ -100,16 +99,14 @@ export function DetailView({ errorGroup, urlParams, kuery }: Props) { - - {i18n.translate( - 'xpack.apm.errorGroupDetails.viewOccurrencesInDiscoverButtonLabel', - { - defaultMessage: - 'View {occurrencesCount} {occurrencesCount, plural, one {occurrence} other {occurrences}} in Discover.', - values: { occurrencesCount }, - } - )} - + {i18n.translate( + 'xpack.apm.errorGroupDetails.viewOccurrencesInDiscoverButtonLabel', + { + defaultMessage: + 'View {occurrencesCount} {occurrencesCount, plural, one {occurrence} other {occurrences}} in Discover.', + values: { occurrencesCount }, + } + )} diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/span_flyout/index.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/span_flyout/index.tsx index 0f7a6a295601bb..477098aa81d043 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/span_flyout/index.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/span_flyout/index.tsx @@ -7,7 +7,6 @@ import { EuiBadge, - EuiButtonEmpty, EuiCallOut, EuiFlexGroup, EuiFlexItem, @@ -126,14 +125,12 @@ export function SpanFlyout({ - - {i18n.translate( - 'xpack.apm.transactionDetails.spanFlyout.viewSpanInDiscoverButtonLabel', - { - defaultMessage: 'View span in Discover', - } - )} - + {i18n.translate( + 'xpack.apm.transactionDetails.spanFlyout.viewSpanInDiscoverButtonLabel', + { + defaultMessage: 'View span in Discover', + } + )} diff --git a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.test.ts b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.test.ts index bcb52f4721dd5e..1d37dde6c63439 100644 --- a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.test.ts +++ b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.test.ts @@ -67,7 +67,7 @@ describe('Transaction action menu', () => { actions: [ { key: 'sampleDocument', - label: 'View sample document', + label: 'View transaction in Discover', href: 'some-basepath/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-24h,to:now))&_a=(index:apm_static_index_pattern_id,interval:auto,query:(language:kuery,query:\'processor.event:"transaction" AND transaction.id:"123" AND trace.id:"123"\'))', condition: true, }, @@ -134,7 +134,7 @@ describe('Transaction action menu', () => { actions: [ { key: 'sampleDocument', - label: 'View sample document', + label: 'View transaction in Discover', href: 'some-basepath/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-24h,to:now))&_a=(index:apm_static_index_pattern_id,interval:auto,query:(language:kuery,query:\'processor.event:"transaction" AND transaction.id:"123" AND trace.id:"123"\'))', condition: true, }, @@ -200,7 +200,7 @@ describe('Transaction action menu', () => { actions: [ { key: 'sampleDocument', - label: 'View sample document', + label: 'View transaction in Discover', href: 'some-basepath/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-24h,to:now))&_a=(index:apm_static_index_pattern_id,interval:auto,query:(language:kuery,query:\'processor.event:"transaction" AND transaction.id:"123" AND trace.id:"123"\'))', condition: true, }, diff --git a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts index daf5cb0833b61c..20c10b6d9557c8 100644 --- a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts +++ b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts @@ -191,7 +191,7 @@ export const getSections = ({ label: i18n.translate( 'xpack.apm.transactionActionMenu.viewSampleDocumentLinkLabel', { - defaultMessage: 'View sample document', + defaultMessage: 'View transaction in Discover', } ), href: getDiscoverHref({ diff --git a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.test.tsx b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.test.tsx index 4d82bb69ee9a50..851472cfedabe4 100644 --- a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.test.tsx +++ b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.test.tsx @@ -74,7 +74,7 @@ describe('TransactionActionMenu component', () => { Transactions.transactionWithMinimalData ); - expect(queryByText('View sample document')).not.toBeNull(); + expect(queryByText('View transaction in Discover')).not.toBeNull(); }); it('always renders the trace logs link', async () => { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.test.tsx index d855f648f5da23..3480662f139137 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.test.tsx @@ -35,8 +35,8 @@ describe('SeriesActions', function () { expect(screen.getByLabelText('Series actions list')).toBeVisible(); }); - it('should display a view sample link', function () { - expect(screen.getByLabelText('View sample documents')).toBeVisible(); + it('should display a view transaction link', function () { + expect(screen.getByLabelText('View transaction in Discover')).toBeVisible(); }); it('should display a hide series link', function () { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.tsx index 22d87e3977abc3..b3430da2d35e26 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.tsx @@ -182,7 +182,7 @@ const DELETE_SERIES_TOOLTIP_LABEL = i18n.translate( const VIEW_SAMPLE_DOCUMENTS_LABEL = i18n.translate( 'xpack.observability.seriesEditor.sampleDocuments', { - defaultMessage: 'View sample documents', + defaultMessage: 'View transaction in Discover', } ); From b8ae75b3f7e809474205661d011cad1377a85f7e Mon Sep 17 00:00:00 2001 From: Michael Dokolin Date: Wed, 16 Feb 2022 18:06:09 +0100 Subject: [PATCH 10/15] [Screenshotting] Simplify authentication headers handling (#125673) * Move stripping unsafe headers to the screenshotting plugin * Encapsulate conditional headers handling in the screenshotting plugin * Add KibanaRequest support on input --- .../reporting/common/constants/index.ts | 22 ---- .../common/get_conditional_headers.test.ts | 51 --------- .../common/get_conditional_headers.ts | 34 ------ .../common/get_custom_logo.test.ts | 16 +-- .../export_types/common/get_custom_logo.ts | 12 +- .../server/export_types/common/index.ts | 14 --- .../common/omit_blocked_headers.test.ts | 34 ------ .../common/omit_blocked_headers.ts | 25 ----- .../png/execute_job/index.test.ts | 5 +- .../export_types/png/execute_job/index.ts | 14 +-- .../export_types/png_v2/execute_job.test.ts | 5 +- .../server/export_types/png_v2/execute_job.ts | 13 +-- .../printable_pdf/execute_job/index.ts | 18 +-- .../printable_pdf_v2/execute_job.ts | 17 +-- .../server/routes/diagnostic/screenshot.ts | 17 +-- .../server/browsers/chromium/driver.ts | 106 +++++++----------- .../chromium/driver_factory/index.test.ts | 2 +- .../browsers/chromium/driver_factory/index.ts | 10 +- .../server/browsers/chromium/index.ts | 2 +- .../chromium/strip_unsafe_headers.test.ts | 37 ++++++ .../browsers/chromium/strip_unsafe_headers.ts | 40 +++++++ .../screenshotting/server/browsers/index.ts | 2 +- .../plugins/screenshotting/server/plugin.ts | 11 +- .../server/screenshots/index.test.ts | 2 +- .../server/screenshots/index.ts | 13 ++- .../server/screenshots/observable.test.ts | 6 +- .../server/screenshots/observable.ts | 8 +- .../server/screenshots/open_url.ts | 8 +- 28 files changed, 186 insertions(+), 358 deletions(-) delete mode 100644 x-pack/plugins/reporting/server/export_types/common/get_conditional_headers.test.ts delete mode 100644 x-pack/plugins/reporting/server/export_types/common/get_conditional_headers.ts delete mode 100644 x-pack/plugins/reporting/server/export_types/common/omit_blocked_headers.test.ts delete mode 100644 x-pack/plugins/reporting/server/export_types/common/omit_blocked_headers.ts create mode 100644 x-pack/plugins/screenshotting/server/browsers/chromium/strip_unsafe_headers.test.ts create mode 100644 x-pack/plugins/screenshotting/server/browsers/chromium/strip_unsafe_headers.ts diff --git a/x-pack/plugins/reporting/common/constants/index.ts b/x-pack/plugins/reporting/common/constants/index.ts index b8cf8a27d7fb42..f6b7efabe6449e 100644 --- a/x-pack/plugins/reporting/common/constants/index.ts +++ b/x-pack/plugins/reporting/common/constants/index.ts @@ -32,28 +32,6 @@ export const ALLOWED_JOB_CONTENT_TYPES = [ 'text/plain', ]; -// See: -// https://github.com/chromium/chromium/blob/3611052c055897e5ebbc5b73ea295092e0c20141/services/network/public/cpp/header_util_unittest.cc#L50 -// For a list of headers that chromium doesn't like -export const KBN_SCREENSHOT_HEADER_BLOCK_LIST = [ - 'accept-encoding', - 'connection', - 'content-length', - 'content-type', - 'host', - 'referer', - // `Transfer-Encoding` is hop-by-hop header that is meaningful - // only for a single transport-level connection, and shouldn't - // be stored by caches or forwarded by proxies. - 'transfer-encoding', - 'trailer', - 'te', - 'upgrade', - 'keep-alive', -]; - -export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-']; - export const UI_SETTINGS_SEARCH_INCLUDE_FROZEN = 'search:includeFrozen'; export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo'; export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator'; diff --git a/x-pack/plugins/reporting/server/export_types/common/get_conditional_headers.test.ts b/x-pack/plugins/reporting/server/export_types/common/get_conditional_headers.test.ts deleted file mode 100644 index 73b5875717d32d..00000000000000 --- a/x-pack/plugins/reporting/server/export_types/common/get_conditional_headers.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ReportingConfig } from '../../'; -import { createMockConfig, createMockConfigSchema } from '../../test_helpers'; -import { getConditionalHeaders } from './'; - -let mockConfig: ReportingConfig; - -beforeEach(async () => { - const reportingConfig = { kibanaServer: { hostname: 'custom-hostname' } }; - const mockSchema = createMockConfigSchema(reportingConfig); - mockConfig = createMockConfig(mockSchema); -}); - -describe('conditions', () => { - test(`uses hostname from reporting config if set`, async () => { - const permittedHeaders = { - foo: 'bar', - baz: 'quix', - }; - - const conditionalHeaders = getConditionalHeaders(mockConfig, permittedHeaders); - - expect(conditionalHeaders.conditions.hostname).toEqual( - mockConfig.get('kibanaServer', 'hostname') - ); - expect(conditionalHeaders.conditions.port).toEqual(mockConfig.get('kibanaServer', 'port')); - expect(conditionalHeaders.conditions.protocol).toEqual( - mockConfig.get('kibanaServer', 'protocol') - ); - expect(conditionalHeaders.conditions.basePath).toEqual( - mockConfig.kbnConfig.get('server', 'basePath') - ); - }); -}); - -describe('config formatting', () => { - test(`lowercases kibanaServer.hostname`, async () => { - const reportingConfig = { kibanaServer: { hostname: 'GREAT-HOSTNAME' } }; - const mockSchema = createMockConfigSchema(reportingConfig); - mockConfig = createMockConfig(mockSchema); - - const conditionalHeaders = getConditionalHeaders(mockConfig, {}); - expect(conditionalHeaders.conditions.hostname).toEqual('great-hostname'); - }); -}); diff --git a/x-pack/plugins/reporting/server/export_types/common/get_conditional_headers.ts b/x-pack/plugins/reporting/server/export_types/common/get_conditional_headers.ts deleted file mode 100644 index 262d9733cb16a8..00000000000000 --- a/x-pack/plugins/reporting/server/export_types/common/get_conditional_headers.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ReportingConfig } from '../../'; -import { ConditionalHeaders } from './'; - -export const getConditionalHeaders = ( - config: ReportingConfig, - filteredHeaders: Record -) => { - const { kbnConfig } = config; - const [hostname, port, basePath, protocol] = [ - config.get('kibanaServer', 'hostname'), - config.get('kibanaServer', 'port'), - kbnConfig.get('server', 'basePath'), - config.get('kibanaServer', 'protocol'), - ] as [string, number, string, string]; - - const conditionalHeaders: ConditionalHeaders = { - headers: filteredHeaders, - conditions: { - hostname: hostname ? hostname.toLowerCase() : hostname, - port, - basePath, - protocol, - }, - }; - - return conditionalHeaders; -}; diff --git a/x-pack/plugins/reporting/server/export_types/common/get_custom_logo.test.ts b/x-pack/plugins/reporting/server/export_types/common/get_custom_logo.test.ts index e21b7404f5ed57..f5675b50cfddd4 100644 --- a/x-pack/plugins/reporting/server/export_types/common/get_custom_logo.test.ts +++ b/x-pack/plugins/reporting/server/export_types/common/get_custom_logo.test.ts @@ -7,12 +7,10 @@ import { ReportingCore } from '../..'; import { - createMockConfig, createMockConfigSchema, createMockLevelLogger, createMockReportingCore, } from '../../test_helpers'; -import { getConditionalHeaders } from '.'; import { getCustomLogo } from './get_custom_logo'; let mockReportingPlugin: ReportingCore; @@ -24,7 +22,7 @@ beforeEach(async () => { }); test(`gets logo from uiSettings`, async () => { - const permittedHeaders = { + const headers = { foo: 'bar', baz: 'quix', }; @@ -40,17 +38,7 @@ test(`gets logo from uiSettings`, async () => { get: mockGet, }); - const conditionalHeaders = getConditionalHeaders( - createMockConfig(createMockConfigSchema()), - permittedHeaders - ); - - const { logo } = await getCustomLogo( - mockReportingPlugin, - conditionalHeaders, - 'spaceyMcSpaceIdFace', - logger - ); + const { logo } = await getCustomLogo(mockReportingPlugin, headers, 'spaceyMcSpaceIdFace', logger); expect(mockGet).toBeCalledWith('xpackReporting:customPdfLogo'); expect(logo).toBe('purple pony'); diff --git a/x-pack/plugins/reporting/server/export_types/common/get_custom_logo.ts b/x-pack/plugins/reporting/server/export_types/common/get_custom_logo.ts index 983f6f41af8d9f..fcabd34a642c8c 100644 --- a/x-pack/plugins/reporting/server/export_types/common/get_custom_logo.ts +++ b/x-pack/plugins/reporting/server/export_types/common/get_custom_logo.ts @@ -5,25 +5,21 @@ * 2.0. */ +import type { Headers } from 'src/core/server'; import { ReportingCore } from '../../'; import { UI_SETTINGS_CUSTOM_PDF_LOGO } from '../../../common/constants'; import { LevelLogger } from '../../lib'; -import { ConditionalHeaders } from '../common'; export const getCustomLogo = async ( reporting: ReportingCore, - conditionalHeaders: ConditionalHeaders, + headers: Headers, spaceId: string | undefined, logger: LevelLogger ) => { - const fakeRequest = reporting.getFakeRequest( - { headers: conditionalHeaders.headers }, - spaceId, - logger - ); + const fakeRequest = reporting.getFakeRequest({ headers }, spaceId, logger); const uiSettingsClient = await reporting.getUiSettingsClient(fakeRequest, logger); const logo: string = await uiSettingsClient.get(UI_SETTINGS_CUSTOM_PDF_LOGO); // continue the pipeline - return { conditionalHeaders, logo }; + return { headers, logo }; }; diff --git a/x-pack/plugins/reporting/server/export_types/common/index.ts b/x-pack/plugins/reporting/server/export_types/common/index.ts index 501de48e0450a3..c9589d5262059c 100644 --- a/x-pack/plugins/reporting/server/export_types/common/index.ts +++ b/x-pack/plugins/reporting/server/export_types/common/index.ts @@ -6,9 +6,7 @@ */ export { decryptJobHeaders } from './decrypt_job_headers'; -export { getConditionalHeaders } from './get_conditional_headers'; export { getFullUrls } from './get_full_urls'; -export { omitBlockedHeaders } from './omit_blocked_headers'; export { validateUrls } from './validate_urls'; export { generatePngObservable } from './generate_png'; export { getCustomLogo } from './get_custom_logo'; @@ -17,15 +15,3 @@ export interface TimeRangeParams { min?: Date | string | number | null; max?: Date | string | number | null; } - -export interface ConditionalHeadersConditions { - protocol: string; - hostname: string; - port: number; - basePath: string; -} - -export interface ConditionalHeaders { - headers: Record; - conditions: ConditionalHeadersConditions; -} diff --git a/x-pack/plugins/reporting/server/export_types/common/omit_blocked_headers.test.ts b/x-pack/plugins/reporting/server/export_types/common/omit_blocked_headers.test.ts deleted file mode 100644 index e7f4f984c2054f..00000000000000 --- a/x-pack/plugins/reporting/server/export_types/common/omit_blocked_headers.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { omitBlockedHeaders } from './index'; - -test(`omits blocked headers`, async () => { - const permittedHeaders = { - foo: 'bar', - baz: 'quix', - }; - - const blockedHeaders = { - 'accept-encoding': '', - connection: 'upgrade', - 'content-length': '', - 'content-type': '', - host: '', - 'transfer-encoding': '', - 'proxy-connection': 'bananas', - 'proxy-authorization': 'some-base64-encoded-thing', - trailer: 's are for trucks', - }; - - const filteredHeaders = omitBlockedHeaders({ - ...permittedHeaders, - ...blockedHeaders, - }); - - expect(filteredHeaders).toEqual(permittedHeaders); -}); diff --git a/x-pack/plugins/reporting/server/export_types/common/omit_blocked_headers.ts b/x-pack/plugins/reporting/server/export_types/common/omit_blocked_headers.ts deleted file mode 100644 index c2ad8fdb507f26..00000000000000 --- a/x-pack/plugins/reporting/server/export_types/common/omit_blocked_headers.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { omitBy } from 'lodash'; -import { - KBN_SCREENSHOT_HEADER_BLOCK_LIST, - KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN, -} from '../../../common/constants'; - -export const omitBlockedHeaders = (decryptedHeaders: Record) => { - const filteredHeaders: Record = omitBy( - decryptedHeaders, - (_value, header: string) => - header && - (KBN_SCREENSHOT_HEADER_BLOCK_LIST.includes(header) || - KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN.some((pattern) => - header?.startsWith(pattern) - )) - ); - return filteredHeaders; -}; diff --git a/x-pack/plugins/reporting/server/export_types/png/execute_job/index.test.ts b/x-pack/plugins/reporting/server/export_types/png/execute_job/index.test.ts index bbdab4d75b7bf6..9069ec63a88250 100644 --- a/x-pack/plugins/reporting/server/export_types/png/execute_job/index.test.ts +++ b/x-pack/plugins/reporting/server/export_types/png/execute_job/index.test.ts @@ -87,10 +87,7 @@ test(`passes browserTimezone to generatePng`, async () => { expect.objectContaining({ urls: ['localhost:80undefined/app/kibana#/something'], browserTimezone: 'UTC', - conditionalHeaders: expect.objectContaining({ - conditions: expect.any(Object), - headers: {}, - }), + headers: {}, }) ); }); diff --git a/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts b/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts index 20a2ea98e06d43..67d013740bedd3 100644 --- a/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts +++ b/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts @@ -11,13 +11,7 @@ import { finalize, map, mergeMap, takeUntil, tap } from 'rxjs/operators'; import { PNG_JOB_TYPE, REPORTING_TRANSACTION_TYPE } from '../../../../common/constants'; import { TaskRunResult } from '../../../lib/tasks'; import { RunTaskFn, RunTaskFnFactory } from '../../../types'; -import { - decryptJobHeaders, - getConditionalHeaders, - getFullUrls, - omitBlockedHeaders, - generatePngObservable, -} from '../../common'; +import { decryptJobHeaders, getFullUrls, generatePngObservable } from '../../common'; import { TaskPayloadPNG } from '../types'; export const runTaskFnFactory: RunTaskFnFactory> = @@ -33,16 +27,14 @@ export const runTaskFnFactory: RunTaskFnFactory> = const jobLogger = parentLogger.clone([PNG_JOB_TYPE, 'execute', jobId]); const process$: Rx.Observable = Rx.of(1).pipe( mergeMap(() => decryptJobHeaders(encryptionKey, job.headers, jobLogger)), - map((decryptedHeaders) => omitBlockedHeaders(decryptedHeaders)), - map((filteredHeaders) => getConditionalHeaders(config, filteredHeaders)), - mergeMap((conditionalHeaders) => { + mergeMap((headers) => { const [url] = getFullUrls(config, job); apmGetAssets?.end(); apmGeneratePng = apmTrans?.startSpan('generate-png-pipeline', 'execute'); return generatePngObservable(reporting, jobLogger, { - conditionalHeaders, + headers, urls: [url], browserTimezone: job.browserTimezone, layout: job.layout, diff --git a/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.test.ts b/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.test.ts index f7c6b6138419cb..1b1ad6878d78fb 100644 --- a/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.test.ts +++ b/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.test.ts @@ -94,10 +94,7 @@ test(`passes browserTimezone to generatePng`, async () => { ], ], browserTimezone: 'UTC', - conditionalHeaders: expect.objectContaining({ - conditions: expect.any(Object), - headers: {}, - }), + headers: {}, }) ); }); diff --git a/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.ts b/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.ts index 1acce6e475630e..51044aa324a1aa 100644 --- a/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.ts +++ b/x-pack/plugins/reporting/server/export_types/png_v2/execute_job.ts @@ -11,12 +11,7 @@ import { finalize, map, mergeMap, takeUntil, tap } from 'rxjs/operators'; import { PNG_JOB_TYPE_V2, REPORTING_TRANSACTION_TYPE } from '../../../common/constants'; import { TaskRunResult } from '../../lib/tasks'; import { RunTaskFn, RunTaskFnFactory } from '../../types'; -import { - decryptJobHeaders, - getConditionalHeaders, - omitBlockedHeaders, - generatePngObservable, -} from '../common'; +import { decryptJobHeaders, generatePngObservable } from '../common'; import { getFullRedirectAppUrl } from '../common/v2/get_full_redirect_app_url'; import { TaskPayloadPNGV2 } from './types'; @@ -33,9 +28,7 @@ export const runTaskFnFactory: RunTaskFnFactory> = const jobLogger = parentLogger.clone([PNG_JOB_TYPE_V2, 'execute', jobId]); const process$: Rx.Observable = Rx.of(1).pipe( mergeMap(() => decryptJobHeaders(encryptionKey, job.headers, jobLogger)), - map((decryptedHeaders) => omitBlockedHeaders(decryptedHeaders)), - map((filteredHeaders) => getConditionalHeaders(config, filteredHeaders)), - mergeMap((conditionalHeaders) => { + mergeMap((headers) => { const url = getFullRedirectAppUrl(config, job.spaceId, job.forceNow); const [locatorParams] = job.locatorParams; @@ -43,7 +36,7 @@ export const runTaskFnFactory: RunTaskFnFactory> = apmGeneratePng = apmTrans?.startSpan('generate-png-pipeline', 'execute'); return generatePngObservable(reporting, jobLogger, { - conditionalHeaders, + headers, browserTimezone: job.browserTimezone, layout: job.layout, urls: [[url, locatorParams]], diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts index 02ba917ce329d1..ab3793935e1d86 100644 --- a/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts +++ b/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts @@ -11,13 +11,7 @@ import { catchError, map, mergeMap, takeUntil, tap } from 'rxjs/operators'; import { PDF_JOB_TYPE, REPORTING_TRANSACTION_TYPE } from '../../../../common/constants'; import { TaskRunResult } from '../../../lib/tasks'; import { RunTaskFn, RunTaskFnFactory } from '../../../types'; -import { - decryptJobHeaders, - getConditionalHeaders, - getFullUrls, - omitBlockedHeaders, - getCustomLogo, -} from '../../common'; +import { decryptJobHeaders, getFullUrls, getCustomLogo } from '../../common'; import { generatePdfObservable } from '../lib/generate_pdf'; import { TaskPayloadPDF } from '../types'; @@ -34,12 +28,8 @@ export const runTaskFnFactory: RunTaskFnFactory> = const process$: Rx.Observable = Rx.of(1).pipe( mergeMap(() => decryptJobHeaders(encryptionKey, job.headers, jobLogger)), - map((decryptedHeaders) => omitBlockedHeaders(decryptedHeaders)), - map((filteredHeaders) => getConditionalHeaders(config, filteredHeaders)), - mergeMap((conditionalHeaders) => - getCustomLogo(reporting, conditionalHeaders, job.spaceId, jobLogger) - ), - mergeMap(({ logo, conditionalHeaders }) => { + mergeMap((headers) => getCustomLogo(reporting, headers, job.spaceId, jobLogger)), + mergeMap(({ headers, logo }) => { const urls = getFullUrls(config, job); const { browserTimezone, layout, title } = job; @@ -53,7 +43,7 @@ export const runTaskFnFactory: RunTaskFnFactory> = { urls, browserTimezone, - conditionalHeaders, + headers, layout, }, logo diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf_v2/execute_job.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf_v2/execute_job.ts index de6f2ae70a756f..85684bca66b869 100644 --- a/x-pack/plugins/reporting/server/export_types/printable_pdf_v2/execute_job.ts +++ b/x-pack/plugins/reporting/server/export_types/printable_pdf_v2/execute_job.ts @@ -11,12 +11,7 @@ import { catchError, map, mergeMap, takeUntil, tap } from 'rxjs/operators'; import { PDF_JOB_TYPE_V2, REPORTING_TRANSACTION_TYPE } from '../../../common/constants'; import { TaskRunResult } from '../../lib/tasks'; import { RunTaskFn, RunTaskFnFactory } from '../../types'; -import { - decryptJobHeaders, - getConditionalHeaders, - omitBlockedHeaders, - getCustomLogo, -} from '../common'; +import { decryptJobHeaders, getCustomLogo } from '../common'; import { generatePdfObservable } from './lib/generate_pdf'; import { TaskPayloadPDFV2 } from './types'; @@ -33,12 +28,8 @@ export const runTaskFnFactory: RunTaskFnFactory> = const process$: Rx.Observable = Rx.of(1).pipe( mergeMap(() => decryptJobHeaders(encryptionKey, job.headers, jobLogger)), - map((decryptedHeaders) => omitBlockedHeaders(decryptedHeaders)), - map((filteredHeaders) => getConditionalHeaders(config, filteredHeaders)), - mergeMap((conditionalHeaders) => - getCustomLogo(reporting, conditionalHeaders, job.spaceId, jobLogger) - ), - mergeMap(({ logo, conditionalHeaders }) => { + mergeMap((headers) => getCustomLogo(reporting, headers, job.spaceId, jobLogger)), + mergeMap(({ logo, headers }) => { const { browserTimezone, layout, title, locatorParams } = job; apmGetAssets?.end(); @@ -51,7 +42,7 @@ export const runTaskFnFactory: RunTaskFnFactory> = locatorParams, { browserTimezone, - conditionalHeaders, + headers, layout, }, logo diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.ts b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.ts index 2d5a2540451046..90b4c9d9a30c6f 100644 --- a/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.ts +++ b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.ts @@ -9,7 +9,7 @@ import { i18n } from '@kbn/i18n'; import { ReportingCore } from '../..'; import { APP_WRAPPER_CLASS } from '../../../../../../src/core/server'; import { API_DIAGNOSE_URL } from '../../../common/constants'; -import { omitBlockedHeaders, generatePngObservable } from '../../export_types/common'; +import { generatePngObservable } from '../../export_types/common'; import { getAbsoluteUrlFactory } from '../../export_types/common/get_absolute_url'; import { LevelLogger as Logger } from '../../lib'; import { authorizedUserPreRouting } from '../lib/authorized_user_pre_routing'; @@ -24,9 +24,8 @@ export const registerDiagnoseScreenshot = (reporting: ReportingCore, logger: Log path: `${API_DIAGNOSE_URL}/screenshot`, validate: {}, }, - authorizedUserPreRouting(reporting, async (_user, _context, req, res) => { + authorizedUserPreRouting(reporting, async (_user, _context, request, res) => { const config = reporting.getConfig(); - const decryptedHeaders = req.headers as Record; const [basePath, protocol, hostname, port] = [ config.kbnConfig.get('server', 'basePath'), config.get('kibanaServer', 'protocol'), @@ -51,19 +50,9 @@ export const registerDiagnoseScreenshot = (reporting: ReportingCore, logger: Log }, }; - const conditionalHeaders = { - headers: omitBlockedHeaders(decryptedHeaders), - conditions: { - hostname, - port: +port, - basePath, - protocol, - }, - }; - return generatePngObservable(reporting, logger, { - conditionalHeaders, layout, + request, browserTimezone: 'America/Los_Angeles', urls: [hashUrl], }) diff --git a/x-pack/plugins/screenshotting/server/browsers/chromium/driver.ts b/x-pack/plugins/screenshotting/server/browsers/chromium/driver.ts index 4e71194486d01e..f9bf31ad35f6fe 100644 --- a/x-pack/plugins/screenshotting/server/browsers/chromium/driver.ts +++ b/x-pack/plugins/screenshotting/server/browsers/chromium/driver.ts @@ -5,29 +5,18 @@ * 2.0. */ -import { map, truncate } from 'lodash'; +import { truncate } from 'lodash'; import open from 'opn'; import puppeteer, { ElementHandle, EvaluateFn, Page, SerializableOrJSHandle } from 'puppeteer'; import { parse as parseUrl } from 'url'; -import { Logger } from 'src/core/server'; +import { Headers, Logger } from 'src/core/server'; import { KBN_SCREENSHOT_MODE_HEADER, ScreenshotModePluginSetup, } from '../../../../../../src/plugins/screenshot_mode/server'; import { ConfigType } from '../../config'; import { allowRequest } from '../network_policy'; - -export interface ConditionalHeadersConditions { - protocol: string; - hostname: string; - port: number; - basePath: string; -} - -export interface ConditionalHeaders { - headers: Record; - conditions: ConditionalHeadersConditions; -} +import { stripUnsafeHeaders } from './strip_unsafe_headers'; export type Context = Record; @@ -52,8 +41,8 @@ export interface Viewport { } interface OpenOptions { - conditionalHeaders: ConditionalHeaders; context?: Context; + headers: Headers; waitForSelector: string; timeout: number; } @@ -104,6 +93,7 @@ export class HeadlessChromiumDriver { constructor( private screenshotMode: ScreenshotModePluginSetup, private config: ConfigType, + private basePath: string, private readonly page: Page ) {} @@ -123,7 +113,7 @@ export class HeadlessChromiumDriver { */ async open( url: string, - { conditionalHeaders, context, waitForSelector: pageLoadSelector, timeout }: OpenOptions, + { headers, context, waitForSelector: pageLoadSelector, timeout }: OpenOptions, logger: Logger ): Promise { logger.info(`opening url ${url}`); @@ -142,7 +132,7 @@ export class HeadlessChromiumDriver { } await this.page.setRequestInterception(true); - this.registerListeners(conditionalHeaders, logger); + this.registerListeners(url, headers, logger); await this.page.goto(url, { waitUntil: 'domcontentloaded' }); if (this.config.browser.chromium.inspect) { @@ -243,14 +233,13 @@ export class HeadlessChromiumDriver { }); } - private registerListeners(conditionalHeaders: ConditionalHeaders, logger: Logger) { + private registerListeners(url: string, customHeaders: Headers, logger: Logger) { if (this.listenersAttached) { return; } - // @ts-ignore // FIXME: retrieve the client in open() and pass in the client - const client = this.page._client; + const client = this.page.client(); // We have to reach into the Chrome Devtools Protocol to apply headers as using // puppeteer's API will cause map tile requests to hang indefinitely: @@ -277,19 +266,17 @@ export class HeadlessChromiumDriver { return; } - if (this._shouldUseCustomHeaders(conditionalHeaders.conditions, interceptedUrl)) { + if (this._shouldUseCustomHeaders(url, interceptedUrl)) { logger.trace(`Using custom headers for ${interceptedUrl}`); - const headers = map( - { - ...interceptedRequest.request.headers, - ...conditionalHeaders.headers, - [KBN_SCREENSHOT_MODE_HEADER]: 'true', - }, - (value, name) => ({ - name, - value, - }) - ); + const headers = Object.entries({ + ...interceptedRequest.request.headers, + ...stripUnsafeHeaders(customHeaders), + [KBN_SCREENSHOT_MODE_HEADER]: 'true', + }).flatMap(([name, rawValue]) => { + const values = Array.isArray(rawValue) ? rawValue : [rawValue ?? '']; + + return values.map((value) => ({ name, value })); + }); try { await client.send('Fetch.continueRequest', { @@ -353,13 +340,27 @@ export class HeadlessChromiumDriver { ); } - private _shouldUseCustomHeaders(conditions: ConditionalHeadersConditions, url: string) { - const { hostname, protocol, port, pathname } = parseUrl(url); + private _shouldUseCustomHeaders(sourceUrl: string, targetUrl: string) { + const { + hostname: sourceHostname, + protocol: sourceProtocol, + port: sourcePort, + } = parseUrl(sourceUrl); + const { + hostname: targetHostname, + protocol: targetProtocol, + port: targetPort, + pathname: targetPathname, + } = parseUrl(targetUrl); + + if (targetPathname === null) { + throw new Error(`URL missing pathname: ${targetUrl}`); + } // `port` is null in URLs that don't explicitly state it, // however we can derive the port from the protocol (http/https) // IE: https://feeds-staging.elastic.co/kibana/v8.0.0.json - const derivedPort = (() => { + const derivedPort = (protocol: string | null, port: string | null, url: string) => { if (port) { return port; } @@ -372,36 +373,15 @@ export class HeadlessChromiumDriver { return '443'; } - return null; - })(); - - if (derivedPort === null) throw new Error(`URL missing port: ${url}`); - if (pathname === null) throw new Error(`URL missing pathname: ${url}`); + throw new Error(`URL missing port: ${url}`); + }; return ( - hostname === conditions.hostname && - protocol === `${conditions.protocol}:` && - this._shouldUseCustomHeadersForPort(conditions, derivedPort) && - pathname.startsWith(`${conditions.basePath}/`) + sourceHostname === targetHostname && + sourceProtocol === targetProtocol && + derivedPort(sourceProtocol, sourcePort, sourceUrl) === + derivedPort(targetProtocol, targetPort, targetUrl) && + targetPathname.startsWith(`${this.basePath}/`) ); } - - private _shouldUseCustomHeadersForPort( - conditions: ConditionalHeadersConditions, - port: string | undefined - ) { - if (conditions.protocol === 'http' && conditions.port === 80) { - return ( - port === undefined || port === null || port === '' || port === conditions.port.toString() - ); - } - - if (conditions.protocol === 'https' && conditions.port === 443) { - return ( - port === undefined || port === null || port === '' || port === conditions.port.toString() - ); - } - - return port === conditions.port.toString(); - } } diff --git a/x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.test.ts b/x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.test.ts index bf8a1786735eb3..5c9136b2728317 100644 --- a/x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.test.ts +++ b/x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.test.ts @@ -61,7 +61,7 @@ describe('HeadlessChromiumDriverFactory', () => { (puppeteer as jest.Mocked).launch.mockResolvedValue(mockBrowser); - factory = new HeadlessChromiumDriverFactory(screenshotMode, config, logger, path); + factory = new HeadlessChromiumDriverFactory(screenshotMode, config, logger, path, ''); jest.spyOn(factory, 'getBrowserLogger').mockReturnValue(Rx.EMPTY); jest.spyOn(factory, 'getProcessLogger').mockReturnValue(Rx.EMPTY); jest.spyOn(factory, 'getPageExit').mockReturnValue(Rx.EMPTY); diff --git a/x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.ts b/x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.ts index 2f4c59707430ec..4d1e0566f99434 100644 --- a/x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.ts @@ -108,7 +108,8 @@ export class HeadlessChromiumDriverFactory { private screenshotMode: ScreenshotModePluginSetup, private config: ConfigType, private logger: Logger, - private binaryPath: string + private binaryPath: string, + private basePath: string ) { if (this.config.browser.chromium.disableSandbox) { logger.warn(`Enabling the Chromium sandbox provides an additional layer of protection.`); @@ -243,7 +244,12 @@ export class HeadlessChromiumDriverFactory { this.getProcessLogger(browser, logger).subscribe(); // HeadlessChromiumDriver: object to "drive" a browser page - const driver = new HeadlessChromiumDriver(this.screenshotMode, this.config, page); + const driver = new HeadlessChromiumDriver( + this.screenshotMode, + this.config, + this.basePath, + page + ); // Rx.Observable: stream to interrupt page capture const unexpectedExit$ = this.getPageExit(browser, page); diff --git a/x-pack/plugins/screenshotting/server/browsers/chromium/index.ts b/x-pack/plugins/screenshotting/server/browsers/chromium/index.ts index 88572081c54913..bc32e719e0f795 100644 --- a/x-pack/plugins/screenshotting/server/browsers/chromium/index.ts +++ b/x-pack/plugins/screenshotting/server/browsers/chromium/index.ts @@ -9,7 +9,7 @@ export const getChromiumDisconnectedError = () => new Error('Browser was closed unexpectedly! Check the server logs for more info.'); export { HeadlessChromiumDriver } from './driver'; -export type { ConditionalHeaders, Context } from './driver'; +export type { Context } from './driver'; export { DEFAULT_VIEWPORT, HeadlessChromiumDriverFactory } from './driver_factory'; export type { PerformanceMetrics } from './driver_factory'; export { ChromiumArchivePaths } from './paths'; diff --git a/x-pack/plugins/screenshotting/server/browsers/chromium/strip_unsafe_headers.test.ts b/x-pack/plugins/screenshotting/server/browsers/chromium/strip_unsafe_headers.test.ts new file mode 100644 index 00000000000000..38feae0b2c0266 --- /dev/null +++ b/x-pack/plugins/screenshotting/server/browsers/chromium/strip_unsafe_headers.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { stripUnsafeHeaders } from './strip_unsafe_headers'; + +describe('stripUnsafeHeaders', () => { + it.each` + header | value + ${'accept-encoding'} | ${''} + ${'connection'} | ${'upgrade'} + ${'content-length'} | ${''} + ${'content-type'} | ${''} + ${'host'} | ${''} + ${'transfer-encoding'} | ${''} + ${'proxy-connection'} | ${'bananas'} + ${'proxy-authorization'} | ${'some-base64-encoded-thing'} + ${'trailer'} | ${'s are for trucks'} + `('should strip unsafe header "$header"', ({ header, value }) => { + const headers = { [header]: value }; + + expect(stripUnsafeHeaders(headers)).toEqual({}); + }); + + it.each` + header | value + ${'foo'} | ${'bar'} + ${'baz'} | ${'quix'} + `('should keep safe header "$header"', ({ header, value }) => { + const headers = { [header]: value }; + + expect(stripUnsafeHeaders(headers)).toEqual(headers); + }); +}); diff --git a/x-pack/plugins/screenshotting/server/browsers/chromium/strip_unsafe_headers.ts b/x-pack/plugins/screenshotting/server/browsers/chromium/strip_unsafe_headers.ts new file mode 100644 index 00000000000000..7e3f46775ea4e8 --- /dev/null +++ b/x-pack/plugins/screenshotting/server/browsers/chromium/strip_unsafe_headers.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { omitBy } from 'lodash'; +import type { Headers } from 'src/core/server'; + +// @see https://github.com/chromium/chromium/blob/3611052c055897e5ebbc5b73ea295092e0c20141/services/network/public/cpp/header_util_unittest.cc#L50 +// For a list of headers that chromium doesn't like +const UNSAFE_HEADERS = [ + 'accept-encoding', + 'connection', + 'content-length', + 'content-type', + 'host', + 'referer', + // `Transfer-Encoding` is hop-by-hop header that is meaningful + // only for a single transport-level connection, and shouldn't + // be stored by caches or forwarded by proxies. + 'transfer-encoding', + 'trailer', + 'te', + 'upgrade', + 'keep-alive', +]; + +const UNSAFE_HEADERS_PATTERNS = [/^proxy-/i]; + +export function stripUnsafeHeaders(headers: Headers): Headers { + return omitBy( + headers, + (_value, header: string) => + header && + (UNSAFE_HEADERS.includes(header) || + UNSAFE_HEADERS_PATTERNS.some((pattern) => pattern.test(header))) + ); +} diff --git a/x-pack/plugins/screenshotting/server/browsers/index.ts b/x-pack/plugins/screenshotting/server/browsers/index.ts index 51687d74ff31de..15c8b2b2db06bb 100644 --- a/x-pack/plugins/screenshotting/server/browsers/index.ts +++ b/x-pack/plugins/screenshotting/server/browsers/index.ts @@ -7,7 +7,7 @@ export { download } from './download'; export { install } from './install'; -export type { ConditionalHeaders, Context, PerformanceMetrics } from './chromium'; +export type { Context, PerformanceMetrics } from './chromium'; export { getChromiumDisconnectedError, ChromiumArchivePaths, diff --git a/x-pack/plugins/screenshotting/server/plugin.ts b/x-pack/plugins/screenshotting/server/plugin.ts index 138193815debea..e0c5d45f67d754 100755 --- a/x-pack/plugins/screenshotting/server/plugin.ts +++ b/x-pack/plugins/screenshotting/server/plugin.ts @@ -54,7 +54,7 @@ export class ScreenshottingPlugin implements Plugin { const paths = new ChromiumArchivePaths(); @@ -63,8 +63,15 @@ export class ScreenshottingPlugin implements Plugin { this.logger.error('Error in screenshotting setup, it may not function properly.'); diff --git a/x-pack/plugins/screenshotting/server/screenshots/index.test.ts b/x-pack/plugins/screenshotting/server/screenshots/index.test.ts index ff5c910e9cc3ee..b129d875f018d7 100644 --- a/x-pack/plugins/screenshotting/server/screenshots/index.test.ts +++ b/x-pack/plugins/screenshotting/server/screenshots/index.test.ts @@ -37,7 +37,7 @@ describe('Screenshot Observable Pipeline', () => { } as unknown as jest.Mocked; options = { browserTimezone: 'UTC', - conditionalHeaders: {}, + headers: {}, layout: {}, timeouts: { loadDelay: 2000, diff --git a/x-pack/plugins/screenshotting/server/screenshots/index.ts b/x-pack/plugins/screenshotting/server/screenshots/index.ts index e8a90145f77e66..b056e9a5450ed5 100644 --- a/x-pack/plugins/screenshotting/server/screenshots/index.ts +++ b/x-pack/plugins/screenshotting/server/screenshots/index.ts @@ -18,7 +18,7 @@ import { tap, toArray, } from 'rxjs/operators'; -import type { Logger } from 'src/core/server'; +import type { KibanaRequest, Logger } from 'src/core/server'; import { LayoutParams } from '../../common'; import type { ConfigType } from '../config'; import type { HeadlessChromiumDriverFactory, PerformanceMetrics } from '../browsers'; @@ -30,6 +30,11 @@ import { Semaphore } from './semaphore'; export interface ScreenshotOptions extends ScreenshotObservableOptions { layout: LayoutParams; + + /** + * Source Kibana request object from where the headers will be extracted. + */ + request?: KibanaRequest; } export interface ScreenshotResult { @@ -77,6 +82,7 @@ export class Screenshots { browserTimezone, timeouts: { openUrl: openUrlTimeout }, } = options; + const headers = { ...(options.request?.headers ?? {}), ...(options.headers ?? {}) }; return this.browserDriverFactory .createPage( @@ -93,7 +99,10 @@ export class Screenshots { apmCreatePage?.end(); unexpectedExit$.subscribe({ error: () => apmTrans?.end() }); - const screen = new ScreenshotObservableHandler(driver, this.logger, layout, options); + const screen = new ScreenshotObservableHandler(driver, this.logger, layout, { + ...options, + headers, + }); return from(options.urls).pipe( concatMap((url, index) => diff --git a/x-pack/plugins/screenshotting/server/screenshots/observable.test.ts b/x-pack/plugins/screenshotting/server/screenshots/observable.test.ts index 5d5fbbde4e0487..83800a168ddbef 100644 --- a/x-pack/plugins/screenshotting/server/screenshots/observable.test.ts +++ b/x-pack/plugins/screenshotting/server/screenshots/observable.test.ts @@ -8,7 +8,6 @@ import { interval, throwError, of } from 'rxjs'; import { map } from 'rxjs/operators'; import type { Logger } from 'src/core/server'; -import type { ConditionalHeaders } from '../browsers'; import { createMockBrowserDriver } from '../browsers/mock'; import { createMockLayout } from '../layouts/mock'; import { ScreenshotObservableHandler, ScreenshotObservableOptions } from './observable'; @@ -24,10 +23,7 @@ describe('ScreenshotObservableHandler', () => { layout = createMockLayout(); logger = { error: jest.fn() } as unknown as jest.Mocked; options = { - conditionalHeaders: { - headers: { testHeader: 'testHeadValue' }, - conditions: {} as unknown as ConditionalHeaders['conditions'], - }, + headers: { testHeader: 'testHeadValue' }, timeouts: { loadDelay: 5000, openUrl: 30000, diff --git a/x-pack/plugins/screenshotting/server/screenshots/observable.ts b/x-pack/plugins/screenshotting/server/screenshots/observable.ts index 5cfda2c841cb8f..a743c206ef98d2 100644 --- a/x-pack/plugins/screenshotting/server/screenshots/observable.ts +++ b/x-pack/plugins/screenshotting/server/screenshots/observable.ts @@ -8,8 +8,8 @@ import type { Transaction } from 'elastic-apm-node'; import { defer, forkJoin, throwError, Observable } from 'rxjs'; import { catchError, mergeMap, switchMapTo, timeoutWith } from 'rxjs/operators'; -import type { Logger } from 'src/core/server'; -import type { ConditionalHeaders, Context, HeadlessChromiumDriver } from '../browsers'; +import type { Headers, Logger } from 'src/core/server'; +import type { Context, HeadlessChromiumDriver } from '../browsers'; import { getChromiumDisconnectedError, DEFAULT_VIEWPORT } from '../browsers'; import type { Layout } from '../layouts'; import type { ElementsPositionAndAttribute } from './get_element_position_data'; @@ -60,7 +60,7 @@ export interface ScreenshotObservableOptions { /** * Custom headers to be sent with each request. */ - conditionalHeaders: ConditionalHeaders; + headers?: Headers; /** * Timeouts for each phase of the screenshot. @@ -177,7 +177,7 @@ export class ScreenshotObservableHandler { index, url, { ...(context ?? {}), layout: this.layout.id }, - this.options.conditionalHeaders + this.options.headers ?? {} ); }).pipe(this.waitUntil(this.options.timeouts.openUrl, 'open URL')); } diff --git a/x-pack/plugins/screenshotting/server/screenshots/open_url.ts b/x-pack/plugins/screenshotting/server/screenshots/open_url.ts index a4c4bd6217205a..8b26f35961db40 100644 --- a/x-pack/plugins/screenshotting/server/screenshots/open_url.ts +++ b/x-pack/plugins/screenshotting/server/screenshots/open_url.ts @@ -6,9 +6,9 @@ */ import apm from 'elastic-apm-node'; -import type { Logger } from 'src/core/server'; +import type { Headers, Logger } from 'src/core/server'; import type { HeadlessChromiumDriver } from '../browsers'; -import type { ConditionalHeaders, Context } from '../browsers'; +import type { Context } from '../browsers'; import { DEFAULT_PAGELOAD_SELECTOR } from './constants'; export const openUrl = async ( @@ -18,7 +18,7 @@ export const openUrl = async ( index: number, url: string, context: Context, - conditionalHeaders: ConditionalHeaders + headers: Headers ): Promise => { // If we're moving to another page in the app, we'll want to wait for the app to tell us // it's loaded the next page. @@ -27,7 +27,7 @@ export const openUrl = async ( const span = apm.startSpan('open_url', 'wait'); try { - await browser.open(url, { conditionalHeaders, context, waitForSelector, timeout }, logger); + await browser.open(url, { context, headers, waitForSelector, timeout }, logger); } catch (err) { logger.error(err); throw new Error(`An error occurred when trying to open the Kibana URL: ${err.message}`); From 7126bc9cc13f8126a5c6bd507c62f7ae9d265d57 Mon Sep 17 00:00:00 2001 From: Jason Rhodes Date: Wed, 16 Feb 2022 12:19:02 -0500 Subject: [PATCH 11/15] [Stack Monitoring] update rules queries to support metricbeat 8.0 [fixed PR] (#125748) * update queries to use metricset.name and fix paths * fix ccr query * fix cluster health query paths * update elasticsearch version mismatch * update fetchLicense paths * use ecs properties in fetch_index_shard_size * Clarified comment about createDatasetFilter * Re-editing a clarifying comment * Small comment edit * Revert "use ecs properties in fetch_index_shard_size" This reverts commit cdf8b2493eece825f3cfbdf206c8ff7cc6ef214e. * simplify fetch_index_shard_size document handling Co-authored-by: neptunian Co-authored-by: klacabane --- .../lib/alerts/create_dataset_query_filter.ts | 25 +++++++- .../alerts/fetch_ccr_read_exceptions.test.ts | 34 ++++++++++- .../lib/alerts/fetch_ccr_read_exceptions.ts | 59 ++++++++++++++----- .../lib/alerts/fetch_cluster_health.test.ts | 3 + .../server/lib/alerts/fetch_cluster_health.ts | 9 ++- .../server/lib/alerts/fetch_clusters.test.ts | 3 + .../server/lib/alerts/fetch_clusters.ts | 55 +++-------------- .../alerts/fetch_cpu_usage_node_stats.test.ts | 1 + .../lib/alerts/fetch_cpu_usage_node_stats.ts | 2 +- .../fetch_disk_usage_node_stats.test.ts | 1 + .../lib/alerts/fetch_disk_usage_node_stats.ts | 2 +- .../fetch_elasticsearch_versions.test.ts | 3 + .../alerts/fetch_elasticsearch_versions.ts | 11 +++- .../lib/alerts/fetch_index_shard_size.test.ts | 3 + .../lib/alerts/fetch_index_shard_size.ts | 10 ++-- .../lib/alerts/fetch_kibana_versions.test.ts | 1 + .../lib/alerts/fetch_kibana_versions.ts | 2 +- .../server/lib/alerts/fetch_licenses.test.ts | 3 + .../server/lib/alerts/fetch_licenses.ts | 9 ++- .../alerts/fetch_logstash_versions.test.ts | 1 + .../lib/alerts/fetch_logstash_versions.ts | 2 +- .../fetch_memory_usage_node_stats.test.ts | 1 + .../alerts/fetch_memory_usage_node_stats.ts | 2 +- .../fetch_missing_monitoring_data.test.ts | 7 ++- .../alerts/fetch_missing_monitoring_data.ts | 9 ++- .../fetch_nodes_from_cluster_stats.test.ts | 5 +- .../alerts/fetch_nodes_from_cluster_stats.ts | 12 ++-- .../fetch_thread_pool_rejections_stats.ts | 21 +++++-- 28 files changed, 193 insertions(+), 103 deletions(-) diff --git a/x-pack/plugins/monitoring/server/lib/alerts/create_dataset_query_filter.ts b/x-pack/plugins/monitoring/server/lib/alerts/create_dataset_query_filter.ts index 7d23240bc1e943..d4d02d4aec349a 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/create_dataset_query_filter.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/create_dataset_query_filter.ts @@ -5,12 +5,33 @@ * 2.0. */ -export const createDatasetFilter = (legacyType: string, dataset: string) => ({ +/** + * We expect that metricset and dataset will be aligned where dataset + * is the full {product}.{metricset}, whereas metricset doesn't include + * the product, e.g. dataset is elasticsearch.cluster_stats and metricset is + * just cluster_stats. + * + * Unfortunately, this doesn't *always* seem to be the case, and sometimes + * the "metricset" value is different. For this reason, we've left these + * two as separate arguments to this function, at least until this is resolved. + * + * More info: https://github.com/elastic/kibana/pull/119112/files#r772605936 + * + * @param {string} type matches legacy data + * @param {string} metricset matches standalone beats + * @param {string} dataset matches agent integration data streams + */ +export const createDatasetFilter = (type: string, metricset: string, dataset: string) => ({ bool: { should: [ { term: { - type: legacyType, + type, + }, + }, + { + term: { + 'metricset.name': metricset, }, }, { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.test.ts index b399f6a876385b..dbfebef6bfe427 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.test.ts @@ -47,15 +47,39 @@ describe('fetchCCReadExceptions', () => { bool: { filter: [ { - nested: { - path: 'ccr_stats.read_exceptions', - query: { exists: { field: 'ccr_stats.read_exceptions.exception' } }, + bool: { + should: [ + { + nested: { + ignore_unmapped: true, + path: 'ccr_stats.read_exceptions', + query: { + exists: { + field: 'ccr_stats.read_exceptions.exception', + }, + }, + }, + }, + { + nested: { + ignore_unmapped: true, + path: 'elasticsearch.ccr.read_exceptions', + query: { + exists: { + field: 'elasticsearch.ccr.read_exceptions.exception', + }, + }, + }, + }, + ], + minimum_should_match: 1, }, }, { bool: { should: [ { term: { type: 'ccr_stats' } }, + { term: { 'metricset.name': 'ccr' } }, { term: { 'data_stream.dataset': 'elasticsearch.ccr' } }, ], minimum_should_match: 1, @@ -82,9 +106,13 @@ describe('fetchCCReadExceptions', () => { _source: { includes: [ 'cluster_uuid', + 'elasticsearch.cluster.id', 'ccr_stats.read_exceptions', + 'elasticsearch.ccr.read_exceptions', 'ccr_stats.shard_id', + 'elasticsearch.ccr.shard_id', 'ccr_stats.leader_index', + 'elasticsearch.ccr.leader.index', ], }, size: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts index cdc136ffd5972c..f6071d84f20d26 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts @@ -35,16 +35,35 @@ export async function fetchCCRReadExceptions( bool: { filter: [ { - nested: { - path: 'ccr_stats.read_exceptions', - query: { - exists: { - field: 'ccr_stats.read_exceptions.exception', + bool: { + should: [ + { + nested: { + ignore_unmapped: true, + path: 'ccr_stats.read_exceptions', + query: { + exists: { + field: 'ccr_stats.read_exceptions.exception', + }, + }, + }, }, - }, + { + nested: { + ignore_unmapped: true, + path: 'elasticsearch.ccr.read_exceptions', + query: { + exists: { + field: 'elasticsearch.ccr.read_exceptions.exception', + }, + }, + }, + }, + ], + minimum_should_match: 1, }, }, - createDatasetFilter('ccr_stats', 'elasticsearch.ccr'), + createDatasetFilter('ccr_stats', 'ccr', 'elasticsearch.ccr'), { range: { timestamp: { @@ -83,9 +102,13 @@ export async function fetchCCRReadExceptions( _source: { includes: [ 'cluster_uuid', + 'elasticsearch.cluster.id', 'ccr_stats.read_exceptions', + 'elasticsearch.ccr.read_exceptions', 'ccr_stats.shard_id', + 'elasticsearch.ccr.shard_id', 'ccr_stats.leader_index', + 'elasticsearch.ccr.leader.index', ], }, size: 1, @@ -123,15 +146,19 @@ export async function fetchCCRReadExceptions( for (const followerIndexBucket of followerIndicesBuckets) { const followerIndex = followerIndexBucket.key; - const { - _index: monitoringIndexName, - _source: { ccr_stats: ccrStats, cluster_uuid: clusterUuid }, - } = get(followerIndexBucket, 'hits.hits.hits[0]'); - const { - read_exceptions: readExceptions, - leader_index: leaderIndex, - shard_id: shardId, - } = ccrStats; + const clusterUuid = + get(followerIndexBucket, 'hits.hits.hits[0]._source.cluster_uuid') || + get(followerIndexBucket, 'hits.hits.hits[0]_source.elasticsearch.cluster.id'); + + const monitoringIndexName = get(followerIndexBucket, 'hits.hits.hits[0]._index'); + const ccrStats = + get(followerIndexBucket, 'hits.hits.hits[0]._source.ccr_stats') || + get(followerIndexBucket, 'hits.hits.hits[0]._source.elasticsearch.ccr'); + + const { read_exceptions: readExceptions, shard_id: shardId } = ccrStats; + + const leaderIndex = ccrStats.leaderIndex || ccrStats.leader.index; + const { exception: lastReadException } = readExceptions[readExceptions.length - 1]; stats.push({ diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.test.ts index 596d95cff93f84..bb2513112af472 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.test.ts @@ -65,7 +65,9 @@ describe('fetchClusterHealth', () => { '*:.monitoring-es-*,.monitoring-es-*,*:metrics-elasticsearch.cluster_stats-*,metrics-elasticsearch.cluster_stats-*', filter_path: [ 'hits.hits._source.cluster_state.status', + 'hits.hits._source.elasticsearch.cluster.stats.status', 'hits.hits._source.cluster_uuid', + 'hits.hits._source.elasticsearch.cluster.id', 'hits.hits._index', ], body: { @@ -79,6 +81,7 @@ describe('fetchClusterHealth', () => { bool: { should: [ { term: { type: 'cluster_stats' } }, + { term: { 'metricset.name': 'cluster_stats' } }, { term: { 'data_stream.dataset': 'elasticsearch.cluster_stats' } }, ], minimum_should_match: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.ts index 7103014cb199e6..5156a865a80e8a 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.ts @@ -27,7 +27,9 @@ export async function fetchClusterHealth( index: indexPatterns, filter_path: [ 'hits.hits._source.cluster_state.status', + 'hits.hits._source.elasticsearch.cluster.stats.status', 'hits.hits._source.cluster_uuid', + 'hits.hits._source.elasticsearch.cluster.id', 'hits.hits._index', ], body: { @@ -48,7 +50,7 @@ export async function fetchClusterHealth( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), + createDatasetFilter('cluster_stats', 'cluster_stats', 'elasticsearch.cluster_stats'), { range: { timestamp: { @@ -77,8 +79,9 @@ export async function fetchClusterHealth( const response = await esClient.search(params); return (response.hits?.hits ?? []).map((hit) => { return { - health: hit._source!.cluster_state?.status, - clusterUuid: hit._source!.cluster_uuid, + health: + hit._source!.cluster_state?.status || hit._source!.elasticsearch?.cluster?.stats?.status, + clusterUuid: hit._source!.cluster_uuid || hit._source!.elasticsearch?.cluster?.id, ccs: hit._index.includes(':') ? hit._index.split(':')[0] : undefined, } as AlertClusterHealth; }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts index 1eeedf3a7a574b..85a027d9cde1d9 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts @@ -87,7 +87,9 @@ describe('fetchClusters', () => { filter_path: [ 'hits.hits._source.cluster_settings.cluster.metadata.display_name', 'hits.hits._source.cluster_uuid', + 'hits.hits._source.elasticsearch.cluster.id', 'hits.hits._source.cluster_name', + 'hits.hits._source.elasticsearch.cluster.name', ], body: { size: 1000, @@ -98,6 +100,7 @@ describe('fetchClusters', () => { bool: { should: [ { term: { type: 'cluster_stats' } }, + { term: { 'metricset.name': 'cluster_stats' } }, { term: { 'data_stream.dataset': 'elasticsearch.cluster_stats' } }, ], minimum_should_match: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts index ac6520eb3948dd..81eb6b724834cb 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts @@ -35,14 +35,16 @@ export async function fetchClusters( filter_path: [ 'hits.hits._source.cluster_settings.cluster.metadata.display_name', 'hits.hits._source.cluster_uuid', + 'hits.hits._source.elasticsearch.cluster.id', 'hits.hits._source.cluster_name', + 'hits.hits._source.elasticsearch.cluster.name', ], body: { size: 1000, query: { bool: { filter: [ - createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), + createDatasetFilter('cluster_stats', 'cluster_stats', 'elasticsearch.cluster_stats'), { range: rangeFilter, }, @@ -56,59 +58,16 @@ export async function fetchClusters( }; const response = await esClient.search(params); - return get(response, 'hits.hits', []).map((hit: any) => { - const clusterName: string = - get(hit, '_source.cluster_settings.cluster.metadata.display_name') || - get(hit, '_source.cluster_name') || - get(hit, '_source.cluster_uuid'); - return { - clusterUuid: get(hit, '_source.cluster_uuid'), - clusterName, - }; - }); -} -export async function fetchClustersLegacy( - callCluster: any, - index: string, - rangeFilter: RangeFilter = { timestamp: { gte: 'now-2m' } } -): Promise { - const params = { - index, - filter_path: [ - 'hits.hits._source.cluster_settings.cluster.metadata.display_name', - 'hits.hits._source.cluster_uuid', - 'hits.hits._source.cluster_name', - ], - body: { - size: 1000, - query: { - bool: { - filter: [ - { - term: { - type: 'cluster_stats', - }, - }, - { - range: rangeFilter, - }, - ], - }, - }, - collapse: { - field: 'cluster_uuid', - }, - }, - }; - const response = await callCluster('search', params); return get(response, 'hits.hits', []).map((hit: any) => { const clusterName: string = get(hit, '_source.cluster_settings.cluster.metadata.display_name') || get(hit, '_source.cluster_name') || - get(hit, '_source.cluster_uuid'); + get(hit, '_source.elasticsearch.cluster.name') || + get(hit, '_source.cluster_uuid') || + get(hit, '_source.elasticsearch.cluster.id'); return { - clusterUuid: get(hit, '_source.cluster_uuid'), + clusterUuid: get(hit, '_source.cluster_uuid') || get(hit, '_source.elasticsearch.cluster.id'), clusterName, }; }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts index b77c2ff686ca83..028cff224afad5 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts @@ -227,6 +227,7 @@ describe('fetchCpuUsageNodeStats', () => { bool: { should: [ { term: { type: 'node_stats' } }, + { term: { 'metricset.name': 'node_stats' } }, { term: { 'data_stream.dataset': 'elasticsearch.node_stats' } }, ], minimum_should_match: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts index aef39d09e26b24..b831db2315e581 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts @@ -58,7 +58,7 @@ export async function fetchCpuUsageNodeStats( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - createDatasetFilter('node_stats', 'elasticsearch.node_stats'), + createDatasetFilter('node_stats', 'node_stats', 'elasticsearch.node_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.test.ts index 9e53c85e743fe4..5126e2d2c408de 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.test.ts @@ -101,6 +101,7 @@ describe('fetchDiskUsageNodeStats', () => { bool: { should: [ { term: { type: 'node_stats' } }, + { term: { 'metricset.name': 'node_stats' } }, { term: { 'data_stream.dataset': 'elasticsearch.node_stats' } }, ], minimum_should_match: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.ts index 6ea9ad51401582..26512b88a08a99 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_disk_usage_node_stats.ts @@ -40,7 +40,7 @@ export async function fetchDiskUsageNodeStats( cluster_uuid: clustersIds, }, }, - createDatasetFilter('node_stats', 'elasticsearch.node_stats'), + createDatasetFilter('node_stats', 'node_stats', 'elasticsearch.node_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.test.ts index 8f34136b70b688..b29dd722366e00 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.test.ts @@ -70,8 +70,10 @@ describe('fetchElasticsearchVersions', () => { '*:.monitoring-es-*,.monitoring-es-*,*:metrics-elasticsearch.cluster_stats-*,metrics-elasticsearch.cluster_stats-*', filter_path: [ 'hits.hits._source.cluster_stats.nodes.versions', + 'hits.hits._source.elasticsearch.cluster.stats.nodes.versions', 'hits.hits._index', 'hits.hits._source.cluster_uuid', + 'hits.hits._source.elasticsearch.cluster.id', ], body: { size: 1, @@ -84,6 +86,7 @@ describe('fetchElasticsearchVersions', () => { bool: { should: [ { term: { type: 'cluster_stats' } }, + { term: { 'metricset.name': 'cluster_stats' } }, { term: { 'data_stream.dataset': 'elasticsearch.cluster_stats' } }, ], minimum_should_match: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.ts index fc5d0cf1cf056b..9816b6facc43f0 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_elasticsearch_versions.ts @@ -28,8 +28,10 @@ export async function fetchElasticsearchVersions( index: indexPatterns, filter_path: [ 'hits.hits._source.cluster_stats.nodes.versions', + 'hits.hits._source.elasticsearch.cluster.stats.nodes.versions', 'hits.hits._index', 'hits.hits._source.cluster_uuid', + 'hits.hits._source.elasticsearch.cluster.id', ], body: { size: clusters.length, @@ -49,7 +51,7 @@ export async function fetchElasticsearchVersions( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), + createDatasetFilter('cluster_stats', 'cluster_stats', 'elasticsearch.cluster_stats'), { range: { timestamp: { @@ -77,10 +79,13 @@ export async function fetchElasticsearchVersions( const response = await esClient.search(params); return (response.hits?.hits ?? []).map((hit) => { - const versions = hit._source!.cluster_stats?.nodes?.versions ?? []; + const versions = + hit._source!.cluster_stats?.nodes?.versions ?? + hit._source!.elasticsearch?.cluster?.stats?.nodes?.versions ?? + []; return { versions, - clusterUuid: hit._source!.cluster_uuid, + clusterUuid: hit._source!.elasticsearch?.cluster?.id || hit._source!.cluster_uuid, ccs: hit._index.includes(':') ? hit._index.split(':')[0] : undefined, }; }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.test.ts index e7020360113e8d..41e84be80d2d50 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.test.ts @@ -161,6 +161,7 @@ describe('fetchIndexShardSize', () => { bool: { should: [ { term: { type: 'index_stats' } }, + { term: { 'metricset.name': 'index' } }, { term: { 'data_stream.dataset': 'elasticsearch.index' } }, ], minimum_should_match: 1, @@ -185,6 +186,8 @@ describe('fetchIndexShardSize', () => { '_index', 'index_stats.shards.primaries', 'index_stats.primaries.store.size_in_bytes', + 'elasticsearch.index.shards.primaries', + 'elasticsearch.index.primaries.store.size_in_bytes', ], }, size: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts index 539b6c20c1e370..5afa9ed007f660 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts @@ -50,7 +50,7 @@ export async function fetchIndexShardSize( query: { bool: { filter: [ - createDatasetFilter('index_stats', 'elasticsearch.index'), + createDatasetFilter('index_stats', 'index', 'elasticsearch.index'), { range: { timestamp: { @@ -90,6 +90,8 @@ export async function fetchIndexShardSize( '_index', 'index_stats.shards.primaries', 'index_stats.primaries.store.size_in_bytes', + 'elasticsearch.index.shards.primaries', + 'elasticsearch.index.primaries.store.size_in_bytes', ], }, size: 1, @@ -131,10 +133,8 @@ export async function fetchIndexShardSize( if (!topHit || !ESGlobPatterns.isValid(shardIndex, validIndexPatterns)) { continue; } - const { - _index: monitoringIndexName, - _source: { index_stats: indexStats }, - } = topHit; + const { _index: monitoringIndexName, _source } = topHit; + const indexStats = _source.index_stats || _source.elasticsearch?.index; if (!indexStats || !indexStats.primaries) { continue; diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.test.ts index 54a7f308fea674..07403388c96f89 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.test.ts @@ -102,6 +102,7 @@ describe('fetchKibanaVersions', () => { bool: { should: [ { term: { type: 'kibana_stats' } }, + { term: { 'metricset.name': 'stats' } }, { term: { 'data_stream.dataset': 'kibana.stats' } }, ], minimum_should_match: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.ts index 886e96ae3331d6..b5ef5d1ade3723 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_kibana_versions.ts @@ -41,7 +41,7 @@ export async function fetchKibanaVersions( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - createDatasetFilter('kibana_stats', 'kibana.stats'), + createDatasetFilter('kibana_stats', 'stats', 'kibana.stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts index 98f52188f0fb68..35fc50a6014b69 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts @@ -84,7 +84,9 @@ describe('fetchLicenses', () => { '*:.monitoring-es-*,.monitoring-es-*,*:metrics-elasticsearch.cluster_stats-*,metrics-elasticsearch.cluster_stats-*', filter_path: [ 'hits.hits._source.license.*', + 'hits.hits._source.elasticsearch.cluster.stats.license.*', 'hits.hits._source.cluster_uuid', + 'hits.hits._source.elasticsearch.cluster.id', 'hits.hits._index', ], body: { @@ -98,6 +100,7 @@ describe('fetchLicenses', () => { bool: { should: [ { term: { type: 'cluster_stats' } }, + { term: { 'metricset.name': 'cluster_stats' } }, { term: { 'data_stream.dataset': 'elasticsearch.cluster_stats' } }, ], minimum_should_match: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts index 10537fdbaaceee..32bbe271da76f1 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts @@ -27,7 +27,9 @@ export async function fetchLicenses( index: indexPatterns, filter_path: [ 'hits.hits._source.license.*', + 'hits.hits._source.elasticsearch.cluster.stats.license.*', 'hits.hits._source.cluster_uuid', + 'hits.hits._source.elasticsearch.cluster.id', 'hits.hits._index', ], body: { @@ -48,7 +50,7 @@ export async function fetchLicenses( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), + createDatasetFilter('cluster_stats', 'cluster_stats', 'elasticsearch.cluster_stats'), { range: { timestamp: { @@ -77,12 +79,13 @@ export async function fetchLicenses( const response = await esClient.search(params); return ( response?.hits?.hits.map((hit) => { - const rawLicense = hit._source!.license ?? {}; + const rawLicense = + hit._source!.license ?? hit._source?.elasticsearch?.cluster?.stats?.license ?? {}; const license: AlertLicense = { status: rawLicense.status ?? '', type: rawLicense.type ?? '', expiryDateMS: rawLicense.expiry_date_in_millis ?? 0, - clusterUuid: hit._source!.cluster_uuid, + clusterUuid: hit._source?.elasticsearch?.cluster?.id || hit._source!.cluster_uuid, ccs: hit._index, }; return license; diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.test.ts index 00f48f1ac27c72..cdb7287ccb77c4 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.test.ts @@ -107,6 +107,7 @@ describe('fetchLogstashVersions', () => { bool: { should: [ { term: { type: 'logstash_stats' } }, + { term: { 'metricset.name': 'node_stats' } }, { term: { 'data_stream.dataset': 'logstash.node_stats' } }, ], minimum_should_match: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.ts index 7f365723267425..2a0a4a283a68de 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_logstash_versions.ts @@ -41,7 +41,7 @@ export async function fetchLogstashVersions( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - createDatasetFilter('logstash_stats', 'logstash.node_stats'), + createDatasetFilter('logstash_stats', 'node_stats', 'logstash.node_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.test.ts index 488818b3167e4b..64fbe542cd35b2 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.test.ts @@ -130,6 +130,7 @@ describe('fetchMemoryUsageNodeStats', () => { bool: { should: [ { term: { type: 'node_stats' } }, + { term: { 'metricset.name': 'node_stats' } }, { term: { 'data_stream.dataset': 'elasticsearch.node_stats' } }, ], minimum_should_match: 1, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts index 01d2b97ebac757..644ef03bd3ec4b 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts @@ -41,7 +41,7 @@ export async function fetchMemoryUsageNodeStats( cluster_uuid: clustersIds, }, }, - createDatasetFilter('node_stats', 'elasticsearch.node_stats'), + createDatasetFilter('node_stats', 'node_stats', 'elasticsearch.node_stats'), { range: { timestamp: { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts index 1d4f00e828c67e..9b1f3f71e2a96b 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts @@ -189,6 +189,7 @@ describe('fetchMissingMonitoringData', () => { bool: { should: [ { term: { type: 'node_stats' } }, + { term: { 'metricset.name': 'node_stats' } }, { term: { 'data_stream.dataset': 'elasticsearch.node_stats' } }, ], minimum_should_match: 1, @@ -210,7 +211,9 @@ describe('fetchMissingMonitoringData', () => { top_hits: { size: 1, sort: [{ timestamp: { order: 'desc', unmapped_type: 'long' } }], - _source: { includes: ['_index', 'source_node.name'] }, + _source: { + includes: ['source_node.name', 'elasticsearch.node.name'], + }, }, }, }, @@ -221,7 +224,7 @@ describe('fetchMissingMonitoringData', () => { }, }); }); - it('should call ES with correct query when ccs disabled', async () => { + it('should call ES with correct query when ccs disabled', async () => { const now = 10; const clusters = [ { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts index f97f0e749f420d..60a5352b8e38f2 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts @@ -74,7 +74,7 @@ export async function fetchMissingMonitoringData( cluster_uuid: clusters.map((cluster) => cluster.clusterUuid), }, }, - createDatasetFilter('node_stats', 'elasticsearch.node_stats'), + createDatasetFilter('node_stats', 'node_stats', 'elasticsearch.node_stats'), { range: { timestamp: { @@ -117,7 +117,7 @@ export async function fetchMissingMonitoringData( }, ], _source: { - includes: ['_index', 'source_node.name'], + includes: ['source_node.name', 'elasticsearch.node.name'], }, }, }, @@ -153,7 +153,10 @@ export async function fetchMissingMonitoringData( const nodeId = uuidBucket.key; const indexName = get(uuidBucket, `document.hits.hits[0]._index`); const differenceInMs = nowInMs - uuidBucket.most_recent.value; - const nodeName = get(uuidBucket, `document.hits.hits[0]._source.source_node.name`, nodeId); + const nodeName = + get(uuidBucket, `document.hits.hits[0]._source.source_node.name`) || + get(uuidBucket, `document.hits.hits[0]._source.elasticsearch.node.name`) || + nodeId; uniqueList[`${clusterUuid}${nodeId}`] = { nodeId, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.test.ts index 82b0c9910976f3..328f8b12836f7f 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.test.ts @@ -172,6 +172,7 @@ describe('fetchNodesFromClusterStats', () => { bool: { should: [ { term: { type: 'cluster_stats' } }, + { term: { 'metricset.name': 'cluster_stats' } }, { term: { 'data_stream.dataset': 'elasticsearch.cluster_stats' } }, ], minimum_should_match: 1, @@ -188,7 +189,9 @@ describe('fetchNodesFromClusterStats', () => { top: { top_hits: { sort: [{ timestamp: { order: 'desc', unmapped_type: 'long' } }], - _source: { includes: ['cluster_state.nodes_hash', 'cluster_state.nodes'] }, + _source: { + includes: ['cluster_state.nodes', 'elasticsearch.cluster.stats.nodes'], + }, size: 2, }, }, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.ts index 40de85226e1f9f..0c367558732b5f 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_nodes_from_cluster_stats.ts @@ -54,7 +54,7 @@ export async function fetchNodesFromClusterStats( query: { bool: { filter: [ - createDatasetFilter('cluster_stats', 'elasticsearch.cluster_stats'), + createDatasetFilter('cluster_stats', 'cluster_stats', 'elasticsearch.cluster_stats'), { range: { timestamp: { @@ -83,7 +83,7 @@ export async function fetchNodesFromClusterStats( }, ], _source: { - includes: ['cluster_state.nodes_hash', 'cluster_state.nodes'], + includes: ['cluster_state.nodes', 'elasticsearch.cluster.stats.nodes'], }, size: 2, }, @@ -116,8 +116,12 @@ export async function fetchNodesFromClusterStats( const indexName = hits[0]._index; nodes.push({ clusterUuid, - recentNodes: formatNode(hits[0]._source.cluster_state?.nodes), - priorNodes: formatNode(hits[1]._source.cluster_state?.nodes), + recentNodes: formatNode( + hits[0]._source.cluster_state?.nodes || hits[0]._source.elasticsearch.cluster.stats.nodes + ), + priorNodes: formatNode( + hits[1]._source.cluster_state?.nodes || hits[1]._source.elasticsearch.cluster.stats.nodes + ), ccs: indexName.includes(':') ? indexName.split(':')[0] : undefined, }); } diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_thread_pool_rejections_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_thread_pool_rejections_stats.ts index 8c3ec15e15407d..8226f0dd012fdc 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_thread_pool_rejections_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_thread_pool_rejections_stats.ts @@ -28,7 +28,12 @@ const getTopHits = (threadType: string, order: 'asc' | 'desc') => ({ }, ], _source: { - includes: [`node_stats.thread_pool.${threadType}.rejected`, 'source_node.name'], + includes: [ + `node_stats.thread_pool.${threadType}.rejected`, + `elasticsearch.node.stats.thread_pool.${threadType}.rejected.count`, + 'source_node.name', + 'elasticsearch.node.name', + ], }, size: 1, }, @@ -62,7 +67,7 @@ export async function fetchThreadPoolRejectionStats( cluster_uuid: clustersIds, }, }, - createDatasetFilter('node_stats', 'elasticsearch.node_stats'), + createDatasetFilter('node_stats', 'node_stats', 'elasticsearch.node_stats'), { range: { timestamp: { @@ -131,8 +136,11 @@ export async function fetchThreadPoolRejectionStats( } const rejectedPath = `_source.node_stats.thread_pool.${threadType}.rejected`; - const newRejectionCount = Number(get(mostRecentDoc, rejectedPath)); - const oldRejectionCount = Number(get(leastRecentDoc, rejectedPath)); + const rejectedPathEcs = `_source.elasticsearch.node.stats.thread_pool.${threadType}.rejected.count`; + const newRejectionCount = + Number(get(mostRecentDoc, rejectedPath)) || Number(get(mostRecentDoc, rejectedPathEcs)); + const oldRejectionCount = + Number(get(leastRecentDoc, rejectedPath)) || Number(get(leastRecentDoc, rejectedPathEcs)); if (invalidNumberValue(newRejectionCount) || invalidNumberValue(oldRejectionCount)) { continue; @@ -143,7 +151,10 @@ export async function fetchThreadPoolRejectionStats( ? newRejectionCount : newRejectionCount - oldRejectionCount; const indexName = mostRecentDoc._index; - const nodeName = get(mostRecentDoc, '_source.source_node.name') || node.key; + const nodeName = + get(mostRecentDoc, '_source.source_node.name') || + get(mostRecentDoc, '_source.elasticsearch.node.name') || + node.key; const nodeStat = { rejectionCount, type: threadType, From 2842ffd432340543d827763110788ab4f0bc4f6a Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 16 Feb 2022 10:31:37 -0700 Subject: [PATCH 12/15] skip flaky suite (#125743) --- x-pack/test/functional_execution_context/tests/browser.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional_execution_context/tests/browser.ts b/x-pack/test/functional_execution_context/tests/browser.ts index ca777e1d4acf32..e2578934cb00b1 100644 --- a/x-pack/test/functional_execution_context/tests/browser.ts +++ b/x-pack/test/functional_execution_context/tests/browser.ts @@ -12,7 +12,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['common', 'dashboard', 'header', 'home']); const retry = getService('retry'); - describe('Browser apps', () => { + // FLAKY: https://github.com/elastic/kibana/issues/125743 + describe.skip('Browser apps', () => { before(async () => { await PageObjects.common.navigateToUrl('home', '/tutorial_directory/sampleData', { useActualUrl: true, From 4137dd8de3c7cfac6c85d7d22fefe032d8d5885a Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 16 Feb 2022 18:33:26 +0100 Subject: [PATCH 13/15] [Discover] Enable document explorer as default document table (#125485) * Enable document explorer in Discover as default document table * Fix lots of functional tests --- .../discover_grid/discover_grid.tsx | 9 ++++++++- .../public/embeddable/saved_search_grid.tsx | 8 +++++++- src/plugins/discover/server/ui_settings.ts | 2 +- test/functional/apps/context/_date_nanos.ts | 1 + .../context/_date_nanos_custom_timestamp.ts | 1 + test/functional/apps/context/_filters.ts | 9 +++++++++ test/functional/apps/context/_size.ts | 1 + .../apps/dashboard/dashboard_time_picker.ts | 4 ++-- .../apps/dashboard/saved_search_embeddable.ts | 2 +- .../apps/discover/_discover_fields_api.ts | 8 +++----- test/functional/page_objects/discover_page.ts | 7 ++++++- .../services/dashboard/expectations.ts | 19 ++++++++++++++----- .../apps/discover/value_suggestions.ts | 5 +++++ .../value_suggestions_non_timebased.ts | 5 +++++ 14 files changed, 64 insertions(+), 17 deletions(-) diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid.tsx b/src/plugins/discover/public/components/discover_grid/discover_grid.tsx index 92ca4ce96bf72c..34673310f2c6e8 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid.tsx +++ b/src/plugins/discover/public/components/discover_grid/discover_grid.tsx @@ -408,7 +408,14 @@ export const DiscoverGrid = ({ if (!rowCount) { return ( -
+
diff --git a/src/plugins/discover/public/embeddable/saved_search_grid.tsx b/src/plugins/discover/public/embeddable/saved_search_grid.tsx index ca750ca6fb9767..ff72ed378aff39 100644 --- a/src/plugins/discover/public/embeddable/saved_search_grid.tsx +++ b/src/plugins/discover/public/embeddable/saved_search_grid.tsx @@ -21,7 +21,13 @@ export function DiscoverGridEmbeddable(props: DiscoverGridEmbeddableProps) { const [expandedDoc, setExpandedDoc] = useState(undefined); return ( - + {props.totalHitCount !== 0 && ( diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index c9c9692e6986b4..afd70cc5bbee7d 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -164,7 +164,7 @@ export const getUiSettings: (docLinks: DocLinksServiceSetup) => Record $(mark).text()); - expect(marks.length).to.above(10); + expect(marks.length).to.above(0); }); it('removing a filter removes highlights', async function () { diff --git a/test/functional/apps/discover/_discover_fields_api.ts b/test/functional/apps/discover/_discover_fields_api.ts index 700c865031cd63..fb3ee3b9858d38 100644 --- a/test/functional/apps/discover/_discover_fields_api.ts +++ b/test/functional/apps/discover/_discover_fields_api.ts @@ -11,7 +11,6 @@ import { FtrProviderContext } from './ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const log = getService('log'); - const docTable = getService('docTable'); const retry = getService('retry'); const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); @@ -19,6 +18,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const defaultSettings = { defaultIndex: 'logstash-*', 'discover:searchFieldsFromSource': false, + 'doc_table:legacy': true, }; describe('discover uses fields API test', function describeIndexTests() { before(async function () { @@ -27,13 +27,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await kibanaServer.uiSettings.replace(defaultSettings); - log.debug('discover'); await PageObjects.common.navigateToApp('discover'); await PageObjects.timePicker.setDefaultAbsoluteRange(); }); after(async () => { - await kibanaServer.uiSettings.replace({ 'discover:searchFieldsFromSource': true }); + await kibanaServer.uiSettings.replace({}); }); it('should correctly display documents', async function () { @@ -61,8 +60,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('displays _source viewer in doc viewer', async function () { - await docTable.clickRowToggle({ rowIndex: 0 }); - + await PageObjects.discover.clickDocTableRowToggle(0); await PageObjects.discover.isShowingDocViewer(); await PageObjects.discover.clickDocViewerTab(1); await PageObjects.discover.expectSourceViewerToExist(); diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index effacb30bdc890..1583903be4991d 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -240,7 +240,7 @@ export class DiscoverPageObject extends FtrService { } public async useLegacyTable() { - return (await this.kibanaServer.uiSettings.get('doc_table:legacy')) !== false; + return (await this.kibanaServer.uiSettings.get('doc_table:legacy')) === true; } public async getDocTableIndex(index: number) { @@ -276,6 +276,11 @@ export class DiscoverPageObject extends FtrService { return result[usedCellIdx]; } + public async clickDocTableRowToggle(rowIndex: number = 0) { + const docTable = await this.getDocTable(); + await docTable.clickRowToggle({ rowIndex }); + } + public async skipToEndOfDocTable() { // add the focus to the button to make it appear const skipButton = await this.testSubjects.find('discoverSkipTableButton'); diff --git a/test/functional/services/dashboard/expectations.ts b/test/functional/services/dashboard/expectations.ts index 75f60b1448eeaa..c56e7c1eae27ef 100644 --- a/test/functional/services/dashboard/expectations.ts +++ b/test/functional/services/dashboard/expectations.ts @@ -226,11 +226,20 @@ export class DashboardExpectService extends FtrService { async savedSearchRowCount(expectedMinCount: number) { this.log.debug(`DashboardExpect.savedSearchRowCount(${expectedMinCount})`); await this.retry.try(async () => { - const savedSearchRows = await this.testSubjects.findAll( - 'docTableExpandToggleColumn', - this.findTimeout - ); - expect(savedSearchRows.length).to.be.above(expectedMinCount); + const gridExists = await this.find.existsByCssSelector('[data-document-number]'); + if (gridExists) { + const grid = await this.find.byCssSelector('[data-document-number]'); + // in this case it's the document explorer + const docNr = Number(await grid.getAttribute('data-document-number')); + expect(docNr).to.be.above(expectedMinCount); + } else { + // in this case it's the classic table + const savedSearchRows = await this.testSubjects.findAll( + 'docTableExpandToggleColumn', + this.findTimeout + ); + expect(savedSearchRows.length).to.be.above(expectedMinCount); + } }); } diff --git a/x-pack/test/functional/apps/discover/value_suggestions.ts b/x-pack/test/functional/apps/discover/value_suggestions.ts index 6cef2722795489..e67d42bf278967 100644 --- a/x-pack/test/functional/apps/discover/value_suggestions.ts +++ b/x-pack/test/functional/apps/discover/value_suggestions.ts @@ -9,6 +9,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; import { UI_SETTINGS } from '../../../../../src/plugins/data/common'; export default function ({ getService, getPageObjects }: FtrProviderContext) { + const kibanaServer = getService('kibanaServer'); const esArchiver = getService('esArchiver'); const queryBar = getService('queryBar'); const filterBar = getService('filterBar'); @@ -28,10 +29,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async function () { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional'); await esArchiver.load('x-pack/test/functional/es_archives/dashboard/drilldowns'); + await kibanaServer.uiSettings.update({ + 'doc_table:legacy': true, + }); }); after(async () => { await esArchiver.unload('x-pack/test/functional/es_archives/dashboard/drilldowns'); + await kibanaServer.uiSettings.unset('doc_table:legacy'); }); describe('useTimeRange enabled', () => { diff --git a/x-pack/test/functional/apps/discover/value_suggestions_non_timebased.ts b/x-pack/test/functional/apps/discover/value_suggestions_non_timebased.ts index b95cbea20cf82e..2a763014c7eb69 100644 --- a/x-pack/test/functional/apps/discover/value_suggestions_non_timebased.ts +++ b/x-pack/test/functional/apps/discover/value_suggestions_non_timebased.ts @@ -11,18 +11,23 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const queryBar = getService('queryBar'); const PageObjects = getPageObjects(['common', 'settings', 'context', 'header']); + const kibanaServer = getService('kibanaServer'); describe('value suggestions non time based', function describeIndexTests() { before(async function () { await esArchiver.loadIfNeeded( 'test/functional/fixtures/es_archiver/index_pattern_without_timefield' ); + await kibanaServer.uiSettings.update({ + 'doc_table:legacy': true, + }); }); after(async () => { await esArchiver.unload( 'test/functional/fixtures/es_archiver/index_pattern_without_timefield' ); + await kibanaServer.uiSettings.unset('doc_table:legacy'); }); it('shows all autosuggest options for a filter in discover context app', async () => { From bcfed9fed352a40c705a9a424d9cc16d5f098925 Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Wed, 16 Feb 2022 12:38:50 -0500 Subject: [PATCH 14/15] [Uptime] implement monitor management telemetry (#124473) * uptime - implement monitor management telemetry * adjust log level * adjust types * add telemetry dependency * remove extra log * update locationsCount * add current channel * adjust types * update location errors * enhance uptime telemetry adapter for synthetics use case * fix test * update xpack_plugins.json * fix tests * fix tests * update snapshots * remove telemetry collectors implementation * update tests and mappings * Update x-pack/plugins/uptime/server/lib/adapters/telemetry/kibana_telemetry_adapter.ts * Update x-pack/plugins/uptime/server/lib/telemetry/sender.ts * move telemetry setup to service conditional Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../monitor_management/config_key.ts | 1 + .../monitor_management/locations.ts | 36 +-- .../monitor_management/monitor_types.ts | 6 +- x-pack/plugins/uptime/kibana.json | 2 +- .../fleet_package/common/formatters.ts | 1 + .../fleet_package/common/normalizers.ts | 1 + .../lib/adapters/framework/adapter_types.ts | 5 + .../synthetics_service/formatters/common.ts | 1 + .../get_service_locations.test.ts | 1 + .../get_service_locations.ts | 5 +- .../server/lib/telemetry/__mocks__/index.ts | 26 +++ .../uptime/server/lib/telemetry/constants.ts | 9 + .../uptime/server/lib/telemetry/queue.test.ts | 39 ++++ .../uptime/server/lib/telemetry/queue.ts | 40 ++++ .../server/lib/telemetry/sender.test.ts | 148 ++++++++++++ .../uptime/server/lib/telemetry/sender.ts | 195 ++++++++++++++++ .../uptime/server/lib/telemetry/types.ts | 32 +++ x-pack/plugins/uptime/server/plugin.ts | 6 + .../synthetics_service/add_monitor.ts | 17 +- .../synthetics_service/delete_monitor.ts | 10 + .../synthetics_service/edit_monitor.ts | 25 +- .../telemetry/monitor_upgrade_sender.test.ts | 215 ++++++++++++++++++ .../telemetry/monitor_upgrade_sender.ts | 147 ++++++++++++ .../apis/uptime/rest/edit_monitor.ts | 2 +- .../uptime/rest/fixtures/http_monitor.json | 3 +- .../apis/uptime/rest/helper/make_checks.ts | 26 +-- .../apis/uptime/rest/helper/make_ping.ts | 5 +- .../uptime/rest/telemetry_collectors_fleet.ts | 10 +- 28 files changed, 960 insertions(+), 54 deletions(-) create mode 100644 x-pack/plugins/uptime/server/lib/telemetry/__mocks__/index.ts create mode 100644 x-pack/plugins/uptime/server/lib/telemetry/constants.ts create mode 100644 x-pack/plugins/uptime/server/lib/telemetry/queue.test.ts create mode 100644 x-pack/plugins/uptime/server/lib/telemetry/queue.ts create mode 100644 x-pack/plugins/uptime/server/lib/telemetry/sender.test.ts create mode 100644 x-pack/plugins/uptime/server/lib/telemetry/sender.ts create mode 100644 x-pack/plugins/uptime/server/lib/telemetry/types.ts create mode 100644 x-pack/plugins/uptime/server/rest_api/synthetics_service/telemetry/monitor_upgrade_sender.test.ts create mode 100644 x-pack/plugins/uptime/server/rest_api/synthetics_service/telemetry/monitor_upgrade_sender.ts diff --git a/x-pack/plugins/uptime/common/runtime_types/monitor_management/config_key.ts b/x-pack/plugins/uptime/common/runtime_types/monitor_management/config_key.ts index f001cbbc10bbd0..595dd1a3e7721a 100644 --- a/x-pack/plugins/uptime/common/runtime_types/monitor_management/config_key.ts +++ b/x-pack/plugins/uptime/common/runtime_types/monitor_management/config_key.ts @@ -37,6 +37,7 @@ export enum ConfigKey { REQUEST_HEADERS_CHECK = 'check.request.headers', REQUEST_METHOD_CHECK = 'check.request.method', REQUEST_SEND_CHECK = 'check.send', + REVISION = 'revision', SCHEDULE = 'schedule', SCREENSHOTS = 'screenshots', SOURCE_INLINE = 'source.inline.script', diff --git a/x-pack/plugins/uptime/common/runtime_types/monitor_management/locations.ts b/x-pack/plugins/uptime/common/runtime_types/monitor_management/locations.ts index 26e3d726a10c0d..56bc00290eecfa 100644 --- a/x-pack/plugins/uptime/common/runtime_types/monitor_management/locations.ts +++ b/x-pack/plugins/uptime/common/runtime_types/monitor_management/locations.ts @@ -30,27 +30,34 @@ export const ServiceLocationCodec = t.interface({ }); export const ServiceLocationErrors = t.array( - t.intersection([ - t.interface({ - locationId: t.string, - error: t.interface({ + t.interface({ + locationId: t.string, + error: t.intersection([ + t.interface({ reason: t.string, status: t.number, }), - }), - t.partial({ - failed_monitors: t.array( - t.interface({ - id: t.string, - message: t.string, - }) - ), - }), - ]) + t.partial({ + failed_monitors: t.array( + t.interface({ + id: t.string, + message: t.string, + }) + ), + }), + ]), + }) ); export const ServiceLocationsCodec = t.array(ServiceLocationCodec); +export const LocationCodec = t.intersection([ + ServiceLocationCodec, + t.partial({ isServiceManaged: t.boolean }), +]); + +export const LocationsCodec = t.array(LocationCodec); + export const isServiceLocationInvalid = (location: ServiceLocation) => isLeft(ServiceLocationCodec.decode(location)); @@ -63,3 +70,4 @@ export type ServiceLocation = t.TypeOf; export type ServiceLocations = t.TypeOf; export type ServiceLocationsApiResponse = t.TypeOf; export type ServiceLocationErrors = t.TypeOf; +export type Locations = t.TypeOf; diff --git a/x-pack/plugins/uptime/common/runtime_types/monitor_management/monitor_types.ts b/x-pack/plugins/uptime/common/runtime_types/monitor_management/monitor_types.ts index 71178528008110..12791edc676a37 100644 --- a/x-pack/plugins/uptime/common/runtime_types/monitor_management/monitor_types.ts +++ b/x-pack/plugins/uptime/common/runtime_types/monitor_management/monitor_types.ts @@ -7,7 +7,7 @@ import * as t from 'io-ts'; import { ConfigKey } from './config_key'; -import { ServiceLocationsCodec } from './locations'; +import { LocationsCodec } from './locations'; import { DataStreamCodec, ModeCodec, @@ -55,13 +55,13 @@ export const CommonFieldsCodec = t.intersection([ [ConfigKey.MONITOR_TYPE]: DataStreamCodec, [ConfigKey.ENABLED]: t.boolean, [ConfigKey.SCHEDULE]: Schedule, - [ConfigKey.LOCATIONS]: t.array(t.string), [ConfigKey.APM_SERVICE_NAME]: t.string, [ConfigKey.TAGS]: t.array(t.string), - [ConfigKey.LOCATIONS]: ServiceLocationsCodec, + [ConfigKey.LOCATIONS]: LocationsCodec, }), t.partial({ [ConfigKey.TIMEOUT]: t.union([t.string, t.null]), + [ConfigKey.REVISION]: t.number, }), ]); diff --git a/x-pack/plugins/uptime/kibana.json b/x-pack/plugins/uptime/kibana.json index 461358c27fe3bd..28a49067b6698a 100644 --- a/x-pack/plugins/uptime/kibana.json +++ b/x-pack/plugins/uptime/kibana.json @@ -2,7 +2,7 @@ "configPath": ["xpack", "uptime"], "id": "uptime", "kibanaVersion": "kibana", - "optionalPlugins": ["cloud", "data", "fleet", "home", "ml"], + "optionalPlugins": ["cloud", "data", "fleet", "home", "ml", "telemetry"], "requiredPlugins": [ "alerting", "cases", diff --git a/x-pack/plugins/uptime/public/components/fleet_package/common/formatters.ts b/x-pack/plugins/uptime/public/components/fleet_package/common/formatters.ts index 72e9cbaa8a03ae..a86894c3d7a480 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/common/formatters.ts +++ b/x-pack/plugins/uptime/public/components/fleet_package/common/formatters.ts @@ -24,6 +24,7 @@ export const commonFormatters: CommonFormatMap = { [ConfigKey.TAGS]: (fields) => arrayToJsonFormatter(fields[ConfigKey.TAGS]), [ConfigKey.TIMEOUT]: (fields) => secondsToCronFormatter(fields[ConfigKey.TIMEOUT] || undefined), [ConfigKey.NAMESPACE]: null, + [ConfigKey.REVISION]: null, }; export const arrayToJsonFormatter = (value: string[] = []) => diff --git a/x-pack/plugins/uptime/public/components/fleet_package/common/normalizers.ts b/x-pack/plugins/uptime/public/components/fleet_package/common/normalizers.ts index a7992b3f935b50..e71a0511cbaf11 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/common/normalizers.ts +++ b/x-pack/plugins/uptime/public/components/fleet_package/common/normalizers.ts @@ -82,4 +82,5 @@ export const commonNormalizers: CommonNormalizerMap = { [ConfigKey.TIMEOUT]: getCommonCronToSecondsNormalizer(ConfigKey.TIMEOUT), [ConfigKey.NAMESPACE]: (fields) => fields?.[ConfigKey.NAMESPACE]?.value ?? DEFAULT_NAMESPACE_STRING, + [ConfigKey.REVISION]: getCommonNormalizer(ConfigKey.REVISION), }; diff --git a/x-pack/plugins/uptime/server/lib/adapters/framework/adapter_types.ts b/x-pack/plugins/uptime/server/lib/adapters/framework/adapter_types.ts index b9dfd61e91b705..9efb7e36ebab9e 100644 --- a/x-pack/plugins/uptime/server/lib/adapters/framework/adapter_types.ts +++ b/x-pack/plugins/uptime/server/lib/adapters/framework/adapter_types.ts @@ -7,6 +7,7 @@ import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import type { SavedObjectsClientContract, IScopedClusterClient, Logger } from 'src/core/server'; +import type { TelemetryPluginSetup, TelemetryPluginStart } from 'src/plugins/telemetry/server'; import { ObservabilityPluginSetup } from '../../../../../observability/server'; import { EncryptedSavedObjectsPluginSetup, @@ -21,6 +22,7 @@ import { PluginSetupContract } from '../../../../../features/server'; import { MlPluginSetup as MlSetup } from '../../../../../ml/server'; import { RuleRegistryPluginSetupContract } from '../../../../../rule_registry/server'; import { UptimeESClient } from '../../lib'; +import type { TelemetryEventsSender } from '../../telemetry/sender'; import type { UptimeRouter } from '../../../types'; import { SecurityPluginStart } from '../../../../../security/server'; import { CloudSetup } from '../../../../../cloud/server'; @@ -52,6 +54,7 @@ export interface UptimeServerSetup { syntheticsService: SyntheticsService; kibanaVersion: string; logger: Logger; + telemetry: TelemetryEventsSender; uptimeEsClient: UptimeESClient; } @@ -65,6 +68,7 @@ export interface UptimeCorePluginsSetup { ruleRegistry: RuleRegistryPluginSetupContract; encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; taskManager: TaskManagerSetupContract; + telemetry: TelemetryPluginSetup; } export interface UptimeCorePluginsStart { @@ -72,6 +76,7 @@ export interface UptimeCorePluginsStart { fleet: FleetStartContract; encryptedSavedObjects: EncryptedSavedObjectsPluginStart; taskManager: TaskManagerStartContract; + telemetry: TelemetryPluginStart; } export interface UMBackendFrameworkAdapter { diff --git a/x-pack/plugins/uptime/server/lib/synthetics_service/formatters/common.ts b/x-pack/plugins/uptime/server/lib/synthetics_service/formatters/common.ts index 73386578618bbd..9aee65e6fa14c8 100644 --- a/x-pack/plugins/uptime/server/lib/synthetics_service/formatters/common.ts +++ b/x-pack/plugins/uptime/server/lib/synthetics_service/formatters/common.ts @@ -25,6 +25,7 @@ export const commonFormatters: CommonFormatMap = { [ConfigKey.TAGS]: (fields) => arrayFormatter(fields[ConfigKey.TAGS]), [ConfigKey.TIMEOUT]: (fields) => secondsToCronFormatter(fields[ConfigKey.TIMEOUT] || undefined), [ConfigKey.NAMESPACE]: null, + [ConfigKey.REVISION]: null, }; export const arrayFormatter = (value: string[] = []) => (value.length ? value : null); diff --git a/x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.test.ts b/x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.test.ts index b2644d53e49d99..1da192ab240587 100644 --- a/x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.test.ts +++ b/x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.test.ts @@ -49,6 +49,7 @@ describe('getServiceLocations', function () { id: 'us_central', label: 'US Central', url: 'https://local.dev', + isServiceManaged: true, }, ], }); diff --git a/x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.ts b/x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.ts index ffec06259f4e7e..e746dafdb55d53 100644 --- a/x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.ts +++ b/x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.ts @@ -8,13 +8,13 @@ import axios from 'axios'; import { ManifestLocation, - ServiceLocations, + Locations, ServiceLocationsApiResponse, } from '../../../common/runtime_types'; import { UptimeServerSetup } from '../adapters/framework'; export async function getServiceLocations(server: UptimeServerSetup) { - const locations: ServiceLocations = []; + const locations: Locations = []; if (!server.config.service?.manifestUrl) { return { locations }; @@ -31,6 +31,7 @@ export async function getServiceLocations(server: UptimeServerSetup) { label: location.geo.name, geo: location.geo.location, url: location.url, + isServiceManaged: true, }); }); diff --git a/x-pack/plugins/uptime/server/lib/telemetry/__mocks__/index.ts b/x-pack/plugins/uptime/server/lib/telemetry/__mocks__/index.ts new file mode 100644 index 00000000000000..2070aeca208617 --- /dev/null +++ b/x-pack/plugins/uptime/server/lib/telemetry/__mocks__/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TelemetryEventsSender } from '../sender'; + +/** + * Creates a mocked Telemetry Events Sender + */ +export const createMockTelemetryEventsSender = ( + enableTelemetry?: boolean +): jest.Mocked => { + return { + setup: jest.fn(), + start: jest.fn(), + stop: jest.fn(), + fetchTelemetryUrl: jest.fn(), + queueTelemetryEvents: jest.fn(), + isTelemetryOptedIn: jest.fn().mockReturnValue(enableTelemetry ?? jest.fn()), + sendIfDue: jest.fn(), + sendEvents: jest.fn(), + } as unknown as jest.Mocked; +}; diff --git a/x-pack/plugins/uptime/server/lib/telemetry/constants.ts b/x-pack/plugins/uptime/server/lib/telemetry/constants.ts new file mode 100644 index 00000000000000..11b5785031149c --- /dev/null +++ b/x-pack/plugins/uptime/server/lib/telemetry/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const MONITOR_UPDATE_CHANNEL = 'synthetics-monitor-update'; +export const MONITOR_CURRENT_CHANNEL = 'synthetics-monitor-current'; diff --git a/x-pack/plugins/uptime/server/lib/telemetry/queue.test.ts b/x-pack/plugins/uptime/server/lib/telemetry/queue.test.ts new file mode 100644 index 00000000000000..510b8983870362 --- /dev/null +++ b/x-pack/plugins/uptime/server/lib/telemetry/queue.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* eslint-disable dot-notation */ +import { TelemetryQueue } from './queue'; + +describe('TelemetryQueue', () => { + describe('queueTelemetryEvents', () => { + it('queues two events', () => { + const queue = new TelemetryQueue(); + queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + expect(queue['queue'].length).toBe(2); + }); + + it('queues more than maxQueueSize events', () => { + const queue = new TelemetryQueue(); + queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + queue['maxQueueSize'] = 5; + queue.addEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); + queue.addEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]); + queue.addEvents([{ 'event.kind': '7' }, { 'event.kind': '8' }]); + expect(queue['queue'].length).toBe(5); + }); + + it('get and clear events', async () => { + const queue = new TelemetryQueue(); + queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + + expect(queue.getEvents().length).toBe(2); + + queue.clearEvents(); + + expect(queue['queue'].length).toBe(0); + }); + }); +}); diff --git a/x-pack/plugins/uptime/server/lib/telemetry/queue.ts b/x-pack/plugins/uptime/server/lib/telemetry/queue.ts new file mode 100644 index 00000000000000..3496cfb94915d3 --- /dev/null +++ b/x-pack/plugins/uptime/server/lib/telemetry/queue.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const TELEMETRY_MAX_QUEUE_SIZE = 100; + +export class TelemetryQueue { + private maxQueueSize = TELEMETRY_MAX_QUEUE_SIZE; + private queue: T[] = []; + + public addEvents(events: T[]) { + const qlength = this.queue.length; + + if (events.length === 0) { + return; + } + + if (qlength >= this.maxQueueSize) { + // we're full already + return; + } + + if (events.length > this.maxQueueSize - qlength) { + this.queue.push(...events.slice(0, this.maxQueueSize - qlength)); + } else { + this.queue.push(...events); + } + } + + public clearEvents() { + this.queue = []; + } + + public getEvents(): T[] { + return this.queue; + } +} diff --git a/x-pack/plugins/uptime/server/lib/telemetry/sender.test.ts b/x-pack/plugins/uptime/server/lib/telemetry/sender.test.ts new file mode 100644 index 00000000000000..91caf31883c511 --- /dev/null +++ b/x-pack/plugins/uptime/server/lib/telemetry/sender.test.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable dot-notation */ + +import { URL } from 'url'; + +import axios from 'axios'; + +import type { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; + +import { loggingSystemMock } from 'src/core/server/mocks'; + +import { MONITOR_UPDATE_CHANNEL } from './constants'; + +import { TelemetryEventsSender } from './sender'; + +jest.mock('axios', () => { + return { + post: jest.fn(), + }; +}); + +describe('TelemetryEventsSender', () => { + let logger: ReturnType; + let sender: TelemetryEventsSender; + const sampleEvent = { + configId: '12345', + stackVersion: '8.1.0', + type: 'http', + locations: ['us_central'], + locationsCount: 1, + monitorNameLength: 8, + monitorInterval: 180000, + revision: 1, + }; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + sender = new TelemetryEventsSender(logger); + sender['fetchClusterInfo'] = jest.fn(async () => { + return { + cluster_uuid: '1', + cluster_name: 'name', + version: { + number: '8.0.0', + }, + } as InfoResponse; + }); + sender.start(undefined, { + elasticsearch: { client: { asInternalUser: { info: jest.fn(async () => ({})) } } }, + } as any); + }); + + describe('queueTelemetryEvents', () => { + it('queues two events', () => { + sender.queueTelemetryEvents(MONITOR_UPDATE_CHANNEL, [sampleEvent]); + expect(sender['queuesPerChannel'][MONITOR_UPDATE_CHANNEL]).toBeDefined(); + }); + + it('should send events when due', async () => { + sender['telemetryStart'] = { + getIsOptedIn: jest.fn(async () => true), + }; + sender['telemetrySetup'] = { + getTelemetryUrl: jest.fn( + async () => new URL('https://telemetry-staging.elastic.co/v3/send/snapshot') + ), + }; + + sender.queueTelemetryEvents(MONITOR_UPDATE_CHANNEL, [sampleEvent]); + sender['sendEvents'] = jest.fn(); + + await sender['sendIfDue'](); + + expect(sender['sendEvents']).toHaveBeenCalledWith( + `https://telemetry-staging.elastic.co/v3-dev/send/${MONITOR_UPDATE_CHANNEL}`, + { cluster_name: 'name', cluster_uuid: '1', version: { number: '8.0.0' } }, + expect.anything() + ); + }); + + it("shouldn't send when telemetry is disabled", async () => { + const telemetryStart = { + getIsOptedIn: jest.fn(async () => false), + }; + sender['telemetryStart'] = telemetryStart; + + sender.queueTelemetryEvents(MONITOR_UPDATE_CHANNEL, [sampleEvent]); + sender['sendEvents'] = jest.fn(); + + await sender['sendIfDue'](); + + expect(sender['sendEvents']).toBeCalledTimes(0); + }); + + it('should send events to separate channels', async () => { + sender['telemetryStart'] = { + getIsOptedIn: jest.fn(async () => true), + }; + sender['telemetrySetup'] = { + getTelemetryUrl: jest.fn( + async () => new URL('https://telemetry.elastic.co/v3/send/snapshot') + ), + }; + + const myChannelEvents = [{ 'event.kind': '1' }, { 'event.kind': '2' }]; + // @ts-ignore + sender.queueTelemetryEvents('my-channel', myChannelEvents); + sender['queuesPerChannel']['my-channel']['getEvents'] = jest.fn(() => myChannelEvents); + + expect(sender['queuesPerChannel']['my-channel']['queue'].length).toBe(2); + + const myChannel2Events = [{ 'event.kind': '3' }]; + // @ts-ignore + sender.queueTelemetryEvents('my-channel2', myChannel2Events); + sender['queuesPerChannel']['my-channel2']['getEvents'] = jest.fn(() => myChannel2Events); + + expect(sender['queuesPerChannel']['my-channel2']['queue'].length).toBe(1); + + await sender['sendIfDue'](); + + expect(sender['queuesPerChannel']['my-channel']['getEvents']).toBeCalledTimes(1); + expect(sender['queuesPerChannel']['my-channel2']['getEvents']).toBeCalledTimes(1); + const headers = { + headers: { + 'Content-Type': 'application/x-ndjson', + 'X-Elastic-Cluster-ID': '1', + 'X-Elastic-Stack-Version': '8.0.0', + }, + }; + expect(axios.post).toHaveBeenCalledWith( + 'https://telemetry.elastic.co/v3/send/my-channel', + '{"event.kind":"1"}\n{"event.kind":"2"}\n', + headers + ); + expect(axios.post).toHaveBeenCalledWith( + 'https://telemetry.elastic.co/v3/send/my-channel2', + '{"event.kind":"3"}\n', + headers + ); + }); + }); +}); diff --git a/x-pack/plugins/uptime/server/lib/telemetry/sender.ts b/x-pack/plugins/uptime/server/lib/telemetry/sender.ts new file mode 100644 index 00000000000000..f1c86b14e50c8b --- /dev/null +++ b/x-pack/plugins/uptime/server/lib/telemetry/sender.ts @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreStart, ElasticsearchClient, Logger } from 'src/core/server'; +import type { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; + +import { cloneDeep } from 'lodash'; + +import axios from 'axios'; + +import type { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; + +import { TelemetryQueue } from './queue'; + +import type { MonitorUpdateTelemetryChannel, MonitorUpdateTelemetryChannelEvents } from './types'; + +/** + * Simplified version of https://github.com/elastic/kibana/blob/master/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts + * Sends batched events to telemetry v3 api + */ +export class TelemetryEventsSender { + private readonly initialCheckDelayMs = 10 * 1000; + private readonly checkIntervalMs = 30 * 1000; + private readonly logger: Logger; + + private telemetryStart?: TelemetryPluginStart; + private telemetrySetup?: TelemetryPluginSetup; + private intervalId?: NodeJS.Timeout; + private isSending = false; + private queuesPerChannel: { [channel: string]: TelemetryQueue } = {}; + private isOptedIn?: boolean = true; // Assume true until the first check + private esClient?: ElasticsearchClient; + private clusterInfo?: InfoResponse; + + constructor(logger: Logger) { + this.logger = logger; + } + + public setup(telemetrySetup?: TelemetryPluginSetup) { + this.telemetrySetup = telemetrySetup; + } + + public async start(telemetryStart?: TelemetryPluginStart, core?: CoreStart) { + this.telemetryStart = telemetryStart; + this.esClient = core?.elasticsearch.client.asInternalUser; + this.clusterInfo = await this.fetchClusterInfo(); + + this.logger.debug(`Starting local task`); + setTimeout(() => { + this.sendIfDue(); + this.intervalId = setInterval(() => this.sendIfDue(), this.checkIntervalMs); + }, this.initialCheckDelayMs); + } + + public stop() { + if (this.intervalId) { + clearInterval(this.intervalId); + } + } + + public queueTelemetryEvents( + channel: T, + events: Array + ) { + if (!this.queuesPerChannel[channel]) { + this.queuesPerChannel[channel] = new TelemetryQueue(); + } + this.queuesPerChannel[channel].addEvents(cloneDeep(events)); + } + + public async isTelemetryOptedIn() { + this.isOptedIn = await this.telemetryStart?.getIsOptedIn(); + return this.isOptedIn === true; + } + + private async sendIfDue() { + if (this.isSending) { + return; + } + + this.isSending = true; + + this.isOptedIn = await this.isTelemetryOptedIn(); + if (!this.isOptedIn) { + this.logger.debug(`Telemetry is not opted-in.`); + for (const channel of Object.keys(this.queuesPerChannel)) { + this.queuesPerChannel[channel].clearEvents(); + } + this.isSending = false; + return; + } + + for (const channel of Object.keys(this.queuesPerChannel)) { + await this.sendEvents( + await this.fetchTelemetryUrl(channel), + this.clusterInfo, + this.queuesPerChannel[channel] + ); + } + + this.isSending = false; + } + + private async fetchClusterInfo(): Promise { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); + } + + return await this.esClient.info(); + } + + public async sendEvents( + telemetryUrl: string, + clusterInfo: InfoResponse | undefined, + queue: TelemetryQueue + ) { + const events = queue.getEvents(); + if (events.length === 0) { + return; + } + + try { + this.logger.debug(`Telemetry URL: ${telemetryUrl}`); + + queue.clearEvents(); + + this.logger.debug(JSON.stringify(events)); + + await this.send( + events, + telemetryUrl, + clusterInfo?.cluster_uuid, + clusterInfo?.version?.number + ); + } catch (err) { + this.logger.debug(`Error sending telemetry events data: ${err}`); + queue.clearEvents(); + } + } + + // Forms URLs like: + // https://telemetry.elastic.co/v3/send/my-channel-name or + // https://telemetry-staging.elastic.co/v3/send/my-channel-name + private async fetchTelemetryUrl(channel: string): Promise { + const telemetryUrl = await this.telemetrySetup?.getTelemetryUrl(); + if (!telemetryUrl) { + throw Error("Couldn't get telemetry URL"); + } + if (!telemetryUrl.hostname.includes('staging')) { + telemetryUrl.pathname = `/v3/send/${channel}`; + } else { + telemetryUrl.pathname = `/v3-dev/send/${channel}`; + } + return telemetryUrl.toString(); + } + + private async send( + events: unknown[], + telemetryUrl: string, + clusterUuid: string | undefined, + clusterVersionNumber: string | undefined + ) { + // using ndjson so that each line will be wrapped in json envelope on server side + // see https://github.com/elastic/infra/blob/master/docs/telemetry/telemetry-next-dataflow.md#json-envelope + const ndjson = this.transformDataToNdjson(events); + + try { + const resp = await axios.post(telemetryUrl, ndjson, { + headers: { + 'Content-Type': 'application/x-ndjson', + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '8.2.0', + }, + }); + this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`); + } catch (err) { + this.logger.debug( + `Error sending events: ${err.response.status} ${JSON.stringify(err.response.data)}` + ); + } + } + + private transformDataToNdjson = (data: unknown[]): string => { + if (data.length !== 0) { + const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n'); + return `${dataString}\n`; + } else { + return ''; + } + }; +} diff --git a/x-pack/plugins/uptime/server/lib/telemetry/types.ts b/x-pack/plugins/uptime/server/lib/telemetry/types.ts new file mode 100644 index 00000000000000..59d060c2f3a76c --- /dev/null +++ b/x-pack/plugins/uptime/server/lib/telemetry/types.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ServiceLocationErrors } from '../../../common/runtime_types/monitor_management'; + +export interface MonitorUpdateEvent { + updatedAt?: string; + lastUpdatedAt?: string; + durationSinceLastUpdated?: number; + deletedAt?: string; + type: string; + stackVersion: string; + monitorNameLength: number; + monitorInterval: number; + locations: string[]; + locationsCount: number; + scriptType?: 'inline' | 'recorder' | 'zip'; + revision?: number; + errors?: ServiceLocationErrors; + configId: string; +} + +export interface MonitorUpdateTelemetryChannelEvents { + // channel name => event type + 'synthetics-monitor-update': MonitorUpdateEvent; + 'synthetics-monitor-current': MonitorUpdateEvent; +} + +export type MonitorUpdateTelemetryChannel = keyof MonitorUpdateTelemetryChannelEvents; diff --git a/x-pack/plugins/uptime/server/plugin.ts b/x-pack/plugins/uptime/server/plugin.ts index 4dc8437cbbcb62..d2afb3f16fb6a5 100644 --- a/x-pack/plugins/uptime/server/plugin.ts +++ b/x-pack/plugins/uptime/server/plugin.ts @@ -21,6 +21,7 @@ import { UptimeCorePluginsStart, UptimeServerSetup, } from './lib/adapters'; +import { TelemetryEventsSender } from './lib/telemetry/sender'; import { registerUptimeSavedObjects, savedObjectsAdapter } from './lib/saved_objects/saved_objects'; import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map'; import { experimentalRuleFieldMap } from '../../rule_registry/common/assets/field_maps/experimental_rule_field_map'; @@ -37,11 +38,13 @@ export class Plugin implements PluginType { private logger: Logger; private server?: UptimeServerSetup; private syntheticService?: SyntheticsService; + private readonly telemetryEventsSender: TelemetryEventsSender; private readonly isServiceEnabled?: boolean; constructor(initializerContext: PluginInitializerContext) { this.initContext = initializerContext; this.logger = initializerContext.logger.get(); + this.telemetryEventsSender = new TelemetryEventsSender(this.logger); const config = this.initContext.config.get(); this.isServiceEnabled = config?.ui?.monitorManagement?.enabled && Boolean(config.service); } @@ -76,6 +79,7 @@ export class Plugin implements PluginType { cloud: plugins.cloud, kibanaVersion: this.initContext.env.packageInfo.version, logger: this.logger, + telemetry: this.telemetryEventsSender, } as UptimeServerSetup; if (this.isServiceEnabled && this.server.config.service) { @@ -86,6 +90,7 @@ export class Plugin implements PluginType { ); this.syntheticService.registerSyncTask(plugins.taskManager); + this.telemetryEventsSender.setup(plugins.telemetry); } initServerWithKibana(this.server, plugins, ruleDataClient, this.logger); @@ -130,6 +135,7 @@ export class Plugin implements PluginType { if (this.server && this.syntheticService) { this.server.syntheticsService = this.syntheticService; } + this.telemetryEventsSender.start(plugins.telemetry, coreStart); } } diff --git a/x-pack/plugins/uptime/server/rest_api/synthetics_service/add_monitor.ts b/x-pack/plugins/uptime/server/rest_api/synthetics_service/add_monitor.ts index 38def648f6e5cd..29c2210efd20a1 100644 --- a/x-pack/plugins/uptime/server/rest_api/synthetics_service/add_monitor.ts +++ b/x-pack/plugins/uptime/server/rest_api/synthetics_service/add_monitor.ts @@ -5,11 +5,13 @@ * 2.0. */ import { schema } from '@kbn/config-schema'; +import { SavedObject } from 'kibana/server'; import { MonitorFields, SyntheticsMonitor } from '../../../common/runtime_types'; import { UMRestApiRouteFactory } from '../types'; import { API_URLS } from '../../../common/constants'; import { syntheticsMonitorType } from '../../lib/saved_objects/synthetics_monitor'; import { validateMonitor } from './monitor_validation'; +import { sendTelemetryEvents, formatTelemetryEvent } from './telemetry/monitor_upgrade_sender'; export const addSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({ method: 'POST', @@ -27,10 +29,11 @@ export const addSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({ return response.badRequest({ body: { message, attributes: { details, ...payload } } }); } - const newMonitor = await savedObjectsClient.create( - syntheticsMonitorType, - monitor - ); + const newMonitor: SavedObject = + await savedObjectsClient.create(syntheticsMonitorType, { + ...monitor, + revision: 1, + }); const { syntheticsService } = server; @@ -45,6 +48,12 @@ export const addSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({ }, ]); + sendTelemetryEvents( + server.logger, + server.telemetry, + formatTelemetryEvent({ monitor: newMonitor, errors, kibanaVersion: server.kibanaVersion }) + ); + if (errors) { return errors; } diff --git a/x-pack/plugins/uptime/server/rest_api/synthetics_service/delete_monitor.ts b/x-pack/plugins/uptime/server/rest_api/synthetics_service/delete_monitor.ts index c1a1ea89456061..bbdb6818b3f6ac 100644 --- a/x-pack/plugins/uptime/server/rest_api/synthetics_service/delete_monitor.ts +++ b/x-pack/plugins/uptime/server/rest_api/synthetics_service/delete_monitor.ts @@ -11,6 +11,10 @@ import { UMRestApiRouteFactory } from '../types'; import { API_URLS } from '../../../common/constants'; import { syntheticsMonitorType } from '../../lib/saved_objects/synthetics_monitor'; import { getMonitorNotFoundResponse } from './service_errors'; +import { + sendTelemetryEvents, + formatTelemetryDeleteEvent, +} from './telemetry/monitor_upgrade_sender'; export const deleteSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({ method: 'DELETE', @@ -36,6 +40,12 @@ export const deleteSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({ { ...monitor.attributes, id: monitorId }, ]); + sendTelemetryEvents( + server.logger, + server.telemetry, + formatTelemetryDeleteEvent(monitor, server.kibanaVersion, new Date().toISOString(), errors) + ); + if (errors) { return errors; } diff --git a/x-pack/plugins/uptime/server/rest_api/synthetics_service/edit_monitor.ts b/x-pack/plugins/uptime/server/rest_api/synthetics_service/edit_monitor.ts index 62a542b2b2037f..d8a6ec85e770ee 100644 --- a/x-pack/plugins/uptime/server/rest_api/synthetics_service/edit_monitor.ts +++ b/x-pack/plugins/uptime/server/rest_api/synthetics_service/edit_monitor.ts @@ -6,14 +6,18 @@ */ import { schema } from '@kbn/config-schema'; -import { SavedObjectsUpdateResponse } from 'kibana/server'; +import { SavedObjectsUpdateResponse, SavedObject } from 'kibana/server'; import { SavedObjectsErrorHelpers } from '../../../../../../src/core/server'; -import { MonitorFields, SyntheticsMonitor } from '../../../common/runtime_types'; +import { MonitorFields, SyntheticsMonitor, ConfigKey } from '../../../common/runtime_types'; import { UMRestApiRouteFactory } from '../types'; import { API_URLS } from '../../../common/constants'; import { syntheticsMonitorType } from '../../lib/saved_objects/synthetics_monitor'; import { validateMonitor } from './monitor_validation'; import { getMonitorNotFoundResponse } from './service_errors'; +import { + sendTelemetryEvents, + formatTelemetryUpdateEvent, +} from './telemetry/monitor_upgrade_sender'; // Simplify return promise type and type it with runtime_types export const editSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({ @@ -40,11 +44,20 @@ export const editSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({ const { syntheticsService } = server; try { + const previousMonitor: SavedObject = await savedObjectsClient.get( + syntheticsMonitorType, + monitorId + ); + const monitorWithRevision = { + ...monitor, + revision: (previousMonitor.attributes[ConfigKey.REVISION] || 0) + 1, + }; + const editMonitor: SavedObjectsUpdateResponse = await savedObjectsClient.update( syntheticsMonitorType, monitorId, - monitor.type === 'browser' ? { ...monitor, urls: '' } : monitor + monitor.type === 'browser' ? { ...monitorWithRevision, urls: '' } : monitorWithRevision ); const errors = await syntheticsService.pushConfigs(request, [ @@ -58,6 +71,12 @@ export const editSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({ }, ]); + sendTelemetryEvents( + server.logger, + server.telemetry, + formatTelemetryUpdateEvent(editMonitor, previousMonitor, server.kibanaVersion, errors) + ); + // Return service sync errors in OK response if (errors) { return errors; diff --git a/x-pack/plugins/uptime/server/rest_api/synthetics_service/telemetry/monitor_upgrade_sender.test.ts b/x-pack/plugins/uptime/server/rest_api/synthetics_service/telemetry/monitor_upgrade_sender.test.ts new file mode 100644 index 00000000000000..24734c88301f1e --- /dev/null +++ b/x-pack/plugins/uptime/server/rest_api/synthetics_service/telemetry/monitor_upgrade_sender.test.ts @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { sha256 } from 'js-sha256'; +import type { Logger } from 'src/core/server'; +import { loggingSystemMock } from 'src/core/server/mocks'; +import { SavedObject } from 'kibana/server'; +import { + SyntheticsMonitor, + ConfigKey, + DataStream, + ScheduleUnit, +} from '../../../../common/runtime_types/monitor_management'; + +import type { TelemetryEventsSender } from '../../../lib/telemetry/sender'; +import { createMockTelemetryEventsSender } from '../../../lib/telemetry/__mocks__'; + +import { MONITOR_UPDATE_CHANNEL, MONITOR_CURRENT_CHANNEL } from '../../../lib/telemetry/constants'; + +import { + formatTelemetryEvent, + formatTelemetryUpdateEvent, + formatTelemetryDeleteEvent, + sendTelemetryEvents, +} from './monitor_upgrade_sender'; + +const kibanaVersion = '8.2.0'; +const id = '123456'; +const errors = [ + { + locationId: 'us_central', + error: { + reason: 'my reason', + status: 400, + }, + }, +]; +const testConfig: SavedObject = { + updated_at: '2011-10-05T14:48:00.000Z', + id, + attributes: { + [ConfigKey.MONITOR_TYPE]: DataStream.HTTP, + [ConfigKey.LOCATIONS]: [ + { + id: 'us_central', + label: 'US Central', + url: 'testurl.com', + geo: { + lat: 0, + lon: 0, + }, + isServiceManaged: true, + }, + { + id: 'custom', + label: 'Custom US Central', + url: 'testurl.com', + geo: { + lat: 0, + lon: 0, + }, + }, + ], + [ConfigKey.SCHEDULE]: { number: '3', unit: ScheduleUnit.MINUTES }, + [ConfigKey.URLS]: 'https://elastic.co', + [ConfigKey.NAME]: 'Test', + [ConfigKey.REVISION]: 1, + } as SyntheticsMonitor, +} as SavedObject; +const createTestConfig = (extraConfigs: Record, updatedAt?: string) => { + return { + ...testConfig, + updated_at: updatedAt || testConfig.updated_at, + attributes: { + ...testConfig.attributes, + ...extraConfigs, + }, + } as SavedObject; +}; + +describe('monitor upgrade telemetry helpers', () => { + it('formats telemetry events', () => { + const actual = formatTelemetryEvent({ monitor: testConfig, kibanaVersion, errors }); + expect(actual).toEqual({ + stackVersion: kibanaVersion, + configId: sha256.create().update(testConfig.id).hex(), + locations: ['us_central', 'other'], + locationsCount: 2, + monitorNameLength: testConfig.attributes[ConfigKey.NAME].length, + updatedAt: testConfig.updated_at, + type: testConfig.attributes[ConfigKey.MONITOR_TYPE], + scriptType: undefined, + monitorInterval: 180000, + lastUpdatedAt: undefined, + deletedAt: undefined, + errors, + durationSinceLastUpdated: undefined, + revision: 1, + }); + }); + + it.each([ + [ConfigKey.SOURCE_INLINE, 'recorder', true], + [ConfigKey.SOURCE_INLINE, 'inline', false], + [ConfigKey.SOURCE_ZIP_URL, 'zip', false], + ])('handles formatting scriptType for browser monitors', (config, scriptType, isRecorder) => { + const actual = formatTelemetryEvent({ + monitor: createTestConfig({ + [config]: 'test', + [ConfigKey.METADATA]: { + script_source: { + is_generated_script: isRecorder, + }, + }, + }), + kibanaVersion, + errors, + }); + expect(actual).toEqual({ + stackVersion: kibanaVersion, + configId: sha256.create().update(testConfig.id).hex(), + locations: ['us_central', 'other'], + locationsCount: 2, + monitorNameLength: testConfig.attributes[ConfigKey.NAME].length, + updatedAt: testConfig.updated_at, + type: testConfig.attributes[ConfigKey.MONITOR_TYPE], + scriptType, + monitorInterval: 180000, + lastUpdatedAt: undefined, + deletedAt: undefined, + errors, + durationSinceLastUpdated: undefined, + revision: 1, + }); + }); + + it('handles formatting update events', () => { + const actual = formatTelemetryUpdateEvent( + createTestConfig({}, '2011-10-05T16:48:00.000Z'), + testConfig, + kibanaVersion, + errors + ); + expect(actual).toEqual({ + stackVersion: kibanaVersion, + configId: sha256.create().update(testConfig.id).hex(), + locations: ['us_central', 'other'], + locationsCount: 2, + monitorNameLength: testConfig.attributes[ConfigKey.NAME].length, + updatedAt: '2011-10-05T16:48:00.000Z', + type: testConfig.attributes[ConfigKey.MONITOR_TYPE], + scriptType: undefined, + monitorInterval: 180000, + lastUpdatedAt: testConfig.updated_at, + deletedAt: undefined, + errors, + durationSinceLastUpdated: 7200000, + revision: 1, + }); + }); + + it('handles formatting delete events', () => { + const actual = formatTelemetryDeleteEvent( + testConfig, + kibanaVersion, + '2011-10-05T16:48:00.000Z', + errors + ); + expect(actual).toEqual({ + stackVersion: kibanaVersion, + configId: sha256.create().update(testConfig.id).hex(), + locations: ['us_central', 'other'], + locationsCount: 2, + monitorNameLength: testConfig.attributes[ConfigKey.NAME].length, + updatedAt: '2011-10-05T16:48:00.000Z', + type: testConfig.attributes[ConfigKey.MONITOR_TYPE], + scriptType: undefined, + monitorInterval: 180000, + lastUpdatedAt: testConfig.updated_at, + deletedAt: '2011-10-05T16:48:00.000Z', + errors, + durationSinceLastUpdated: 7200000, + revision: 1, + }); + }); +}); + +describe('sendTelemetryEvents', () => { + let eventsTelemetryMock: jest.Mocked; + let loggerMock: jest.Mocked; + + beforeEach(() => { + eventsTelemetryMock = createMockTelemetryEventsSender(); + loggerMock = loggingSystemMock.createLogger(); + }); + + it('should queue telemetry events with generic error', () => { + const event = formatTelemetryEvent({ monitor: testConfig, kibanaVersion, errors }); + sendTelemetryEvents( + loggerMock, + eventsTelemetryMock, + formatTelemetryEvent({ monitor: testConfig, kibanaVersion, errors }) + ); + + expect(eventsTelemetryMock.queueTelemetryEvents).toHaveBeenCalledWith(MONITOR_UPDATE_CHANNEL, [ + event, + ]); + expect(eventsTelemetryMock.queueTelemetryEvents).toHaveBeenCalledWith(MONITOR_CURRENT_CHANNEL, [ + event, + ]); + }); +}); diff --git a/x-pack/plugins/uptime/server/rest_api/synthetics_service/telemetry/monitor_upgrade_sender.ts b/x-pack/plugins/uptime/server/rest_api/synthetics_service/telemetry/monitor_upgrade_sender.ts new file mode 100644 index 00000000000000..d4ac7c85c9586e --- /dev/null +++ b/x-pack/plugins/uptime/server/rest_api/synthetics_service/telemetry/monitor_upgrade_sender.ts @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { sha256 } from 'js-sha256'; +import type { Logger } from 'src/core/server'; +import { SavedObjectsUpdateResponse, SavedObject } from 'kibana/server'; +import { + MonitorFields, + SyntheticsMonitor, + ConfigKey, + ServiceLocationErrors, +} from '../../../../common/runtime_types'; +import type { MonitorUpdateEvent } from '../../../lib/telemetry/types'; + +import { TelemetryEventsSender } from '../../../lib/telemetry/sender'; +import { MONITOR_UPDATE_CHANNEL, MONITOR_CURRENT_CHANNEL } from '../../../lib/telemetry/constants'; + +export interface UpgradeError { + key?: string; + message: string | string[]; +} + +export function sendTelemetryEvents( + logger: Logger, + eventsTelemetry: TelemetryEventsSender | undefined, + updateEvent: MonitorUpdateEvent +) { + if (eventsTelemetry === undefined) { + return; + } + + try { + eventsTelemetry.queueTelemetryEvents(MONITOR_UPDATE_CHANNEL, [updateEvent]); + eventsTelemetry.queueTelemetryEvents(MONITOR_CURRENT_CHANNEL, [updateEvent]); + } catch (exc) { + logger.error(`queing telemetry events failed ${exc}`); + } +} + +export function formatTelemetryEvent({ + monitor, + kibanaVersion, + lastUpdatedAt, + durationSinceLastUpdated, + deletedAt, + errors, +}: { + monitor: SavedObject; + kibanaVersion: string; + lastUpdatedAt?: string; + durationSinceLastUpdated?: number; + deletedAt?: string; + errors?: ServiceLocationErrors; +}) { + const { attributes } = monitor; + + return { + updatedAt: deletedAt || monitor.updated_at, + lastUpdatedAt, + durationSinceLastUpdated, + deletedAt, + type: attributes[ConfigKey.MONITOR_TYPE], + locations: attributes[ConfigKey.LOCATIONS].map((location) => + location.isServiceManaged ? location.id : 'other' + ), // mark self-managed locations as other + locationsCount: attributes[ConfigKey.LOCATIONS].length, + monitorNameLength: attributes[ConfigKey.NAME].length, + monitorInterval: parseInt(attributes[ConfigKey.SCHEDULE].number, 10) * 60 * 1000, + stackVersion: kibanaVersion, + scriptType: getScriptType(attributes as MonitorFields), + errors: + errors && errors?.length + ? errors.map((e) => ({ + locationId: e.locationId, + error: { + // don't expose failed_monitors on error object + status: e.error?.status, + reason: e.error?.reason, + }, + })) + : undefined, + configId: sha256.create().update(monitor.id).hex(), + revision: attributes[ConfigKey.REVISION], + }; +} + +export function formatTelemetryUpdateEvent( + currentMonitor: SavedObjectsUpdateResponse, + previousMonitor: SavedObject, + kibanaVersion: string, + errors?: ServiceLocationErrors +) { + let durationSinceLastUpdated: number = 0; + if (currentMonitor.updated_at && previousMonitor.updated_at) { + durationSinceLastUpdated = + new Date(currentMonitor.updated_at).getTime() - + new Date(previousMonitor.updated_at).getTime(); + } + + return formatTelemetryEvent({ + monitor: currentMonitor as SavedObject, + kibanaVersion, + durationSinceLastUpdated, + lastUpdatedAt: previousMonitor.updated_at, + errors, + }); +} + +export function formatTelemetryDeleteEvent( + previousMonitor: SavedObject, + kibanaVersion: string, + deletedAt: string, + errors?: ServiceLocationErrors +) { + let durationSinceLastUpdated: number = 0; + if (deletedAt && previousMonitor.updated_at) { + durationSinceLastUpdated = + new Date(deletedAt).getTime() - new Date(previousMonitor.updated_at).getTime(); + } + + return formatTelemetryEvent({ + monitor: previousMonitor as SavedObject, + kibanaVersion, + durationSinceLastUpdated, + lastUpdatedAt: previousMonitor.updated_at, + deletedAt, + errors, + }); +} + +function getScriptType(attributes: MonitorFields): 'inline' | 'recorder' | 'zip' | undefined { + if (attributes[ConfigKey.SOURCE_ZIP_URL]) { + return 'zip'; + } else if ( + attributes[ConfigKey.SOURCE_INLINE] && + attributes[ConfigKey.METADATA].script_source?.is_generated_script + ) { + return 'recorder'; + } else if (attributes[ConfigKey.SOURCE_INLINE]) { + return 'inline'; + } + + return undefined; +} diff --git a/x-pack/test/api_integration/apis/uptime/rest/edit_monitor.ts b/x-pack/test/api_integration/apis/uptime/rest/edit_monitor.ts index 13581bcfc3e7f0..c453d28259a2c0 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/edit_monitor.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/edit_monitor.ts @@ -53,7 +53,7 @@ export default function ({ getService }: FtrProviderContext) { [ConfigKey.NAME]: 'Modified name', }; - const modifiedMonitor = { ...savedMonitor, ...updates }; + const modifiedMonitor = { ...savedMonitor, ...updates, revision: 2 }; const editResponse = await supertest .put(API_URLS.SYNTHETICS_MONITORS + '/' + monitorId) diff --git a/x-pack/test/api_integration/apis/uptime/rest/fixtures/http_monitor.json b/x-pack/test/api_integration/apis/uptime/rest/fixtures/http_monitor.json index 01a8ecd409fb31..cd75c25a077988 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/fixtures/http_monitor.json +++ b/x-pack/test/api_integration/apis/uptime/rest/fixtures/http_monitor.json @@ -58,5 +58,6 @@ }, "url": "https://example-url.com" }], - "namespace": "testnamespace" + "namespace": "testnamespace", + "revision": 1 } diff --git a/x-pack/test/api_integration/apis/uptime/rest/helper/make_checks.ts b/x-pack/test/api_integration/apis/uptime/rest/helper/make_checks.ts index b9e913524cb1f5..a6ff54ccc7eb26 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/helper/make_checks.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/helper/make_checks.ts @@ -19,7 +19,7 @@ interface CheckProps { mogrify?: (doc: any) => any; refresh?: boolean; tls?: boolean | TlsProps; - isFleetManaged?: boolean; + customIndex?: string; } const getRandomMonitorId = () => { @@ -33,11 +33,11 @@ export const makeCheck = async ({ mogrify = (d) => d, refresh = true, tls = false, - isFleetManaged = false, + customIndex, }: CheckProps): Promise<{ monitorId: string; docs: any }> => { const cgFields = { monitor: { - check_group: uuid.v4(), + check_group: fields.monitor?.check_group || uuid.v4(), }, }; @@ -52,18 +52,10 @@ export const makeCheck = async ({ ip: `127.0.0.${i}`, }, }); - if (i === numIps - 1) { + if (i === numIps - 1 && fields.summary !== null) { pingFields.summary = summary; } - const doc = await makePing( - es, - monitorId, - pingFields, - mogrify, - false, - tls as any, - isFleetManaged - ); + const doc = await makePing(es, monitorId, pingFields, mogrify, false, tls as any, customIndex); docs.push(doc); // @ts-ignore summary[doc.monitor.status]++; @@ -85,7 +77,7 @@ export const makeChecks = async ( fields: { [key: string]: any } = {}, mogrify: (doc: any) => any = (d) => d, refresh: boolean = true, - isFleetManaged: boolean = false + customIndex?: string ) => { const checks = []; const oldestTime = new Date().getTime() - numChecks * every; @@ -109,7 +101,7 @@ export const makeChecks = async ( fields, mogrify, refresh: false, - isFleetManaged, + customIndex, }); checks.push(docs); } @@ -131,7 +123,7 @@ export const makeChecksWithStatus = async ( status: 'up' | 'down', mogrify: (doc: any) => any = (d) => d, refresh: boolean = true, - isFleetManaged: boolean = false + customIndex?: string ) => { const oppositeStatus = status === 'up' ? 'down' : 'up'; @@ -152,7 +144,7 @@ export const makeChecksWithStatus = async ( return mogrify(d); }, refresh, - isFleetManaged + customIndex ); }; diff --git a/x-pack/test/api_integration/apis/uptime/rest/helper/make_ping.ts b/x-pack/test/api_integration/apis/uptime/rest/helper/make_ping.ts index 29421345393a8e..e663aa36cadc51 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/helper/make_ping.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/helper/make_ping.ts @@ -11,7 +11,6 @@ import type { Client } from '@elastic/elasticsearch'; import { makeTls, TlsProps } from './make_tls'; const DEFAULT_INDEX_NAME = 'heartbeat-8-generated-test'; -const DATA_STREAM_INDEX_NAME = 'synthetics-http-default'; export const makePing = async ( es: Client, @@ -20,7 +19,7 @@ export const makePing = async ( mogrify: (doc: any) => any, refresh: boolean = true, tls: boolean | TlsProps = false, - isFleetManaged: boolean | undefined = false + customIndex?: string ) => { const timestamp = new Date(); const baseDoc: any = { @@ -118,7 +117,7 @@ export const makePing = async ( const doc = mogrify(merge(baseDoc, fields)); await es.index({ - index: isFleetManaged ? DATA_STREAM_INDEX_NAME : DEFAULT_INDEX_NAME, + index: customIndex || DEFAULT_INDEX_NAME, refresh, body: doc, }); diff --git a/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors_fleet.ts b/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors_fleet.ts index 84d6d71a4ab6d1..ff257c152daa9e 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors_fleet.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors_fleet.ts @@ -81,7 +81,7 @@ export default function ({ getService }: FtrProviderContext) { 'up', undefined, undefined, - true + testDataStreamName ); await makeChecksWithStatus( @@ -100,7 +100,7 @@ export default function ({ getService }: FtrProviderContext) { 'down', undefined, undefined, - true + testDataStreamName ); await makeChecksWithStatus( @@ -118,7 +118,7 @@ export default function ({ getService }: FtrProviderContext) { 'down', undefined, undefined, - true + testDataStreamName ); await makeChecksWithStatus( @@ -137,7 +137,7 @@ export default function ({ getService }: FtrProviderContext) { 'down', undefined, undefined, - true + testDataStreamName ); await makeChecksWithStatus( @@ -150,7 +150,7 @@ export default function ({ getService }: FtrProviderContext) { 'down', undefined, undefined, - true + testDataStreamName ); await client.indices.refresh(); }); From 09f8eefe918ccf38b9857a4ccc6681572a89383d Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 16 Feb 2022 12:45:53 -0500 Subject: [PATCH 15/15] [Fleet] Remove unused api_key property from the output model (#125801) --- x-pack/plugins/fleet/common/openapi/bundled.json | 3 --- x-pack/plugins/fleet/common/openapi/bundled.yaml | 2 -- .../fleet/common/openapi/components/schemas/output.yaml | 2 -- x-pack/plugins/fleet/common/types/models/agent_policy.ts | 2 +- x-pack/plugins/fleet/common/types/models/output.ts | 1 - .../__snapshots__/full_agent_policy.test.ts.snap | 6 ------ .../services/agent_policies/full_agent_policy.test.ts | 7 ------- .../server/services/agent_policies/full_agent_policy.ts | 4 +--- x-pack/plugins/fleet/server/types/models/output.ts | 1 - 9 files changed, 2 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index 432e72db05e8c8..bc6fd8bff0da0e 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -4226,9 +4226,6 @@ "ca_trusted_fingerprint": { "type": "string" }, - "api_key": { - "type": "string" - }, "config": { "type": "object" }, diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index 439f56da63e5ed..50a7463029f980 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -2664,8 +2664,6 @@ components: type: string ca_trusted_fingerprint: type: string - api_key: - type: string config: type: object config_yaml: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output.yaml index a91ec2cb14e94d..75823d02af8650 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output.yaml @@ -20,8 +20,6 @@ properties: type: string ca_trusted_fingerprint: type: string - api_key: - type: string config: type: object config_yaml: diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts index d8dec7e64a6afb..6fbb423507c3b0 100644 --- a/x-pack/plugins/fleet/common/types/models/agent_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/agent_policy.ts @@ -75,7 +75,7 @@ export interface FullAgentPolicyOutputPermissions { }; } -export type FullAgentPolicyOutput = Pick & { +export type FullAgentPolicyOutput = Pick & { [key: string]: any; }; diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 2ff50c0fc7bdb7..ba09d41d2874f0 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -18,7 +18,6 @@ export interface NewOutput { hosts?: string[]; ca_sha256?: string; ca_trusted_fingerprint?: string; - api_key?: string; config_yaml?: string; is_preconfigured?: boolean; } diff --git a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index 585eca4919eafb..eccd218b42447a 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -46,7 +46,6 @@ Object { }, "outputs": Object { "data-output-id": Object { - "api_key": undefined, "ca_sha256": undefined, "hosts": Array [ "http://es-data.co:9201", @@ -54,7 +53,6 @@ Object { "type": "elasticsearch", }, "default": Object { - "api_key": undefined, "ca_sha256": undefined, "hosts": Array [ "http://127.0.0.1:9201", @@ -112,7 +110,6 @@ Object { }, "outputs": Object { "default": Object { - "api_key": undefined, "ca_sha256": undefined, "hosts": Array [ "http://127.0.0.1:9201", @@ -120,7 +117,6 @@ Object { "type": "elasticsearch", }, "monitoring-output-id": Object { - "api_key": undefined, "ca_sha256": undefined, "hosts": Array [ "http://es-monitoring.co:9201", @@ -178,7 +174,6 @@ Object { }, "outputs": Object { "data-output-id": Object { - "api_key": undefined, "ca_sha256": undefined, "hosts": Array [ "http://es-data.co:9201", @@ -186,7 +181,6 @@ Object { "type": "elasticsearch", }, "monitoring-output-id": Object { - "api_key": undefined, "ca_sha256": undefined, "hosts": Array [ "http://es-monitoring.co:9201", diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index 1bc19192262489..67be0b6914537b 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -145,7 +145,6 @@ describe('getFullAgentPolicy', () => { type: 'elasticsearch', hosts: ['http://127.0.0.1:9201'], ca_sha256: undefined, - api_key: undefined, }, }, inputs: [], @@ -178,7 +177,6 @@ describe('getFullAgentPolicy', () => { type: 'elasticsearch', hosts: ['http://127.0.0.1:9201'], ca_sha256: undefined, - api_key: undefined, }, }, inputs: [], @@ -213,7 +211,6 @@ describe('getFullAgentPolicy', () => { type: 'elasticsearch', hosts: ['http://127.0.0.1:9201'], ca_sha256: undefined, - api_key: undefined, }, }, inputs: [], @@ -315,12 +312,10 @@ describe('transformOutputToFullPolicyOutput', () => { is_default_monitoring: false, name: 'test output', type: 'elasticsearch', - api_key: 'apikey123', }); expect(policyOutput).toMatchInlineSnapshot(` Object { - "api_key": "apikey123", "ca_sha256": undefined, "hosts": Array [ "http://host.fr", @@ -337,7 +332,6 @@ describe('transformOutputToFullPolicyOutput', () => { is_default_monitoring: false, name: 'test output', type: 'elasticsearch', - api_key: 'apikey123', ca_trusted_fingerprint: 'fingerprint123', config_yaml: ` test: 1234 @@ -347,7 +341,6 @@ ssl.test: 123 expect(policyOutput).toMatchInlineSnapshot(` Object { - "api_key": "apikey123", "ca_sha256": undefined, "hosts": Array [ "http://host.fr", diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index 0bb6b4a33f6a3a..17762ad5ada836 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -171,19 +171,17 @@ export function transformOutputToFullPolicyOutput( standalone = false ): FullAgentPolicyOutput { // eslint-disable-next-line @typescript-eslint/naming-convention - const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint, api_key } = output; + const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint } = output; const configJs = config_yaml ? safeLoad(config_yaml) : {}; const newOutput: FullAgentPolicyOutput = { ...configJs, type, hosts, ca_sha256, - api_key, ...(ca_trusted_fingerprint ? { 'ssl.ca_trusted_fingerprint': ca_trusted_fingerprint } : {}), }; if (standalone) { - delete newOutput.api_key; newOutput.username = '{ES_USERNAME}'; newOutput.password = '{ES_PASSWORD}'; } diff --git a/x-pack/plugins/fleet/server/types/models/output.ts b/x-pack/plugins/fleet/server/types/models/output.ts index 83119657ac2098..7d9232862092ea 100644 --- a/x-pack/plugins/fleet/server/types/models/output.ts +++ b/x-pack/plugins/fleet/server/types/models/output.ts @@ -13,7 +13,6 @@ const OutputBaseSchema = { name: schema.string(), type: schema.oneOf([schema.literal(outputType.Elasticsearch)]), hosts: schema.maybe(schema.arrayOf(schema.string())), - api_key: schema.maybe(schema.string()), config: schema.maybe(schema.recordOf(schema.string(), schema.any())), config_yaml: schema.maybe(schema.string()), };