From daceebb329aadd2e74c77a8ab26339ce1b551e76 Mon Sep 17 00:00:00 2001 From: Sandra Gonzales Date: Tue, 5 May 2020 19:36:47 -0400 Subject: [PATCH] [Ingest] check and create default indices during setup (#64809) * check and create default indices during setup * fix typos * add kibana issue * add date mapping * add removeDocValues to each field to stop apps from creating a new index pattern * update snapshot * add property to test data --- .../ingest_manager/common/constants/epm.ts | 1 + .../ingest_manager/server/constants/index.ts | 1 + .../__snapshots__/install.test.ts.snap | 88 ++++++++++++++----- .../epm/kibana/index_pattern/install.ts | 51 ++++++++++- .../kibana/index_pattern/tests/test_data.ts | 8 ++ .../ingest_manager/server/services/setup.ts | 2 + 6 files changed, 128 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/constants/epm.ts b/x-pack/plugins/ingest_manager/common/constants/epm.ts index 4fb259609493dc..97b5cca3692980 100644 --- a/x-pack/plugins/ingest_manager/common/constants/epm.ts +++ b/x-pack/plugins/ingest_manager/common/constants/epm.ts @@ -7,3 +7,4 @@ export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages'; export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern'; export const DEFAULT_REGISTRY_URL = 'https://epr.elastic.co'; +export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder'; diff --git a/x-pack/plugins/ingest_manager/server/constants/index.ts b/x-pack/plugins/ingest_manager/server/constants/index.ts index 3468c56cc877fa..6e633c04ed8169 100644 --- a/x-pack/plugins/ingest_manager/server/constants/index.ts +++ b/x-pack/plugins/ingest_manager/server/constants/index.ts @@ -9,6 +9,7 @@ export { AGENT_TYPE_TEMPORARY, AGENT_POLLING_THRESHOLD_MS, AGENT_POLLING_INTERVAL, + INDEX_PATTERN_PLACEHOLDER_SUFFIX, // Routes PLUGIN_ID, EPM_API_ROUTES, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap index d063ebe92f9387..029e278b5aa93c 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap @@ -40,7 +40,7 @@ exports[`creating index patterns from yaml fields createIndexPattern function cr { "title": "logs-*", "timeFieldName": "@timestamp", - "fields": "[{\\"name\\":\\"coredns.id\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.allParams\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.query.length\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.query.size\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.query.class\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.query.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.query.type\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.response.code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.response.flags\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.response.size\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.dnssec_ok\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"boolean\\"},{\\"name\\":\\"@timestamp\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"date\\"},{\\"name\\":\\"labels\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"message\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"tags\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.ephemeral_id\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.id\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.type\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.version\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"as.number\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"as.organization.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"nginx.access.remote_ip_list\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.body_sent.bytes\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.user_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.method\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.url\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.http_version\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.response_code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.referrer\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.agent\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.user_agent.device\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.user_agent.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.user_agent.os\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.user_agent.os_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.user_agent.original\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.geoip.continent_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"nginx.access.geoip.country_iso_code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.geoip.location\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.geoip.region_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.geoip.city_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"nginx.access.geoip.region_iso_code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true},{\\"name\\":\\"source.geo.continent_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"country\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"country.keyword\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"country.text\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"type\\":\\"string\\"}]", + "fields": "[{\\"name\\":\\"coredns.id\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.allParams\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.query.length\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.query.size\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.query.class\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.query.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.query.type\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.response.code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.response.flags\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"coredns.response.size\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"coredns.dnssec_ok\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"boolean\\"},{\\"name\\":\\"@timestamp\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"date\\"},{\\"name\\":\\"labels\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"message\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"tags\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.ephemeral_id\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.id\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.type\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"agent.version\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"as.number\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"number\\"},{\\"name\\":\\"as.organization.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"nginx.access.remote_ip_list\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.body_sent.bytes\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.method\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.url\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.http_version\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.response_code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.referrer\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.agent\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.device\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.os\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.os_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.original\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.continent_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"nginx.access.geoip.country_iso_code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.location\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.region_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.city_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.region_iso_code\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"source.geo.continent_name\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"country\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"country.keyword\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"},{\\"name\\":\\"country.text\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"analyzed\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"doc_values\\":true,\\"readFromDocValues\\":true,\\"type\\":\\"string\\"}]", "fieldFormatMap": "{\\"coredns.allParams\\":{\\"id\\":\\"bytes\\",\\"params\\":{\\"pattern\\":\\"patternValQueryWeight\\",\\"inputFormat\\":\\"inputFormatVal,\\",\\"outputFormat\\":\\"outputFormalVal,\\",\\"outputPrecision\\":\\"3,\\",\\"labelTemplate\\":\\"labelTemplateVal,\\",\\"urlTemplate\\":\\"urlTemplateVal,\\"}},\\"coredns.query.length\\":{\\"params\\":{\\"pattern\\":\\"patternValQueryLength\\"}},\\"coredns.query.size\\":{\\"id\\":\\"bytes\\",\\"params\\":{\\"pattern\\":\\"patternValQuerySize\\"}},\\"coredns.response.size\\":{\\"id\\":\\"bytes\\"}}" } `; @@ -57,6 +57,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -68,6 +69,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "number" }, { @@ -79,6 +81,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "number" }, { @@ -90,6 +93,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "number" }, { @@ -101,6 +105,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -112,6 +117,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -123,6 +129,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -134,6 +141,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -145,6 +153,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -156,6 +165,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "number" }, { @@ -167,6 +177,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "boolean" }, { @@ -178,6 +189,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "date" }, { @@ -188,7 +200,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "message", @@ -199,6 +212,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": false, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -210,6 +224,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -221,6 +236,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -232,6 +248,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -243,6 +260,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -254,6 +272,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -265,6 +284,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -276,6 +296,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "number" }, { @@ -287,6 +308,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -297,7 +319,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.body_sent.bytes", @@ -307,7 +330,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.user_name", @@ -317,7 +341,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.method", @@ -327,7 +352,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.url", @@ -337,7 +363,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.http_version", @@ -347,7 +374,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.response_code", @@ -357,7 +385,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.referrer", @@ -367,7 +396,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.agent", @@ -377,7 +407,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.user_agent.device", @@ -387,7 +418,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.user_agent.name", @@ -397,7 +429,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.user_agent.os", @@ -407,7 +440,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.user_agent.os_name", @@ -417,7 +451,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.user_agent.original", @@ -427,7 +462,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.geoip.continent_name", @@ -438,6 +474,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": false, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -448,7 +485,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.geoip.location", @@ -458,7 +496,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.geoip.region_name", @@ -468,7 +507,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.geoip.city_name", @@ -478,7 +518,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "nginx.access.geoip.region_iso_code", @@ -488,7 +529,8 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "analyzed": false, "searchable": true, "aggregatable": true, - "doc_values": true + "doc_values": true, + "readFromDocValues": true }, { "name": "source.geo.continent_name", @@ -499,6 +541,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": false, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -510,6 +553,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -521,6 +565,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": true, "doc_values": true, + "readFromDocValues": true, "type": "string" }, { @@ -532,6 +577,7 @@ exports[`creating index patterns from yaml fields createIndexPatternFields funct "searchable": true, "aggregatable": false, "doc_values": true, + "readFromDocValues": true, "type": "string" } ], diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts index ec657820a22251..f6db5dfe353ea5 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts @@ -5,11 +5,14 @@ */ import { SavedObjectsClientContract } from 'src/core/server'; -import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../../constants'; +import { + INDEX_PATTERN_SAVED_OBJECT_TYPE, + INDEX_PATTERN_PLACEHOLDER_SUFFIX, +} from '../../../../constants'; import * as Registry from '../../registry'; import { loadFieldsFromYaml, Fields, Field } from '../../fields/field'; import { getPackageKeysByStatus } from '../../packages/get'; -import { InstallationStatus, RegistryPackage } from '../../../../types'; +import { InstallationStatus, RegistryPackage, CallESAsCurrentUser } from '../../../../types'; interface FieldFormatMap { [key: string]: FieldFormatMapItem; @@ -63,6 +66,7 @@ export interface IndexPatternField { enabled?: boolean; script?: string; lang?: string; + readFromDocValues: boolean; } export enum IndexPatternType { logs = 'logs', @@ -234,6 +238,7 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa searchable: field.searchable ?? true, aggregatable: field.aggregatable ?? true, doc_values: field.doc_values ?? true, + readFromDocValues: field.doc_values ?? true, }; // if type exists, check if it exists in the map @@ -251,6 +256,7 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa newField.aggregatable = false; newField.analyzed = false; newField.doc_values = field.doc_values ?? false; + newField.readFromDocValues = field.doc_values ?? false; newField.indexed = false; newField.searchable = false; } @@ -262,6 +268,7 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa newField.aggregatable = false; newField.analyzed = false; newField.doc_values = false; + newField.readFromDocValues = false; newField.indexed = false; newField.searchable = false; } @@ -276,6 +283,7 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa newField.script = field.script; newField.lang = 'painless'; newField.doc_values = false; + newField.readFromDocValues = false; } return newField; @@ -357,3 +365,42 @@ const getFieldFormatParams = (field: Field): FieldFormatParams => { if (field.open_link_in_current_tab) params.openLinkInCurrentTab = field.open_link_in_current_tab; return params; }; + +export const ensureDefaultIndices = async (callCluster: CallESAsCurrentUser) => + // create placeholder indices to supress errors in the kibana Dashboards app + // that no matching indices exist https://github.com/elastic/kibana/issues/62343 + Promise.all( + Object.keys(IndexPatternType).map(async indexPattern => { + const defaultIndexPatternName = indexPattern + INDEX_PATTERN_PLACEHOLDER_SUFFIX; + const indexExists = await doesIndexExist(defaultIndexPatternName, callCluster); + if (!indexExists) { + try { + await callCluster('transport.request', { + method: 'PUT', + path: `/${defaultIndexPatternName}`, + body: { + mappings: { + properties: { + '@timestamp': { type: 'date' }, + }, + }, + }, + }); + } catch (putErr) { + throw new Error(`${defaultIndexPatternName} could not be created`); + } + } + }) + ); + +export const doesIndexExist = async (indexName: string, callCluster: CallESAsCurrentUser) => { + try { + await callCluster('transport.request', { + method: 'HEAD', + path: indexName, + }); + return true; + } catch (err) { + return false; + } +}; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/tests/test_data.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/tests/test_data.ts index 13bef1b6ddb2cf..879db91bfacaa5 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/tests/test_data.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/tests/test_data.ts @@ -15,6 +15,7 @@ export const dupeFields: IndexPatternField[] = [ count: 0, indexed: true, doc_values: true, + readFromDocValues: true, scripted: false, analyzed: true, }, @@ -26,6 +27,7 @@ export const dupeFields: IndexPatternField[] = [ count: 0, indexed: true, doc_values: true, + readFromDocValues: true, scripted: false, analyzed: true, }, @@ -37,6 +39,7 @@ export const dupeFields: IndexPatternField[] = [ count: 0, indexed: true, doc_values: true, + readFromDocValues: true, scripted: false, analyzed: true, }, @@ -48,6 +51,7 @@ export const dupeFields: IndexPatternField[] = [ count: 2, indexed: true, doc_values: true, + readFromDocValues: true, scripted: false, analyzed: true, }, @@ -59,6 +63,7 @@ export const dupeFields: IndexPatternField[] = [ count: 0, indexed: true, doc_values: true, + readFromDocValues: true, scripted: false, analyzed: true, }, @@ -70,6 +75,7 @@ export const dupeFields: IndexPatternField[] = [ count: 0, indexed: true, doc_values: true, + readFromDocValues: true, scripted: false, analyzed: true, }, @@ -81,6 +87,7 @@ export const dupeFields: IndexPatternField[] = [ count: 0, indexed: true, doc_values: true, + readFromDocValues: true, scripted: false, analyzed: true, }, @@ -92,6 +99,7 @@ export const dupeFields: IndexPatternField[] = [ count: 1, indexed: true, doc_values: true, + readFromDocValues: true, scripted: false, analyzed: false, }, diff --git a/x-pack/plugins/ingest_manager/server/services/setup.ts b/x-pack/plugins/ingest_manager/server/services/setup.ts index 14837d9edd1ccb..2861458fde9d79 100644 --- a/x-pack/plugins/ingest_manager/server/services/setup.ts +++ b/x-pack/plugins/ingest_manager/server/services/setup.ts @@ -11,6 +11,7 @@ import { CallESAsCurrentUser } from '../types'; import { agentConfigService } from './agent_config'; import { outputService } from './output'; import { ensureInstalledDefaultPackages } from './epm/packages/install'; +import { ensureDefaultIndices } from './epm/kibana/index_pattern/install'; import { packageToConfigDatasource, Datasource, @@ -38,6 +39,7 @@ export async function setupIngestManager( ensureInstalledDefaultPackages(soClient, callCluster), outputService.ensureDefaultOutput(soClient), agentConfigService.ensureDefaultAgentConfig(soClient), + ensureDefaultIndices(callCluster), settingsService.getSettings(soClient).catch((e: any) => { if (e.isBoom && e.output.statusCode === 404) { const http = appContextService.getHttpSetup();