From d3f54b5bfb80f573748b82836b8bf55da33bfc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Fri, 10 Mar 2023 04:41:51 -0500 Subject: [PATCH] [Profiling] Using json to create indices (#153064) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read the ES mappings from JSON files instead of having them hard-coded as JS/TS. We currently need the same mappings in the another repository in JSON format. Using JSON files in both places eases automated comparison to detect diversions. --------- Co-authored-by: Tim Rühsen --- .../setup/steps/get_create_indices_step.ts | 341 +----------------- .../profiling_returnpads_private.json | 49 +++ .../mappings/profiling_sq_executables.json | 34 ++ .../mappings/profiling_sq_leafframes.json | 34 ++ .../steps/mappings/profiling_symbols.json | 93 +++++ .../mappings/profiling_symbols_private.json | 93 +++++ 6 files changed, 313 insertions(+), 331 deletions(-) create mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_returnpads_private.json create mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_executables.json create mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_leafframes.json create mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols.json create mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols_private.json diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts index 11b2798720b9afb..30d5c49a50c491e 100644 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts +++ b/x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts @@ -5,9 +5,13 @@ * 2.0. */ -import { MappingSourceField } from '@elastic/elasticsearch/lib/api/types'; import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception'; +import profilingReturnpadsPrivateMapping from './mappings/profiling_returnpads_private.json'; +import profilingSymbolsPrivateMapping from './mappings/profiling_symbols_private.json'; +import profilingSymbolsMapping from './mappings/profiling_symbols.json'; +import profilingSQLeafframesMapping from './mappings/profiling_sq_leafframes.json'; +import profilingSQExecutablesMapping from './mappings/profiling_sq_executables.json'; const RETURNPADS_PRIVATE_INDEX = 'profiling-returnpads-private'; const SQ_EXECUTABLES_INDEX = 'profiling-sq-executables'; @@ -116,359 +120,34 @@ export function getCreateIndicesStep({ }) .catch(catchResourceAlreadyExistsException); }), - // TODO: read the settings and mappings from the .json files esClient.indices .create({ index: SQ_EXECUTABLES_INDEX, - settings: { - index: { - refresh_interval: '10s', - }, - }, - mappings: { - _source: { - mode: 'synthetic', - } as MappingSourceField, - properties: { - 'ecs.version': { - type: 'keyword', - index: true, - }, - 'Executable.file.id': { - type: 'keyword', - index: false, - }, - 'Time.created': { - type: 'date', - index: true, - }, - 'Symbolization.time.next': { - type: 'date', - index: true, - }, - 'Symbolization.retries': { - type: 'short', - index: true, - }, - }, - }, + ...profilingSQExecutablesMapping, }) .catch(catchResourceAlreadyExistsException), esClient.indices .create({ index: SQ_LEAFFRAMES_INDEX, - settings: { - index: { - refresh_interval: '10s', - }, - }, - mappings: { - _source: { - mode: 'synthetic', - } as MappingSourceField, - properties: { - 'ecs.version': { - type: 'keyword', - index: true, - }, - 'Stacktrace.frame.id': { - type: 'keyword', - index: false, - }, - 'Time.created': { - type: 'date', - index: true, - }, - 'Symbolization.time.next': { - type: 'date', - index: true, - }, - 'Symbolization.retries': { - type: 'short', - index: true, - }, - }, - }, + ...profilingSQLeafframesMapping, }) .catch(catchResourceAlreadyExistsException), esClient.indices .create({ index: SYMBOLS_INDEX, - settings: { - index: { - number_of_shards: '16', - refresh_interval: '10s', - }, - }, - mappings: { - _source: { - enabled: true, - } as MappingSourceField, - properties: { - 'ecs.version': { - type: 'keyword', - index: true, - doc_values: false, - store: false, - }, - 'Symbol.function.name': { - // name of the function - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.file.name': { - // file path - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.call.file.name': { - // (for inlined functions) file path where inline function was called - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.call.line': { - // (for inlined functions) line where inline function was called - type: 'integer', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.function.line': { - // function start line (only available from DWARF). Currently unused. - type: 'integer', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.depth': { - // inline depth - type: 'integer', - index: false, - doc_values: false, - store: false, - }, - // pairs of (32bit PC offset, 32bit line number) followed by 64bit PC range base at the end. - // To find line number for a given PC: find lowest offset such as offsetBase+PC >= offset, then read corresponding line number. - // offsetBase could seemingly be available from exec_pc_range (it's the first value of the pair), but it's not the case. - // Ranges are stored as points, which cannot be retrieve when disabling _source. - // See https://www.elastic.co/guide/en/elasticsearch/reference/current/point.html . - 'Symbol.linetable.base': { - // Linetable: base for offsets (64bit PC range base) - type: 'unsigned_long', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.linetable.length': { - // Linetable: length of range (PC range is [base, base+length)) - type: 'unsigned_long', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.linetable.offsets': { - // Linetable: concatenated offsets (each value is ULEB128encoded) - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.linetable.lines': { - // Linetable: concatenated lines (each value is ULEB128 encoded) - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.file.id': { - // fileID. used for deletion and Symbol.exec.pcrange collision handling on symbolization - type: 'keyword', - index: true, - doc_values: false, - store: false, - }, - 'Symbol.exec.pcrange': { - // PC ranges [begin, end) - type: 'ip_range', - index: true, - doc_values: false, - store: false, - }, - }, - }, + ...profilingSymbolsMapping, }) .catch(catchResourceAlreadyExistsException), esClient.indices .create({ index: SYMBOLS_PRIVATE_INDEX, - settings: { - index: { - number_of_shards: '16', - refresh_interval: '10s', - }, - }, - mappings: { - _source: { - enabled: true, - } as MappingSourceField, - properties: { - 'ecs.version': { - type: 'keyword', - index: true, - doc_values: false, - store: false, - }, - 'Symbol.function.name': { - // name of the function - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.file.name': { - // file path - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.call.file.name': { - // (for inlined functions) file path where inline function was called - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.call.line': { - // (for inlined functions) line where inline function was called - type: 'integer', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.function.line': { - // function start line (only available from DWARF). Currently unused. - type: 'integer', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.depth': { - // inline depth - type: 'integer', - index: false, - doc_values: false, - store: false, - }, - // pairs of (32bit PC offset, 32bit line number) followed by 64bit PC range base at the end. - // To find line number for a given PC: find lowest offset such as offsetBase+PC >= offset, then read corresponding line number. - // offsetBase could seemingly be available from exec_pc_range (it's the first value of the pair), but it's not the case. - // Ranges are stored as points, which cannot be retrieve when disabling _source. - // See https://www.elastic.co/guide/en/elasticsearch/reference/current/point.html . - 'Symbol.linetable.base': { - // Linetable: base for offsets (64bit PC range base) - type: 'unsigned_long', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.linetable.length': { - // Linetable: length of range (PC range is [base, base+length)) - type: 'unsigned_long', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.linetable.offsets': { - // Linetable: concatenated offsets (each value is ULEB128encoded) - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.linetable.lines': { - // Linetable: concatenated lines (each value is ULEB128 encoded) - type: 'keyword', - index: false, - doc_values: false, - store: false, - }, - 'Symbol.file.id': { - // fileID. used for deletion and Symbol.exec.pcrange collision handling on symbolization - type: 'keyword', - index: true, - doc_values: false, - store: false, - }, - 'Symbol.exec.pcrange': { - // PC ranges [begin, end) - type: 'ip_range', - index: true, - doc_values: false, - store: false, - }, - }, - }, + ...profilingSymbolsPrivateMapping, }) .catch(catchResourceAlreadyExistsException), esClient.indices .create({ index: RETURNPADS_PRIVATE_INDEX, - settings: { - index: { - refresh_interval: '10s', - }, - }, - mappings: { - _source: { - enabled: true, - } as MappingSourceField, - properties: { - 'ecs.version': { - type: 'keyword', - index: true, - doc_values: false, - store: false, - }, - 'Symbfile.created': { - // name of the function - type: 'date', - index: false, - doc_values: true, - store: false, - }, - 'Symbfile.file.id': { - // file path - type: 'keyword', - index: true, - doc_values: false, - store: false, - }, - 'Symbfile.part': { - type: 'short', - index: false, - doc_values: false, - store: false, - }, - 'Symbfile.parts': { - type: 'short', - index: false, - doc_values: false, - store: false, - }, - 'Symbfile.data': { - type: 'binary', - doc_values: false, - store: false, - }, - }, - }, + ...profilingReturnpadsPrivateMapping, }) .catch(catchResourceAlreadyExistsException), esClient.indices diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_returnpads_private.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_returnpads_private.json new file mode 100644 index 000000000000000..533b196ff9ce1de --- /dev/null +++ b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_returnpads_private.json @@ -0,0 +1,49 @@ +{ + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbfile.created": { + "type": "date", + "doc_values": false, + "index": true, + "store": false + }, + "Symbfile.file.id": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbfile.part": { + "type": "short", + "index": false, + "doc_values": false, + "store": false + }, + "Symbfile.parts": { + "type": "short", + "index": false, + "doc_values": false, + "store": false + }, + "Symbfile.data": { + "type": "binary", + "doc_values": false, + "store": false + } + } + } +} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_executables.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_executables.json new file mode 100644 index 000000000000000..8e5e9935c6ecb9a --- /dev/null +++ b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_executables.json @@ -0,0 +1,34 @@ +{ + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "Executable.file.id": { + "type": "keyword", + "index": false + }, + "Time.created": { + "type": "date", + "index": true + }, + "Symbolization.time.next": { + "type": "date", + "index": true + }, + "Symbolization.retries": { + "type": "short", + "index": true + } + } + } +} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_leafframes.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_leafframes.json new file mode 100644 index 000000000000000..ed778a1309bc5e7 --- /dev/null +++ b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_leafframes.json @@ -0,0 +1,34 @@ +{ + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "Stacktrace.frame.id": { + "type": "keyword", + "index": false + }, + "Time.created": { + "type": "date", + "index": true + }, + "Symbolization.time.next": { + "type": "date", + "index": true + }, + "Symbolization.retries": { + "type": "short", + "index": true + } + } + } +} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols.json new file mode 100644 index 000000000000000..7736a6202a9ad90 --- /dev/null +++ b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols.json @@ -0,0 +1,93 @@ +{ + "settings": { + "index": { + "number_of_shards": "16", + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.function.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.function.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.depth": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.base": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.length": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.offsets": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.lines": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.id": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.exec.pcrange": { + "type": "ip_range", + "index": true, + "doc_values": false, + "store": false + } + } + } +} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols_private.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols_private.json new file mode 100644 index 000000000000000..7736a6202a9ad90 --- /dev/null +++ b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols_private.json @@ -0,0 +1,93 @@ +{ + "settings": { + "index": { + "number_of_shards": "16", + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.function.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.function.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.depth": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.base": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.length": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.offsets": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.lines": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.id": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.exec.pcrange": { + "type": "ip_range", + "index": true, + "doc_values": false, + "store": false + } + } + } +}