Skip to content

Commit

Permalink
[Profiling] Using json to create indices (#153064)
Browse files Browse the repository at this point in the history 
Read the ES mappings from JSON files instead of having them hard-coded as JS/TS.

We currently need the same mappings in the another repository in JSON format.
Using JSON files in both places eases automated comparison to detect diversions.
---------

Co-authored-by: Tim Rühsen <tim.ruehsen@gmx.de>
  • Loading branch information
@cauemarcondes @rockdaboot @nkhristinin
2 people authored and nkhristinin committed Mar 22, 2023
1 parent 4ffc5f1 commit d3f54b5
Show file tree
Hide file tree
Showing 6 changed files with 313 additions and 331 deletions.
341 changes: 10 additions & 331 deletions x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@
* 2.0.
*/

import { MappingSourceField } from '@elastic/elasticsearch/lib/api/types';
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception';
import profilingReturnpadsPrivateMapping from './mappings/profiling_returnpads_private.json';
import profilingSymbolsPrivateMapping from './mappings/profiling_symbols_private.json';
import profilingSymbolsMapping from './mappings/profiling_symbols.json';
import profilingSQLeafframesMapping from './mappings/profiling_sq_leafframes.json';
import profilingSQExecutablesMapping from './mappings/profiling_sq_executables.json';

const RETURNPADS_PRIVATE_INDEX = 'profiling-returnpads-private';
const SQ_EXECUTABLES_INDEX = 'profiling-sq-executables';
Expand Down Expand Up @@ -116,359 +120,34 @@ export function getCreateIndicesStep({
})
.catch(catchResourceAlreadyExistsException);
}),
// TODO: read the settings and mappings from the .json files
esClient.indices
.create({
index: SQ_EXECUTABLES_INDEX,
settings: {
index: {
refresh_interval: '10s',
},
},
mappings: {
_source: {
mode: 'synthetic',
} as MappingSourceField,
properties: {
'ecs.version': {
type: 'keyword',
index: true,
},
'Executable.file.id': {
type: 'keyword',
index: false,
},
'Time.created': {
type: 'date',
index: true,
},
'Symbolization.time.next': {
type: 'date',
index: true,
},
'Symbolization.retries': {
type: 'short',
index: true,
},
},
},
...profilingSQExecutablesMapping,
})
.catch(catchResourceAlreadyExistsException),
esClient.indices
.create({
index: SQ_LEAFFRAMES_INDEX,
settings: {
index: {
refresh_interval: '10s',
},
},
mappings: {
_source: {
mode: 'synthetic',
} as MappingSourceField,
properties: {
'ecs.version': {
type: 'keyword',
index: true,
},
'Stacktrace.frame.id': {
type: 'keyword',
index: false,
},
'Time.created': {
type: 'date',
index: true,
},
'Symbolization.time.next': {
type: 'date',
index: true,
},
'Symbolization.retries': {
type: 'short',
index: true,
},
},
},
...profilingSQLeafframesMapping,
})
.catch(catchResourceAlreadyExistsException),
esClient.indices
.create({
index: SYMBOLS_INDEX,
settings: {
index: {
number_of_shards: '16',
refresh_interval: '10s',
},
},
mappings: {
_source: {
enabled: true,
} as MappingSourceField,
properties: {
'ecs.version': {
type: 'keyword',
index: true,
doc_values: false,
store: false,
},
'Symbol.function.name': {
// name of the function
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.file.name': {
// file path
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.call.file.name': {
// (for inlined functions) file path where inline function was called
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.call.line': {
// (for inlined functions) line where inline function was called
type: 'integer',
index: false,
doc_values: false,
store: false,
},
'Symbol.function.line': {
// function start line (only available from DWARF). Currently unused.
type: 'integer',
index: false,
doc_values: false,
store: false,
},
'Symbol.depth': {
// inline depth
type: 'integer',
index: false,
doc_values: false,
store: false,
},
// pairs of (32bit PC offset, 32bit line number) followed by 64bit PC range base at the end.
// To find line number for a given PC: find lowest offset such as offsetBase+PC >= offset, then read corresponding line number.
// offsetBase could seemingly be available from exec_pc_range (it's the first value of the pair), but it's not the case.
// Ranges are stored as points, which cannot be retrieve when disabling _source.
// See https://www.elastic.co/guide/en/elasticsearch/reference/current/point.html .
'Symbol.linetable.base': {
// Linetable: base for offsets (64bit PC range base)
type: 'unsigned_long',
index: false,
doc_values: false,
store: false,
},
'Symbol.linetable.length': {
// Linetable: length of range (PC range is [base, base+length))
type: 'unsigned_long',
index: false,
doc_values: false,
store: false,
},
'Symbol.linetable.offsets': {
// Linetable: concatenated offsets (each value is ULEB128encoded)
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.linetable.lines': {
// Linetable: concatenated lines (each value is ULEB128 encoded)
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.file.id': {
// fileID. used for deletion and Symbol.exec.pcrange collision handling on symbolization
type: 'keyword',
index: true,
doc_values: false,
store: false,
},
'Symbol.exec.pcrange': {
// PC ranges [begin, end)
type: 'ip_range',
index: true,
doc_values: false,
store: false,
},
},
},
...profilingSymbolsMapping,
})
.catch(catchResourceAlreadyExistsException),
esClient.indices
.create({
index: SYMBOLS_PRIVATE_INDEX,
settings: {
index: {
number_of_shards: '16',
refresh_interval: '10s',
},
},
mappings: {
_source: {
enabled: true,
} as MappingSourceField,
properties: {
'ecs.version': {
type: 'keyword',
index: true,
doc_values: false,
store: false,
},
'Symbol.function.name': {
// name of the function
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.file.name': {
// file path
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.call.file.name': {
// (for inlined functions) file path where inline function was called
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.call.line': {
// (for inlined functions) line where inline function was called
type: 'integer',
index: false,
doc_values: false,
store: false,
},
'Symbol.function.line': {
// function start line (only available from DWARF). Currently unused.
type: 'integer',
index: false,
doc_values: false,
store: false,
},
'Symbol.depth': {
// inline depth
type: 'integer',
index: false,
doc_values: false,
store: false,
},
// pairs of (32bit PC offset, 32bit line number) followed by 64bit PC range base at the end.
// To find line number for a given PC: find lowest offset such as offsetBase+PC >= offset, then read corresponding line number.
// offsetBase could seemingly be available from exec_pc_range (it's the first value of the pair), but it's not the case.
// Ranges are stored as points, which cannot be retrieve when disabling _source.
// See https://www.elastic.co/guide/en/elasticsearch/reference/current/point.html .
'Symbol.linetable.base': {
// Linetable: base for offsets (64bit PC range base)
type: 'unsigned_long',
index: false,
doc_values: false,
store: false,
},
'Symbol.linetable.length': {
// Linetable: length of range (PC range is [base, base+length))
type: 'unsigned_long',
index: false,
doc_values: false,
store: false,
},
'Symbol.linetable.offsets': {
// Linetable: concatenated offsets (each value is ULEB128encoded)
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.linetable.lines': {
// Linetable: concatenated lines (each value is ULEB128 encoded)
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.file.id': {
// fileID. used for deletion and Symbol.exec.pcrange collision handling on symbolization
type: 'keyword',
index: true,
doc_values: false,
store: false,
},
'Symbol.exec.pcrange': {
// PC ranges [begin, end)
type: 'ip_range',
index: true,
doc_values: false,
store: false,
},
},
},
...profilingSymbolsPrivateMapping,
})
.catch(catchResourceAlreadyExistsException),
esClient.indices
.create({
index: RETURNPADS_PRIVATE_INDEX,
settings: {
index: {
refresh_interval: '10s',
},
},
mappings: {
_source: {
enabled: true,
} as MappingSourceField,
properties: {
'ecs.version': {
type: 'keyword',
index: true,
doc_values: false,
store: false,
},
'Symbfile.created': {
// name of the function
type: 'date',
index: false,
doc_values: true,
store: false,
},
'Symbfile.file.id': {
// file path
type: 'keyword',
index: true,
doc_values: false,
store: false,
},
'Symbfile.part': {
type: 'short',
index: false,
doc_values: false,
store: false,
},
'Symbfile.parts': {
type: 'short',
index: false,
doc_values: false,
store: false,
},
'Symbfile.data': {
type: 'binary',
doc_values: false,
store: false,
},
},
},
...profilingReturnpadsPrivateMapping,
})
.catch(catchResourceAlreadyExistsException),
esClient.indices
Expand Down
Loading

0 comments on commit d3f54b5

Please sign in to comment.