Skip to content

Commit

Permalink
[8.10] [Fleet] fix index template from datastream name (#166941) (#16…
Browse files Browse the repository at this point in the history 
…7008)

# Backport

This will backport the following commits from `main` to `8.10`:
- [[Fleet] fix index template from datastream name
(#166941)](#166941)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Julia
Bardi","email":"90178898+juliaElastic@users.noreply.github.com"},"sourceCommit":{"committedDate":"2023-09-22T08:20:05Z","message":"[Fleet]
fix index template from datastream name (#166941)\n\n##
Summary\r\n\r\nResolve
#164269 context why
I picked this up
now:\r\nhttps://github.com//issues/162772#issuecomment-1728031080\r\n\r\nTo
verify:\r\n- Make sure 8.8+ apm package is installed\r\n- Create data
stream `PUT _data_stream/metrics-apm.app.default-default`\r\n- Reinstall
apm package from API or UI\r\n- Check kibana info logs, expect to not
see simulate template error and\r\nrollover like
below\r\n\r\n```\r\n[2023-09-21T15:54:36.559+02:00][INFO
][plugins.fleet] Mappings update for metrics-apm.app.default-default
failed due to ResponseError: illegal_argument_exception\r\n Root
causes:\r\n illegal_argument_exception: unable to simulate template
[metrics-apm.app.default] that does not
exist\r\n[2023-09-21T15:54:36.559+02:00][INFO ][plugins.fleet]
Triggering a rollover for
metrics-apm.app.default-default\r\n```\r\n\r\n\r\n### Checklist\r\n\r\n-
[x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"3ff82f2c17e532fed5d5544ed9bbae4f6e7331af","branchLabelMapping":{"^v8.11.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","Team:Fleet","backport:prev-minor","v8.11.0"],"number":166941,"url":"#166941
fix index template from datastream name (#166941)\n\n##
Summary\r\n\r\nResolve
#164269 context why
I picked this up
now:\r\nhttps://github.com//issues/162772#issuecomment-1728031080\r\n\r\nTo
verify:\r\n- Make sure 8.8+ apm package is installed\r\n- Create data
stream `PUT _data_stream/metrics-apm.app.default-default`\r\n- Reinstall
apm package from API or UI\r\n- Check kibana info logs, expect to not
see simulate template error and\r\nrollover like
below\r\n\r\n```\r\n[2023-09-21T15:54:36.559+02:00][INFO
][plugins.fleet] Mappings update for metrics-apm.app.default-default
failed due to ResponseError: illegal_argument_exception\r\n Root
causes:\r\n illegal_argument_exception: unable to simulate template
[metrics-apm.app.default] that does not
exist\r\n[2023-09-21T15:54:36.559+02:00][INFO ][plugins.fleet]
Triggering a rollover for
metrics-apm.app.default-default\r\n```\r\n\r\n\r\n### Checklist\r\n\r\n-
[x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"3ff82f2c17e532fed5d5544ed9bbae4f6e7331af"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.11.0","labelRegex":"^v8.11.0$","isSourceBranch":true,"state":"MERGED","url":"#166941
fix index template from datastream name (#166941)\n\n##
Summary\r\n\r\nResolve
#164269 context why
I picked this up
now:\r\nhttps://github.com//issues/162772#issuecomment-1728031080\r\n\r\nTo
verify:\r\n- Make sure 8.8+ apm package is installed\r\n- Create data
stream `PUT _data_stream/metrics-apm.app.default-default`\r\n- Reinstall
apm package from API or UI\r\n- Check kibana info logs, expect to not
see simulate template error and\r\nrollover like
below\r\n\r\n```\r\n[2023-09-21T15:54:36.559+02:00][INFO
][plugins.fleet] Mappings update for metrics-apm.app.default-default
failed due to ResponseError: illegal_argument_exception\r\n Root
causes:\r\n illegal_argument_exception: unable to simulate template
[metrics-apm.app.default] that does not
exist\r\n[2023-09-21T15:54:36.559+02:00][INFO ][plugins.fleet]
Triggering a rollover for
metrics-apm.app.default-default\r\n```\r\n\r\n\r\n### Checklist\r\n\r\n-
[x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"3ff82f2c17e532fed5d5544ed9bbae4f6e7331af"}}]}]
BACKPORT-->

Co-authored-by: Julia Bardi <90178898+juliaElastic@users.noreply.github.com>
  • Loading branch information
@kibanamachine @juliaElastic
kibanamachine and juliaElastic committed Sep 26, 2023
1 parent b60a772 commit 09ee1ee
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 6 deletions.
16 changes: 11 additions & 5 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
View file
Expand Up @@ -534,9 +534,15 @@ export function generateTemplateName(dataStream: RegistryDataStream): string {
/**
* Given a data stream name, return the indexTemplate name
*/
function dataStreamNameToIndexTemplateName(dataStreamName: string): string {
const [type, dataset] = dataStreamName.split('-'); // ignore namespace at the end
return [type, dataset].join('-');
async function getIndexTemplate(
esClient: ElasticsearchClient,
dataStreamName: string
): Promise<string> {
const dataStream = await esClient.indices.getDataStream({
name: dataStreamName,
expand_wildcards: ['open', 'hidden'],
});
return dataStream.data_streams[0].template;
}

export function generateTemplateIndexPattern(dataStream: RegistryDataStream): string {
Expand Down Expand Up @@ -757,9 +763,9 @@ const updateExistingDataStream = async ({
let lifecycle: any;

try {
const simulateResult = await retryTransientEsErrors(() =>
const simulateResult = await retryTransientEsErrors(async () =>
esClient.indices.simulateTemplate({
name: dataStreamNameToIndexTemplateName(dataStreamName),
name: await getIndexTemplate(esClient, dataStreamName),
})
);

Expand Down
57 changes: 56 additions & 1 deletion x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts
View file
Expand Up @@ -23,7 +23,7 @@ export default function (providerContext: FtrProviderContext) {
skipIfNoDockerRegistry(providerContext);
setupFleetAndAgents(providerContext);

after(async () => {
afterEach(async () => {
await deletePackage('apm', '8.8.0');
});

Expand Down Expand Up @@ -88,5 +88,60 @@ export default function (providerContext: FtrProviderContext) {
// datastream rolled over
expect(Object.keys(ds).length).greaterThan(1);
});

it('should not rollover datastreams when successfully updated mappings', async function () {
await supertest
.post(`/api/fleet/epm/packages/apm/8.8.0`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true })
.expect(200);

await es.index({
index: 'metrics-apm.app.default-default',
document: {
'@timestamp': '2023-05-30T07:50:00.000Z',
agent: {
name: 'go',
},
data_stream: {
dataset: 'metrics-apm.app.default',
namespace: 'default',
type: 'metrics',
},
ecs: {
version: '8.8.0-dev',
},
event: {
agent_id_status: 'missing',
ingested: '2023-05-30T07:57:12Z',
},
observer: {
hostname: '047e282994fb',
type: 'apm-server',
version: '8.8.0',
},
},
});

let ds = await es.indices.get({
index: 'metrics-apm.app.default*',
expand_wildcards: ['open', 'hidden'],
});
const indicesBefore = Object.keys(ds).length;

await supertest
.post(`/api/fleet/epm/packages/apm/8.8.0`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true })
.expect(200);

ds = await es.indices.get({
index: 'metrics-apm.app.default*',
expand_wildcards: ['open', 'hidden'],
});
const indicesAfter = Object.keys(ds).length;
// datastream did not roll over
expect(indicesAfter).equal(indicesBefore);
});
});
}

0 comments on commit 09ee1ee

Please sign in to comment.