Skip to content

Commit

Permalink
[Fleet] fix index template from datastream name (elastic#166941)
Browse files Browse the repository at this point in the history
## Summary

Resolve elastic#164269

Some context why I picked this up now:
elastic#162772 (comment)

To verify:
- Make sure 8.8+ apm package is installed
- Create data stream `PUT _data_stream/metrics-apm.app.default-default`
- Reinstall apm package from API or UI
- Check kibana info logs, expect to not see simulate template error and
rollover like below

```
[2023-09-21T15:54:36.559+02:00][INFO ][plugins.fleet] Mappings update for metrics-apm.app.default-default failed due to ResponseError: illegal_argument_exception
        Root causes:
                illegal_argument_exception: unable to simulate template [metrics-apm.app.default] that does not exist
[2023-09-21T15:54:36.559+02:00][INFO ][plugins.fleet] Triggering a rollover for metrics-apm.app.default-default
```


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
  • Loading branch information
juliaElastic committed Sep 22, 2023
1 parent 98d2766 commit 3ff82f2
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 6 deletions.
Expand Up @@ -534,9 +534,15 @@ export function generateTemplateName(dataStream: RegistryDataStream): string {
/**
* Given a data stream name, return the indexTemplate name
*/
function dataStreamNameToIndexTemplateName(dataStreamName: string): string {
const [type, dataset] = dataStreamName.split('-'); // ignore namespace at the end
return [type, dataset].join('-');
async function getIndexTemplate(
esClient: ElasticsearchClient,
dataStreamName: string
): Promise<string> {
const dataStream = await esClient.indices.getDataStream({
name: dataStreamName,
expand_wildcards: ['open', 'hidden'],
});
return dataStream.data_streams[0].template;
}

export function generateTemplateIndexPattern(dataStream: RegistryDataStream): string {
Expand Down Expand Up @@ -757,9 +763,9 @@ const updateExistingDataStream = async ({
let lifecycle: any;

try {
const simulateResult = await retryTransientEsErrors(() =>
const simulateResult = await retryTransientEsErrors(async () =>
esClient.indices.simulateTemplate({
name: dataStreamNameToIndexTemplateName(dataStreamName),
name: await getIndexTemplate(esClient, dataStreamName),
})
);

Expand Down
Expand Up @@ -23,7 +23,7 @@ export default function (providerContext: FtrProviderContext) {
skipIfNoDockerRegistry(providerContext);
setupFleetAndAgents(providerContext);

after(async () => {
afterEach(async () => {
await deletePackage('apm', '8.8.0');
});

Expand Down Expand Up @@ -88,5 +88,60 @@ export default function (providerContext: FtrProviderContext) {
// datastream rolled over
expect(Object.keys(ds).length).greaterThan(1);
});

it('should not rollover datastreams when successfully updated mappings', async function () {
await supertest
.post(`/api/fleet/epm/packages/apm/8.8.0`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true })
.expect(200);

await es.index({
index: 'metrics-apm.app.default-default',
document: {
'@timestamp': '2023-05-30T07:50:00.000Z',
agent: {
name: 'go',
},
data_stream: {
dataset: 'metrics-apm.app.default',
namespace: 'default',
type: 'metrics',
},
ecs: {
version: '8.8.0-dev',
},
event: {
agent_id_status: 'missing',
ingested: '2023-05-30T07:57:12Z',
},
observer: {
hostname: '047e282994fb',
type: 'apm-server',
version: '8.8.0',
},
},
});

let ds = await es.indices.get({
index: 'metrics-apm.app.default*',
expand_wildcards: ['open', 'hidden'],
});
const indicesBefore = Object.keys(ds).length;

await supertest
.post(`/api/fleet/epm/packages/apm/8.8.0`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true })
.expect(200);

ds = await es.indices.get({
index: 'metrics-apm.app.default*',
expand_wildcards: ['open', 'hidden'],
});
const indicesAfter = Object.keys(ds).length;
// datastream did not roll over
expect(indicesAfter).equal(indicesBefore);
});
});
}

0 comments on commit 3ff82f2

Please sign in to comment.