New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[TIP] add field existence check to the painless script #144344
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -32,13 +32,12 @@ export enum RawIndicatorFieldId { | |
FileSha3512 = 'threat.indicator.file.hash.sha3-512', | ||
FileSha512224 = 'threat.indicator.file.hash.sha512/224', | ||
FileSha512256 = 'threat.indicator.file.hash.sha512/256', | ||
FileSSDeep = 'threat.indicator.file.ssdeep', | ||
FileTlsh = 'threat.indicator.file.tlsh', | ||
FileImpfuzzy = 'threat.indicator.file.impfuzzy', | ||
FileImphash = 'threat.indicator.file.imphash', | ||
FilePehash = 'threat.indicator.file.pehash', | ||
FileVhash = 'threat.indicator.file.vhash', | ||
FileTelfhash = 'threat.indicator.file.elf.telfhash', | ||
FileSSDeep = 'threat.indicator.file.hash.ssdeep', | ||
FileTlsh = 'threat.indicator.file.hash.tlsh', | ||
FileImpfuzzy = 'threat.indicator.file.hash.impfuzzy', | ||
FileImphash = 'threat.indicator.file.hash.imphash', | ||
FilePehash = 'threat.indicator.file.hash.pehash', | ||
FileVhash = 'threat.indicator.file.hash.vhash', | ||
X509Serial = 'threat.indicator.x509.serial_number', | ||
WindowsRegistryKey = 'threat.indicator.registry.key', | ||
WindowsRegistryPath = 'threat.indicator.registry.path', | ||
|
@@ -56,12 +55,24 @@ export enum RawIndicatorFieldId { | |
* (reverse of https://github.com/elastic/kibana/blob/main/x-pack/plugins/security_solution/common/cti/constants.ts#L35) | ||
*/ | ||
export const IndicatorFieldEventEnrichmentMap: { [id: string]: string[] } = { | ||
[RawIndicatorFieldId.FileSha256]: ['file.hash.sha256'], | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this fixes an unrelated bug: happens when the grid is trying to display the investigate in timeline button, the data grid crashes. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Are we sure that this fixes it for good? What if there is no There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I also think this is a good candidate for 8.5 backporting There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it all depends on how we handle the original bug: if we show the value in the grid with a Once we figure out the original bug, I can update this piece of code as well There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I added a bit more safety to the |
||
[RawIndicatorFieldId.FileMd5]: ['file.hash.md5'], | ||
[RawIndicatorFieldId.FileSha1]: ['file.hash.sha1'], | ||
[RawIndicatorFieldId.FileSha256]: ['file.hash.sha256'], | ||
[RawIndicatorFieldId.FileImphash]: ['file.pe.imphash'], | ||
[RawIndicatorFieldId.FileTelfhash]: ['file.elf.telfhash'], | ||
[RawIndicatorFieldId.FileSha224]: ['file.hash.sha224'], | ||
[RawIndicatorFieldId.FileSha3224]: ['file.hash.sha3-224'], | ||
[RawIndicatorFieldId.FileSha3256]: ['file.hash.sha3-256'], | ||
[RawIndicatorFieldId.FileSha384]: ['file.hash.sha384'], | ||
[RawIndicatorFieldId.FileSha3384]: ['file.hash.sha3-384'], | ||
[RawIndicatorFieldId.FileSha512]: ['file.hash.sha512'], | ||
[RawIndicatorFieldId.FileSha3512]: ['file.hash.sha3-512'], | ||
[RawIndicatorFieldId.FileSha512224]: ['file.hash.sha512/224'], | ||
[RawIndicatorFieldId.FileSha512256]: ['file.hash.sha512/256'], | ||
[RawIndicatorFieldId.FileSSDeep]: ['file.hash.ssdeep'], | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same here, modifying following this |
||
[RawIndicatorFieldId.FileTlsh]: ['file.hash.tlsh'], | ||
[RawIndicatorFieldId.FileImpfuzzy]: ['file.hash.impfuzzy'], | ||
[RawIndicatorFieldId.FileImphash]: ['file.hash.imphash'], | ||
[RawIndicatorFieldId.FilePehash]: ['file.hash.pehash'], | ||
[RawIndicatorFieldId.FileVhash]: ['file.hash.vhash'], | ||
[RawIndicatorFieldId.Ip]: ['source.ip', 'destination.ip'], | ||
[RawIndicatorFieldId.UrlFull]: ['url.full'], | ||
[RawIndicatorFieldId.WindowsRegistryPath]: ['registry.path'], | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -26,9 +26,13 @@ import { | |
INDICATORS_TABLE, | ||
INDICATORS_TABLE_FEED_NAME_COLUMN_HEADER, | ||
INDICATORS_TABLE_FIRST_SEEN_COLUMN_HEADER, | ||
INDICATORS_TABLE_INDICATOR_NAME_CELL, | ||
INDICATORS_TABLE_INDICATOR_NAME_COLUMN_HEADER, | ||
INDICATORS_TABLE_INDICATOR_TYPE_CELL, | ||
INDICATORS_TABLE_INDICATOR_TYPE_COLUMN_HEADER, | ||
INDICATORS_TABLE_INVESTIGATE_IN_TIMELINE_BUTTON_ICON, | ||
INDICATORS_TABLE_LAST_SEEN_COLUMN_HEADER, | ||
INDICATORS_TABLE_ROW_CELL, | ||
INSPECTOR_BUTTON, | ||
INSPECTOR_PANEL, | ||
LEADING_BREADCRUMB, | ||
|
@@ -50,12 +54,88 @@ const THREAT_INTELLIGENCE = '/app/security/threat_intelligence/indicators'; | |
const URL_WITH_CONTRADICTORY_FILTERS = | ||
'/app/security/threat_intelligence/indicators?indicators=(filterQuery:(language:kuery,query:%27%27),filters:!((%27$state%27:(store:appState),meta:(alias:!n,disabled:!f,index:%27%27,key:threat.indicator.type,negate:!f,params:(query:file),type:phrase),query:(match_phrase:(threat.indicator.type:file))),(%27$state%27:(store:appState),meta:(alias:!n,disabled:!f,index:%27%27,key:threat.indicator.type,negate:!f,params:(query:url),type:phrase),query:(match_phrase:(threat.indicator.type:url)))),timeRange:(from:now/d,to:now/d))'; | ||
|
||
describe('Invalid Indicators', () => { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this section tests that the table shows up even with missing fields. The following logic is applied:
The UI Indicator name column should display the proper value for the present fields. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. added more tests for the following:
I think this is overkill but we should be safe |
||
describe('verify the grid loads even with missing fields', () => { | ||
before(() => { | ||
esArchiverLoad('threat_intelligence/invalid_indicators_data'); | ||
|
||
cy.visit(THREAT_INTELLIGENCE); | ||
selectRange(); | ||
}); | ||
after(() => { | ||
esArchiverUnload('threat_intelligence/invalid_indicators_data'); | ||
}); | ||
|
||
it('should display data grid despite the missing fields', () => { | ||
cy.get(INDICATORS_TABLE).should('exist'); | ||
|
||
// there are 19 documents in the x-pack/test/threat_intelligence_cypress/es_archives/threat_intelligence/invalid_indicators_data/data.json | ||
const documentsNumber = 22; | ||
cy.get(INDICATORS_TABLE_ROW_CELL).should('have.length.gte', documentsNumber); | ||
|
||
// the last 3 documents have no hash so the investigate in timeline button isn't rendered | ||
cy.get(INDICATORS_TABLE_INVESTIGATE_IN_TIMELINE_BUTTON_ICON).should( | ||
'have.length', | ||
documentsNumber - 4 | ||
); | ||
|
||
// we should have 21 documents plus the header | ||
cy.get(INDICATORS_TABLE_INDICATOR_NAME_CELL).should('have.length', documentsNumber + 1); | ||
|
||
// this entry has no hash to we show - in the Indicator Name column | ||
cy.get(INDICATORS_TABLE_INDICATOR_NAME_CELL) | ||
.eq(documentsNumber - 3) | ||
.should('contain.text', '-'); | ||
|
||
// this entry is missing the file key entirely | ||
cy.get(INDICATORS_TABLE_INDICATOR_NAME_CELL) | ||
.eq(documentsNumber - 2) | ||
.should('contain.text', '-'); | ||
|
||
// this entry is missing the type field | ||
cy.get(INDICATORS_TABLE_INDICATOR_NAME_CELL) | ||
.eq(documentsNumber - 1) | ||
.should('contain.text', '-'); | ||
cy.get(INDICATORS_TABLE_INDICATOR_TYPE_CELL) | ||
.eq(documentsNumber - 1) | ||
.should('contain.text', '-'); | ||
|
||
// this entry is missing the type field | ||
cy.get(INDICATORS_TABLE_INDICATOR_NAME_CELL).last().should('contain.text', '-'); | ||
cy.get(INDICATORS_TABLE_INDICATOR_TYPE_CELL).last().should('contain.text', '-'); | ||
}); | ||
}); | ||
|
||
describe('verify the grid loads even with missing mappings and missing fields', () => { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same tests as above but this time with a missing |
||
before(() => { | ||
esArchiverLoad('threat_intelligence/missing_mappings_indicators_data'); | ||
|
||
cy.visit(THREAT_INTELLIGENCE); | ||
selectRange(); | ||
}); | ||
after(() => { | ||
esArchiverUnload('threat_intelligence/missing_mappings_indicators_data'); | ||
}); | ||
|
||
it('should display data grid despite the missing mappings and missing fields', () => { | ||
cy.get(INDICATORS_TABLE).should('exist'); | ||
|
||
// there are 2 documents in the x-pack/test/threat_intelligence_cypress/es_archives/threat_intelligence/missing_mappings_indicators_data/data.json | ||
const documentsNumber = 2; | ||
cy.get(INDICATORS_TABLE_ROW_CELL).should('have.length.gte', documentsNumber); | ||
|
||
// we should have 2 documents plus the header | ||
cy.get(INDICATORS_TABLE_INDICATOR_NAME_CELL).should('have.length', documentsNumber + 1); | ||
}); | ||
}); | ||
}); | ||
|
||
describe('Indicators', () => { | ||
before(() => { | ||
esArchiverLoad('threat_intelligence'); | ||
esArchiverLoad('threat_intelligence/indicators_data'); | ||
}); | ||
after(() => { | ||
esArchiverUnload('threat_intelligence'); | ||
esArchiverUnload('threat_intelligence/indicators_data'); | ||
}); | ||
|
||
describe('Indicators page loading', () => { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -42,17 +42,23 @@ export const useInvestigateInTimeline = ({ | |
const securitySolutionContext = useContext(SecuritySolutionContext); | ||
|
||
const { key, value } = getIndicatorFieldAndValue(indicator, RawIndicatorFieldId.Name); | ||
if (!fieldAndValueValid(key, value)) { | ||
const sourceEventField = IndicatorFieldEventEnrichmentMap[key]; | ||
|
||
if (!fieldAndValueValid(key, value) || !sourceEventField) { | ||
return {} as unknown as UseInvestigateInTimelineValue; | ||
} | ||
|
||
const dataProviders: DataProvider[] = [...IndicatorFieldEventEnrichmentMap[key], key].map( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I believe this bug shows that we should be more strict with the types as we rely on a very specific set of fields for this feature |
||
(e: string) => generateDataProvider(e, value as string) | ||
const dataProviders: DataProvider[] = [...sourceEventField, key].map((e: string) => | ||
generateDataProvider(e, value as string) | ||
); | ||
|
||
const to = unwrapValue(indicator, RawIndicatorFieldId.TimeStamp) as string; | ||
const from = moment(to).subtract(10, 'm').toISOString(); | ||
|
||
if (!to || !from) { | ||
return {} as unknown as UseInvestigateInTimelineValue; | ||
} | ||
|
||
const investigateInTimelineFn = securitySolutionContext?.getUseInvestigateInTimeline({ | ||
dataProviders, | ||
from, | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
modified these using this source of truth