From 93914b6cb5afbc382699818c1beab6568a5dabac Mon Sep 17 00:00:00 2001 From: marshallmain <55718608+marshallmain@users.noreply.github.com> Date: Mon, 16 Mar 2020 15:53:49 -0400 Subject: [PATCH] [Endpoint] Sample data generator CLI script (#59952) * start on cli * make it work * cleanup * remove failed attempt code * update package and tsconfig * remove empty file * generate resolver events from multiple endpoints * re-add child randomization * align index names with real plugin * remove duplication * better naming * add temporary mapping to sample data generator * error handling, move tsconfig * add readme * Update README.md * move mapping from common to scripts * make delete index option * remove unnecessary map call * fix import style Co-authored-by: Elastic Machine --- .../endpoint/common/generate_data.test.ts | 2 +- .../plugins/endpoint/common/generate_data.ts | 111 +- x-pack/plugins/endpoint/package.json | 7 +- .../endpoint/store/managing/index.test.ts | 2 +- .../store/managing/middleware.test.ts | 2 +- x-pack/plugins/endpoint/scripts/README.md | 46 + .../endpoint/scripts/cli_tsconfig.json | 8 + x-pack/plugins/endpoint/scripts/mapping.json | 2367 +++++++++++++++++ .../endpoint/scripts/resolver_generator.ts | 161 ++ 9 files changed, 2634 insertions(+), 72 deletions(-) create mode 100644 x-pack/plugins/endpoint/scripts/README.md create mode 100644 x-pack/plugins/endpoint/scripts/cli_tsconfig.json create mode 100644 x-pack/plugins/endpoint/scripts/mapping.json create mode 100644 x-pack/plugins/endpoint/scripts/resolver_generator.ts diff --git a/x-pack/plugins/endpoint/common/generate_data.test.ts b/x-pack/plugins/endpoint/common/generate_data.test.ts index ebe3c25eef829c..e14f506c825f2c 100644 --- a/x-pack/plugins/endpoint/common/generate_data.test.ts +++ b/x-pack/plugins/endpoint/common/generate_data.test.ts @@ -151,7 +151,7 @@ describe('data generator', () => { const timestamp = new Date().getTime(); const root = generator.generateEvent({ timestamp }); const generations = 2; - const events = generator.generateDescendantsTree(root, generations); + const events = [root, ...generator.generateDescendantsTree(root, generations)]; const rootNode = buildResolverTree(events); const visitedEvents = countResolverEvents(rootNode, generations); expect(visitedEvents).toEqual(events.length); diff --git a/x-pack/plugins/endpoint/common/generate_data.ts b/x-pack/plugins/endpoint/common/generate_data.ts index a91cf0ffca7836..b539e309d76f72 100644 --- a/x-pack/plugins/endpoint/common/generate_data.ts +++ b/x-pack/plugins/endpoint/common/generate_data.ts @@ -6,7 +6,7 @@ import uuid from 'uuid'; import seedrandom from 'seedrandom'; -import { AlertEvent, EndpointEvent, EndpointMetadata, OSFields } from './types'; +import { AlertEvent, EndpointEvent, EndpointMetadata, OSFields, HostFields } from './types'; export type Event = AlertEvent | EndpointEvent; @@ -67,31 +67,51 @@ const FILE_OPERATIONS: string[] = ['creation', 'open', 'rename', 'execution', 'd // These are from the v1 schemas and aren't all valid ECS event categories, still in flux const OTHER_EVENT_CATEGORIES: string[] = ['driver', 'file', 'library', 'network', 'registry']; +interface HostInfo { + agent: { + version: string; + id: string; + }; + host: HostFields; + endpoint: { + policy: { + id: string; + }; + }; +} + export class EndpointDocGenerator { - agentId: string; - hostId: string; - hostname: string; - macAddress: string[]; - ip: string[]; - agentVersion: string; - os: OSFields; - policy: { name: string; id: string }; + commonInfo: HostInfo; random: seedrandom.prng; constructor(seed = Math.random().toString()) { this.random = seedrandom(seed); - this.hostId = this.seededUUIDv4(); - this.agentId = this.seededUUIDv4(); - this.hostname = this.randomHostname(); - this.ip = this.randomArray(3, () => this.randomIP()); - this.macAddress = this.randomArray(3, () => this.randomMac()); - this.agentVersion = this.randomVersion(); - this.os = this.randomChoice(OS); - this.policy = this.randomChoice(POLICIES); + this.commonInfo = this.createHostData(); } - public randomizeIPs() { - this.ip = this.randomArray(3, () => this.randomIP()); + // This function will create new values for all the host fields, so documents from a different endpoint can be created + // This provides a convenient way to make documents from multiple endpoints that are all tied to a single seed value + public randomizeHostData() { + this.commonInfo = this.createHostData(); + } + + private createHostData(): HostInfo { + return { + agent: { + version: this.randomVersion(), + id: this.seededUUIDv4(), + }, + host: { + id: this.seededUUIDv4(), + hostname: this.randomHostname(), + ip: this.randomArray(3, () => this.randomIP()), + mac: this.randomArray(3, () => this.randomMac()), + os: this.randomChoice(OS), + }, + endpoint: { + policy: this.randomChoice(POLICIES), + }, + }; } public generateEndpointMetadata(ts = new Date().getTime()): EndpointMetadata { @@ -100,22 +120,7 @@ export class EndpointDocGenerator { event: { created: ts, }, - endpoint: { - policy: { - id: this.policy.id, - }, - }, - agent: { - version: this.agentVersion, - id: this.agentId, - }, - host: { - id: this.hostId, - hostname: this.hostname, - ip: this.ip, - mac: this.macAddress, - os: this.os, - }, + ...this.commonInfo, }; } @@ -125,11 +130,8 @@ export class EndpointDocGenerator { parentEntityID?: string ): AlertEvent { return { + ...this.commonInfo, '@timestamp': ts, - agent: { - id: this.agentId, - version: this.agentVersion, - }, event: { action: this.randomChoice(FILE_OPERATIONS), kind: 'alert', @@ -139,11 +141,6 @@ export class EndpointDocGenerator { module: 'endpoint', type: 'creation', }, - endpoint: { - policy: { - id: this.policy.id, - }, - }, file: { owner: 'SYSTEM', name: 'fake_malware.exe', @@ -169,13 +166,6 @@ export class EndpointDocGenerator { }, temp_file_path: 'C:/temp/fake_malware.exe', }, - host: { - id: this.hostId, - hostname: this.hostname, - ip: this.ip, - mac: this.macAddress, - os: this.os, - }, process: { pid: 2, name: 'malware writer', @@ -243,11 +233,7 @@ export class EndpointDocGenerator { public generateEvent(options: EventOptions = {}): EndpointEvent { return { '@timestamp': options.timestamp ? options.timestamp : new Date().getTime(), - agent: { - id: this.agentId, - version: this.agentVersion, - type: 'endpoint', - }, + agent: { ...this.commonInfo.agent, type: 'endgame' }, ecs: { version: '1.4.0', }, @@ -257,13 +243,7 @@ export class EndpointDocGenerator { type: options.eventType ? options.eventType : 'creation', id: this.seededUUIDv4(), }, - host: { - id: this.hostId, - hostname: this.hostname, - ip: this.ip, - mac: this.macAddress, - os: this.os, - }, + host: this.commonInfo.host, process: { entity_id: options.entityID ? options.entityID : this.randomString(10), parent: options.parentEntityID ? { entity_id: options.parentEntityID } : undefined, @@ -323,14 +303,13 @@ export class EndpointDocGenerator { percentNodesWithRelated = 100, percentChildrenTerminated = 100 ): Event[] { - let events: Event[] = [root]; + let events: Event[] = []; let parents = [root]; let timestamp = root['@timestamp']; for (let i = 0; i < generations; i++) { const newParents: EndpointEvent[] = []; parents.forEach(element => { - // const numChildren = randomN(maxChildrenPerNode); - const numChildren = maxChildrenPerNode; + const numChildren = this.randomN(maxChildrenPerNode); for (let j = 0; j < numChildren; j++) { timestamp = timestamp + 1000; const child = this.generateEvent({ diff --git a/x-pack/plugins/endpoint/package.json b/x-pack/plugins/endpoint/package.json index c7ba8b3fb41960..fc4f4bd586bef7 100644 --- a/x-pack/plugins/endpoint/package.json +++ b/x-pack/plugins/endpoint/package.json @@ -4,10 +4,11 @@ "version": "0.0.0", "private": true, "license": "Elastic-License", - "scripts": {}, + "scripts": { + "test:generate": "ts-node --project scripts/cli_tsconfig.json scripts/resolver_generator.ts" + }, "dependencies": { - "react-redux": "^7.1.0", - "seedrandom": "^3.0.5" + "react-redux": "^7.1.0" }, "devDependencies": { "@types/seedrandom": ">=2.0.0 <4.0.0", diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/store/managing/index.test.ts b/x-pack/plugins/endpoint/public/applications/endpoint/store/managing/index.test.ts index fba1dacb0d3bdc..e435fded13f4ce 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/store/managing/index.test.ts +++ b/x-pack/plugins/endpoint/public/applications/endpoint/store/managing/index.test.ts @@ -20,7 +20,7 @@ describe('endpoint_list store concerns', () => { dispatch = store.dispatch; }; const generateEndpoint = (): EndpointMetadata => { - return generator.generateEndpointMetadata(new Date().getTime()); + return generator.generateEndpointMetadata(); }; const loadDataToStore = () => { dispatch({ diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/store/managing/middleware.test.ts b/x-pack/plugins/endpoint/public/applications/endpoint/store/managing/middleware.test.ts index d98dc82624149e..459a1789a58da7 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/store/managing/middleware.test.ts +++ b/x-pack/plugins/endpoint/public/applications/endpoint/store/managing/middleware.test.ts @@ -27,7 +27,7 @@ describe('endpoint list saga', () => { const generator = new EndpointDocGenerator(); // https://github.com/elastic/endpoint-app-team/issues/131 const generateEndpoint = (): EndpointMetadata => { - return generator.generateEndpointMetadata(new Date().getTime()); + return generator.generateEndpointMetadata(); }; let history: History; diff --git a/x-pack/plugins/endpoint/scripts/README.md b/x-pack/plugins/endpoint/scripts/README.md new file mode 100644 index 00000000000000..f0c8c5a9b0b660 --- /dev/null +++ b/x-pack/plugins/endpoint/scripts/README.md @@ -0,0 +1,46 @@ +This script makes it easy to create the endpoint metadata, alert, and event documents needed to test Resolver in Kibana. +The default behavior is to create 1 endpoint with 1 alert and a moderate number of events (random, typically on the order of 20). +A seed value can be provided as a string for the random number generator for repeatable behavior, useful for demos etc. +Use the `-d` option if you want to delete and remake the indices, otherwise it will add documents to existing indices. + +The sample data generator script depends on ts-node, install with npm: + +```npm install -g ts-node``` + +Example command sequence to get ES and kibana running with sample data after installing ts-node: + +```yarn es snapshot``` -> starts ES + +```npx yarn start --xpack.endpoint.enabled=true --no-base-path``` -> starts kibana + +```cd ~/path/to/kibana/x-pack/plugins/endpoint``` + +```yarn test:generate --auth elastic:changeme``` -> run the resolver_generator.ts script + +Resolver generator CLI options: +```--help Show help [boolean] + --seed, -s random seed to use for document generator [string] + --node, -n elasticsearch node url + [string] [default: "http://localhost:9200"] + --eventIndex, --ei index to store events in + [string] [default: "events-endpoint-1"] + --metadataIndex, --mi index to store endpoint metadata in + [string] [default: "endpoint-agent-1"] + --auth elasticsearch username and password, separated by + a colon [string] + --ancestors, --anc number of ancestors of origin to create + [number] [default: 3] + --generations, --gen number of child generations to create + [number] [default: 3] + --children, --ch maximum number of children per node + [number] [default: 3] + --relatedEvents, --related number of related events to create for each + process event [number] [default: 5] + --percentWithRelated, --pr percent of process events to add related events to + [number] [default: 30] + --percentTerminated, --pt percent of process events to add termination event + for [number] [default: 30] + --numEndpoints, --ne number of different endpoints to generate alerts + for [number] [default: 1] + --alertsPerEndpoint, --ape number of resolver trees to make for each endpoint + [number] [default: 1]``` diff --git a/x-pack/plugins/endpoint/scripts/cli_tsconfig.json b/x-pack/plugins/endpoint/scripts/cli_tsconfig.json new file mode 100644 index 00000000000000..25afe109a42ead --- /dev/null +++ b/x-pack/plugins/endpoint/scripts/cli_tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "target": "es2019", + "resolveJsonModule": true + } + } + \ No newline at end of file diff --git a/x-pack/plugins/endpoint/scripts/mapping.json b/x-pack/plugins/endpoint/scripts/mapping.json new file mode 100644 index 00000000000000..34c039d6435171 --- /dev/null +++ b/x-pack/plugins/endpoint/scripts/mapping.json @@ -0,0 +1,2367 @@ +{ + "mappings": { + "_meta": { + "version": "1.5.0-dev" + }, + "date_detection": false, + "dynamic": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "dll": { + "properties": { + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "compile_time": { + "type": "date" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "malware_classifier": { + "properties": { + "features": { + "properties": { + "data": { + "properties": { + "buffer": { + "ignore_above": 1024, + "type": "keyword" + }, + "decompressed_size": { + "type": "integer" + }, + "encoding": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "score": { + "type": "double" + }, + "threshold": { + "type": "double" + }, + "upx_packed": { + "type": "boolean" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "mapped_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "mapped_size": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "endpoint": { + "properties": { + "artifact": { + "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "policy": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequence": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "file": { + "properties": { + "accessed": { + "type": "date" + }, + "attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "created": { + "type": "date" + }, + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "directory": { + "ignore_above": 1024, + "type": "keyword" + }, + "drive_letter": { + "ignore_above": 1, + "type": "keyword" + }, + "entry_modified": { + "type": "double" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "macro": { + "properties": { + "code_page": { + "type": "long" + }, + "collection": { + "properties": { + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + }, + "errors": { + "properties": { + "count": { + "type": "long" + }, + "error_type": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + }, + "file_extension": { + "type": "long" + }, + "project_file": { + "properties": { + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + }, + "stream": { + "properties": { + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "raw_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "raw_code_size": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + } + } + }, + "malware_classifier": { + "properties": { + "features": { + "properties": { + "data": { + "properties": { + "buffer": { + "ignore_above": 1024, + "type": "keyword" + }, + "decompressed_size": { + "type": "integer" + }, + "encoding": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "score": { + "type": "double" + }, + "threshold": { + "type": "double" + }, + "upx_packed": { + "type": "boolean" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "size": { + "type": "long" + }, + "target_path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "temp_file_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "cpu_percent": { + "type": "double" + }, + "cwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "env_variables": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "handles": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "malware_classifier": { + "properties": { + "features": { + "properties": { + "data": { + "properties": { + "buffer": { + "ignore_above": 1024, + "type": "keyword" + }, + "decompressed_size": { + "type": "integer" + }, + "encoding": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "score": { + "type": "double" + }, + "threshold": { + "type": "double" + }, + "upx_packed": { + "type": "boolean" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "memory_percent": { + "type": "double" + }, + "memory_region": { + "properties": { + "allocation_base": { + "ignore_above": 1024, + "type": "keyword" + }, + "allocation_protection": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "histogram": { + "properties": { + "histogram_array": { + "ignore_above": 1024, + "type": "keyword" + }, + "histogram_flavor": { + "ignore_above": 1024, + "type": "keyword" + }, + "histogram_resolution": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + }, + "length": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "module_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "permission": { + "ignore_above": 1024, + "type": "keyword" + }, + "protection": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_base": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_size": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_tag": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "unbacked_on_disk": { + "type": "boolean" + } + }, + "type": "nested" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "num_threads": { + "type": "long" + }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "entrypoint": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "start_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_address_module": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "phys_memory_bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "services": { + "ignore_above": 1024, + "type": "keyword" + }, + "session_id": { + "type": "long" + }, + "short_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sid": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "call_stack": { + "properties": { + "instruction_pointer": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory_section": { + "properties": { + "memory_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory_size": { + "ignore_above": 1024, + "type": "keyword" + }, + "protection": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "module_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "rva": { + "ignore_above": 1024, + "type": "keyword" + }, + "symbol_info": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "entrypoint": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "start_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_address_module": { + "ignore_above": 1024, + "type": "keyword" + }, + "token": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "impersonation_level": { + "ignore_above": 1024, + "type": "keyword" + }, + "integrity_level": { + "type": "long" + }, + "integrity_level_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_appcontainer": { + "type": "boolean" + }, + "privileges": { + "properties": { + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + }, + "sid": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uptime": { + "type": "long" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "token": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "impersonation_level": { + "ignore_above": 1024, + "type": "keyword" + }, + "integrity_level": { + "type": "long" + }, + "integrity_level_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_appcontainer": { + "type": "boolean" + }, + "privileges": { + "properties": { + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + }, + "sid": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tty_device": { + "properties": { + "major_number": { + "type": "integer" + }, + "minor_number": { + "type": "integer" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uptime": { + "type": "long" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + }, + "virt_memory_bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "target": { + "properties": { + "dll": { + "properties": { + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "compile_time": { + "type": "date" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "malware_classifier": { + "properties": { + "features": { + "properties": { + "data": { + "properties": { + "buffer": { + "ignore_above": 1024, + "type": "keyword" + }, + "decompressed_size": { + "type": "integer" + }, + "encoding": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "score": { + "type": "double" + }, + "threshold": { + "type": "double" + }, + "upx_packed": { + "type": "boolean" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "mapped_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "mapped_size": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "cpu_percent": { + "type": "double" + }, + "cwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "env_variables": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "handles": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "malware_classifier": { + "properties": { + "features": { + "properties": { + "data": { + "properties": { + "buffer": { + "ignore_above": 1024, + "type": "keyword" + }, + "decompressed_size": { + "type": "integer" + }, + "encoding": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "score": { + "type": "double" + }, + "threshold": { + "type": "double" + }, + "upx_packed": { + "type": "boolean" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "memory_percent": { + "type": "double" + }, + "memory_region": { + "properties": { + "allocation_base": { + "ignore_above": 1024, + "type": "keyword" + }, + "allocation_protection": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "histogram": { + "properties": { + "histogram_array": { + "ignore_above": 1024, + "type": "keyword" + }, + "histogram_flavor": { + "ignore_above": 1024, + "type": "keyword" + }, + "histogram_resolution": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + }, + "length": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "module_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "permission": { + "ignore_above": 1024, + "type": "keyword" + }, + "protection": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_base": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_size": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_tag": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "unbacked_on_disk": { + "type": "boolean" + } + }, + "type": "nested" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "num_threads": { + "type": "long" + }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "entrypoint": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "start_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_address_module": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "phys_memory_bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "services": { + "ignore_above": 1024, + "type": "keyword" + }, + "session_id": { + "type": "long" + }, + "short_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sid": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "call_stack": { + "properties": { + "instruction_pointer": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory_section": { + "properties": { + "memory_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory_size": { + "ignore_above": 1024, + "type": "keyword" + }, + "protection": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "module_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "rva": { + "ignore_above": 1024, + "type": "keyword" + }, + "symbol_info": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "entrypoint": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "start_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_address_module": { + "ignore_above": 1024, + "type": "keyword" + }, + "token": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "impersonation_level": { + "ignore_above": 1024, + "type": "keyword" + }, + "integrity_level": { + "type": "long" + }, + "integrity_level_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_appcontainer": { + "type": "boolean" + }, + "privileges": { + "properties": { + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + }, + "sid": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uptime": { + "type": "long" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "token": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "impersonation_level": { + "ignore_above": 1024, + "type": "keyword" + }, + "integrity_level": { + "type": "long" + }, + "integrity_level_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_appcontainer": { + "type": "boolean" + }, + "privileges": { + "properties": { + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "nested" + }, + "sid": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tty_device": { + "properties": { + "major_number": { + "type": "integer" + }, + "minor_number": { + "type": "integer" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uptime": { + "type": "long" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + }, + "virt_memory_bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": 10000 + } + }, + "refresh_interval": "5s" + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/endpoint/scripts/resolver_generator.ts b/x-pack/plugins/endpoint/scripts/resolver_generator.ts new file mode 100644 index 00000000000000..a3e56497f0790c --- /dev/null +++ b/x-pack/plugins/endpoint/scripts/resolver_generator.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import * as yargs from 'yargs'; +import { Client, ClientOptions } from '@elastic/elasticsearch'; +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { EndpointDocGenerator } from '../common/generate_data'; +import { default as mapping } from './mapping.json'; + +main(); + +async function main() { + const argv = yargs.help().options({ + seed: { + alias: 's', + describe: 'random seed to use for document generator', + type: 'string', + }, + node: { + alias: 'n', + describe: 'elasticsearch node url', + default: 'http://localhost:9200', + type: 'string', + }, + eventIndex: { + alias: 'ei', + describe: 'index to store events in', + default: 'events-endpoint-1', + type: 'string', + }, + metadataIndex: { + alias: 'mi', + describe: 'index to store endpoint metadata in', + default: 'endpoint-agent-1', + type: 'string', + }, + auth: { + describe: 'elasticsearch username and password, separated by a colon', + type: 'string', + }, + ancestors: { + alias: 'anc', + describe: 'number of ancestors of origin to create', + type: 'number', + default: 3, + }, + generations: { + alias: 'gen', + describe: 'number of child generations to create', + type: 'number', + default: 3, + }, + children: { + alias: 'ch', + describe: 'maximum number of children per node', + type: 'number', + default: 3, + }, + relatedEvents: { + alias: 'related', + describe: 'number of related events to create for each process event', + type: 'number', + default: 5, + }, + percentWithRelated: { + alias: 'pr', + describe: 'percent of process events to add related events to', + type: 'number', + default: 30, + }, + percentTerminated: { + alias: 'pt', + describe: 'percent of process events to add termination event for', + type: 'number', + default: 30, + }, + numEndpoints: { + alias: 'ne', + describe: 'number of different endpoints to generate alerts for', + type: 'number', + default: 1, + }, + alertsPerEndpoint: { + alias: 'ape', + describe: 'number of resolver trees to make for each endpoint', + type: 'number', + default: 1, + }, + delete: { + alias: 'd', + describe: 'delete indices and remake them', + type: 'boolean', + default: false, + }, + }).argv; + const clientOptions: ClientOptions = { + node: argv.node, + }; + if (argv.auth) { + const [username, password]: string[] = argv.auth.split(':', 2); + clientOptions.auth = { username, password }; + } + const client = new Client(clientOptions); + if (argv.delete) { + try { + await client.indices.delete({ + index: [argv.eventIndex, argv.metadataIndex], + }); + } catch (err) { + if (err instanceof ResponseError && err.statusCode !== 404) { + // eslint-disable-next-line no-console + console.log(err); + process.exit(1); + } + } + } + try { + await client.indices.create({ + index: argv.eventIndex, + body: mapping, + }); + } catch (err) { + if ( + err instanceof ResponseError && + err.body.error.type !== 'resource_already_exists_exception' + ) { + // eslint-disable-next-line no-console + console.log(err.body); + process.exit(1); + } + } + + const generator = new EndpointDocGenerator(argv.seed); + for (let i = 0; i < argv.numEndpoints; i++) { + await client.index({ + index: argv.metadataIndex, + body: generator.generateEndpointMetadata(), + }); + for (let j = 0; j < argv.alertsPerEndpoint; j++) { + const resolverDocs = generator.generateFullResolverTree( + argv.ancestors, + argv.generations, + argv.children, + argv.relatedEvents, + argv.percentWithRelated, + argv.percentTerminated + ); + const body = resolverDocs.reduce( + (array: Array>, doc) => ( + array.push({ index: { _index: argv.eventIndex } }, doc), array + ), + [] + ); + + await client.bulk({ body }); + } + generator.randomizeHostData(); + } +}