New package for Carbon Black EDR logs (#1527)

Adds a new package for VMware Carbon Black EDR logs ingested via CB Event Forwarder.

packages/carbonback_edr/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@1.11

packages/carbonback_edr/_dev/build/docs/README.md

@@ -0,0 +1,35 @@
+# VMware Carbon Black EDR Integration
+
+The VMware Carbon Black EDR integration collects EDR Server and raw Endpoint events exported by [Carbon Black EDR Event Forwarder.](https://github.com/carbonblack/cb-event-forwarder) The following output methods are supported: `http`, `tcp`, `udp` and `file`.
+
+## Compatibility
+
+This integration has been tested with the 3.7.4 version of EDR Event Forwarder.
+
+## Configuration
+
+The following configuration is necessary in `cb-event-forwarder.conf`:
+
+- `output_format=json` (default)
+
+For `http` output:
+  - `output_type=http`
+  - `http_post_template=[{{"{{"}}range .Events}}{{"{{"}}.EventText}}{{"{{"}}end}}]`
+  - `content_type=application/json` (default)
+
+For `tcp` output:
+  - `output_type=tcp`
+  - `tcpout=<Address of Elastic Agent>:<port>`
+
+For `udp` output:
+- `output_type=tcp`
+- `tcpout=<Address of Elastic Agent>:<port>`
+
+For `file` output:
+- `output_type=file`
+- `outfile=<path to a file readable by Elastic Agent>`
+
+{{event "log"}}
+
+{{fields "log"}}
+

packages/carbonback_edr/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2.3'
+services:
+  carbonblack_edr-logfile:
+    image: alpine
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+      - ${SERVICE_LOGS_DIR}:/var/log
+    command: /bin/sh -c "cp /sample_logs/* /var/log/"
+  carbonblack_edr-http:
+    image: akroh/stream:v0.2.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    environment:
+      - STREAM_PROTOCOL=webhook
+      - STREAM_ADDR=http://elastic-agent:9080/
+    command: log --start-signal=SIGHUP --delay=5s /sample_logs/cb_edr.ndjson.log
+  carbonblack_edr-tcp:
+    image: akroh/stream:v0.2.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    environment:
+      - STREAM_PROTOCOL=tcp
+      - STREAM_ADDR=elastic-agent:9081
+    command: log --start-signal=SIGHUP --delay=5s /sample_logs/cb_edr.ndjson.log
+  carbonblack_edr-udp:
+    image: akroh/stream:v0.2.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    environment:
+      - STREAM_PROTOCOL=udp
+      - STREAM_ADDR=elastic-agent:9081
+    command: log --start-signal=SIGHUP --delay=5s /sample_logs/cb_edr.ndjson.log

packages/carbonback_edr/_dev/deploy/docker/sample_logs/cb_edr.ndjson.log

@@ -0,0 +1,21 @@
+{"server_name":"cb-enterprise-testing.local","docs":[{"process_md5":"a3ccfd0aa0b17fd23aa9fd0d84b86c05","sensor_id":1,"modload_count":49,"parent_unique_id":"00000001-0000-09e4-01cf-a5dee70168f2-00000001","cmdline":"\"c:\\users\\admin\\desktop\\putty.exe\" ","filemod_count":0,"id":"00000001-0000-afbc-01cf-b31b9e83777f","parent_name":"explorer.exe","parent_md5":"332feab1435662fc6c672e25beb37be3","group":"Default Group","hostname":"WIN8-TEST","last_update":"2014-08-08T15:15:47.544Z","start":"2014-08-08T15:15:42.193Z","regmod_count":6,"process_pid":44988,"username":"win8-test\\admin","process_name":"putty.exe","path":"c:\\users\\admin\\desktop\\putty.exe","netconn_count":1,"parent_pid":2532,"segment_id":1,"host_type":"workstation","os_type":"windows","childproc_count":0,"unique_id":"00000001-0000-afbc-01cf-b31b9e83777f-00000001"}],"event_timestamp":1407362104.19,"watchlist_id":10,"cb_version":"4.2.1.140808.1059","watchlist_name":"Tor Feed"}
+{"server_name":"cb-enterprise-testing.local","docs":[{"digsig_result":"Signed","observed_filename":["c:\\windows\\system32\\prncache.dll"],"product_version":"6.1.7601.17514","signed":"Signed","digsig_sign_time":"2010-11-21T00:37:00Z","is_executable_image":true,"orig_mod_len":183808,"is_64bit":true,"digsig_publisher":"Microsoft Corporation","group":["Default Group"],"file_version":"6.1.7601.17514 (win7sp1_rtm.101119-1850)","company_name":"Microsoft Corporation","internal_name":"PrintCache","product_name":"Microsoft® Windows® Operating System","digsig_result_code":"0","timestamp":"2014-08-09T11:19:04.009Z","copied_mod_len":183808,"server_added_timestamp":"2014-08-09T11:19:04.009Z","md5":"A1CDE92DDC170D307DB3C5BAA348811B","endpoint":["WIN8-TEST|1"],"legal_copyright":"© Microsoft Corporation. All rights reserved.","original_filename":"PrnCache.dll","os_type":"Windows","file_desc":"Print UI Cache"}],"event_timestamp":1407583203.5,"watchlist_id":10,"cb_version":"4.2.1.140811.29","watchlist_name":"SRS Trust"}
+{"process_id":"00000001-0000-afbc-01cf-b31b9e83777f","report_id":"TOR-Node-38.229.70.52","ioc_type":"ipv4","ioc_value":"38.229.70.52","ioc_attr":{"port":22,"protocol":"TCP","direction":"Outbound"},"hostname":"FS-NYC-1","sensor_id":1,"cb_version":"4.2.1.140808.1059","server_name":"localhost.localdomain","feed_id":14,"feed_name":"tor","event_timestamp":1407362000}
+{"md5":"506708142BC63DABA64F2D3AD1DCD5BF","report_id":"dxmtest1_04","ioc_type":"md5","ioc_value":"506708142bc63daba64f2d3ad1dcd5bf","ioc_attr":{},"feed_id":7,"hostname":"FS-SEA-529","sensor_id":3321,"cb_version":"4.2.1.140808.1059","server_name":"localhost.localdomain","feed_name":"dxmtest1","event_timestamp":1397244093.682}
+{"process_id":"00000001-0000-afbc-01cf-b31b9e83777f","segment_id":1,"docs":{"modload_count":0,"host_type":"workstation","netconn_count":"1","os_type":"windows","unique_id":"00000001-0000-afbc-01cf-b31b9e83777f-00000001","username":"win8-test\\admin","last_update":"2014-08-08T15:15:47.544Z","parent_md5":"332feab1435662fc6c672e25beb37be3","path":"c:\\users\\admin\\desktop\\putty.exe","filemod_count":0,"regmod_count":6,"process_name":"putty.exe","cmdline":"\"c:\\users\\admin\\desktop\\putty.exe\" ","parent_unique_id":"00000001-0000-09e4-01cf-a5dee70168f2-00000001","childproc_count":0,"process_pid":"44988","start":"2014-08-08T15:15:42.193Z","process_md5":"a3ccfd0aa0b17fd23aa9fd0d84b86c05","parent_name":"explorer.exe","parent_pid":"2532","group":"Default Group"},"report_id":"TOR-Node-38.229.70.52","ioc_type":"ipv4","ioc_value":"38.229.70.52","ioc_attr":{"port":"22","protocol":"TCP","direction":"Outbound"},"hostname":"WIN8-TEST","sensor_id":1,"cb_version":"4.2.1.140808.1059","server_name":"localhost","feed_id":14,"feed_name":"tor","event_timestamp":1407362099.567}
+{"md5":"C3489639EC8E181044F6C6BFD3D01AC9","docs":[{"file_version":"6.1.7601.17514 (win7sp1_rtm.101119-1850)","product_name":"Microsoft Windows Operating System","is_executable_image":"true","digsig_result":"Signed","observed_filename":["c:\\windows\\system32\\sndvol.exe","C:\\Windows\\system32\\sndvol.exe"],"os_type":"Windows","orig_mod_len":"273920","company_name":"Microsoft Corporation","server_added_timestamp":"Aug 9, 2014 5:27:56 PM","internal_name":"Volume Control Applet","copied_mod_len":"0","product_version":"6.1.7601.17514","digsig_sign_time":"2010-11-21T00:37:00.000Z","alliance_score_srstrust":"-100","digsig_result_code":"0","file_desc":"Volume Mixer","endpoint":"WIN8-TEST|1","legal_copyright":"Microsoft Corporation. All rights reserved.","original_filename":"SndVol.exe","is_64bit":"true","md5":"C3489639EC8E181044F6C6BFD3D01AC9","digsig_publisher":"Microsoft Corporation","group":"Default Group"}],"report_id":"c3489639ec8e181044f6c6bfd3d01ac9","ioc_type":"md5","ioc_value":"c3489639ec8e181044f6c6bfd3d01ac9","ioc_attr":{},"hostname":"WIN8-TEST","sensor_id":1,"cb_version":"4.2.1.140811.1054","server_name":"localhost","feed_id":2,"feed_name":"srstrust","event_timestamp":1407621575.945}
+{"process_id":"00000001-0000-1098-01cf-cc5fea563f8f","sensor_id":1,"segment_id":1,"docs":[{"username":"WIN7X64-BUILDER\\User","process_md5":"f2c7bb8acc97f92e987a2d4087d021b1","modload_count":20,"parent_unique_id":"00000001-0000-0a84-01cf-c240c9d1f378-00000001","process_name":"notepad.exe","cmdline":"\"c:\\windows\\system32\\notepad.exe\" ","os_type":"windows","path":"c:\\windows\\system32\\notepad.exe","last_update":"2014-09-09T18:57:34.267Z","parent_pid":2692,"crossproc_count":0,"parent_name":"explorer.exe","parent_md5":"000000000000000000000000000000","group":"Default Group","netconn_count":0,"hostname":"WIN7X64-BUILDER","host_type":"workstation","filemod_count":0,"start":"2014-09-09T18:57:34.251Z","unique_id":"00000001-0000-1098-01cf-cc5fea563f8f-00000001","regmod_count":0,"childproc_count":0,"process_pid":4248}],"hostname":"DXM021-VM1","event_timestamp":1410289221.38,"feed_name":"dxmtest2","feed_id":12,"ioc_value":"cb.urlver=1&cb.q.process_name=notepad.exe&sort=start%20desc&rows=10&start=0","ioc_type":"query","ioc_attrs":{"highlights":["PREPREPREnotepad.exePOSTPOSTPOST","c:\\windows\\system32\\PREPREPREnotepad.exePOSTPOSTPOST"]},"report_id":"notepad_proc"}
+{"sensor_id":1,"docs":[{"host_count":1,"digsig_result":"Unsigned","observed_filename":["c:\\program files (x86)\\programmer's notepad\\pn.exe"],"product_version":"2.3.4.0-charles","signed":"Unsigned","is_executable_image":false,"orig_mod_len":3092992,"is_64bit":false,"group":["Default Group"],"file_version":"2.3.4.0","company_name":"Simon Steele (Echo Software)","internal_name":"PNWTL","product_name":"Programmer's Notepad","digsig_result_code":"2148204800","timestamp":"2014-09-09T21:00:29.875Z","copied_mod_len":3092992,"server_added_timestamp":"2014-09-09T21:00:29.875Z","md5":"EFA7ECAF4468E0106E8B1041C5CE450E","endpoint":["WIN7X64-BUILDER|1"],"legal_copyright":"Copyright © 2002-2010 Simon Steele (Echo Software)","original_filename":"pn.exe","os_type":"Windows","file_desc":"Programmer's Notepad 2","last_seen":"2014-09-09T21:00:29.875Z"}],"hostname":"DXM021-VM1","event_timestamp":1410296635.26,"feed_name":"dxmtest2","feed_id":12,"ioc_value":"cb.urlver=1&cb.q.process_name=notepad.exe&sort=start%20desc&rows=10&start=0","ioc_type":"query","md5":"EFA7ECAF4468E0106E8B1041C5CE450E","report_id":"Newly Loaded Modules"}
+{"md5":"9E4B0E7472B4CEBA9E17F440B8CB0AB8","event_timestamp":1397248033.914,"scores":{"alliance_score_virustotal":16}}
+{"md5":"9E4B0E7472B4CEBA9E17F440B8CB0AB8","hostname":"FS-HQ","sensor_id":1021,"event_timestamp":1397248033.914,"scores":{"alliance_score_virustotal":16},"watchlists":{"watchlist_7":"2014-02-13T00:30:11.247Z","watchlist_9":"2014-02-13T00:21:13.009Z"}}
+{"md5":"9E4B0E7472B4CEBA9E17F440B8CB0AB8","file_path":"/var/cb/data/modulestore/FE2/AFA/FE2AFACC396DC37F51421DE4A08DA8A7.zip","size":320000,"compressed_size":126857,"event_timestamp":1397248033.914}
+{"action":"writeval","actiontype":2,"cb_server":"cbserver","computer_name":"JASON-WIN81-VM","event_type":"regmod","link_process":"https://cbtests/#analyze/00000001-0000-0484-01d1-1e951b7c000b/1","link_sensor":"https://cbtests/#/host/1","md5":"0E7196981EDE614F1F54FFF2C3843ADF","path":"\\registry\\user\\s-1-5-21-2709706146-4189370754-997381202-1001\\software\\microsoft\\vscommon\\12.0\\sqm\\pids\\1156\\stillalive","pid":1156,"process_guid":"00000001-0000-0484-01d1-1e951b7c000b","sensor_id":1,"timestamp":1447696798,"type":"ingress.event.regmod"}
+{"action":"create","actiontype":1,"cb_server":"cbserver","computer_name":"JASON-WIN81-VM","event_type":"filemod","filetype":0,"filetype_name":"Unknown","link_process":"https://cbtests/#analyze/00000001-0000-0c70-01d1-1e951aae7e2f/1","link_sensor":"https://cbtests/#/host/1","md5":"7A2870C2A8283B3630BF7670D0362B94","path":"c:\\users\\admin\\appdata\\local\\google\\chrome\\user data\\b5e2.tmp","pid":3184,"process_guid":"00000001-0000-0c70-01d1-1e951aae7e2f","sensor_id":1,"timestamp":1447696804,"type":"ingress.event.filemod"}
+{"cb_server":"cbserver","computer_name":"WIN-OTEMNUTBS23","direction":"outbound","domain":"","event_type":"netconn","ipv4":"23.4.187.27","link_process":"https://cbtests/#analyze/00000007-0000-090c-01d1-2099b8f18a82/1","link_sensor":"https://cbtests/#/host/7","local_ip":"172.31.30.0","local_port":49352,"md5":"C10A66189DC8C090E7C84873EDCEBC88","pid":2316,"port":80,"process_guid":"00000007-0000-090c-01d1-2099b8f18a82","protocol":6,"remote_ip":"23.4.187.27","remote_port":80,"sensor_id":7,"timestamp":1447697666,"type":"ingress.event.netconn"}
+{"cb_server":"cbserver","computer_name":"JASON-WIN81-VM","event_type":"modload","link_process":"https://cbtests/#analyze/00000001-0000-07b4-01d1-209a100bc217/1","link_sensor":"https://cbtests/#/host/1","md5":"3D136E8D4C0407D9C40FD8BDD649B587","path":"c:\\windows\\system32\\ntdll.dll","pid":1972,"process_guid":"00000001-0000-07b4-01d1-209a100bc217","sensor_id":1,"timestamp":1447697423,"type":"ingress.event.moduleload"}
+{"cb_server":"cbserver","child_process_guid":"00000001-0000-07b4-01d1-209a100bc217","computer_name":"JASON-WIN81-VM","created":true,"event_type":"childproc","link_child":"https://cbtests/#analyze/00000001-0000-07b4-01d1-209a100bc217/1","link_process":"https://cbtests/#analyze/00000001-0000-0af4-01d1-1e444bf4c3dd/1","link_sensor":"https://cbtests/#/host/1","md5":"D6021013D7C4E248AEB8BED12D3DCC88","pid":2804,"process_guid":"00000001-0000-0af4-01d1-1e444bf4c3dd","sensor_id":1,"timestamp":1447697423,"type":"ingress.event.childproc"}
+{"cb_server":"cbserver","command_line":"\"C:\\Windows\\system32\\SearchProtocolHost.exe\" Global\\UsGthrFltPipeMssGthrPipe253_ Global\\UsGthrCtrlFltPipeMssGthrPipe253 1 -2147483646 \"Software\\Microsoft\\Windows Search\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)\" \"C:\\ProgramData\\Microsoft\\Search\\Data\\Temp\\usgthrsvc\" \"DownLevelDaemon\" ","computer_name":"JASON-WIN81-VM","event_type":"proc","expect_followon_w_md5":false,"link_parent":"https://cbtests/#analyze/00000001-0000-0af4-01d1-1e444bf4c3dd/1","link_process":"https://cbtests/#analyze/00000001-0000-07b4-01d1-209a100bc217/1","link_sensor":"https://cbtests/#/host/1","md5":"D6021013D7C4E248AEB8BED12D3DCC88","parent_create_time":1447440685,"parent_md5":"79227C1E2225DE455F365B607A6D46FB","parent_path":"c:\\windows\\system32\\searchindexer.exe","parent_process_guid":"00000001-0000-0af4-01d1-1e444bf4c3dd","path":"c:\\windows\\system32\\searchprotocolhost.exe","pid":1972,"process_guid":"00000001-0000-07b4-01d1-209a100bc217","sensor_id":1,"timestamp":1447697423,"type":"ingress.event.procstart","username":"SYSTEM"}
+{"cb_server":"cbserver","computer_name":"WIN-OTEMNUTBS23","cross_process_type":"open_process","event_type":"cross_process","is_target":false,"link_process":"https://cbtests/#analyze/00000007-0000-0ccc-01d1-209ab5339f45/1","link_sensor":"https://cbtests/#/host/7","link_target":"https://cbtests/#analyze/00000007-0000-02c4-01d1-20982cef85d3/1","md5":"053EEEE1ABAE53F044F1E386E22AE525","pid":3276,"process_guid":"00000007-0000-0ccc-01d1-209ab5339f45","requested_access":5136,"sensor_id":7,"target_create_time":130921702131467730,"target_md5":"382100E75B6F4668AEAEF228C6CEFFAD","target_path":"c:\\windows\\system32\\lsass.exe","target_pid":708,"target_process_guid":"00000007-0000-02c4-01d1-20982cef85d3","timestamp":1447697702,"type":"ingress.event.crossprocopen"}
+{"blocked":true,"cb_server":"cbserver","computer_name":"JASON-WIN81-VM","emet_timestamp":130949318600000000,"event_type":"emet_mitigation","link_process":"https://cbtests/#analyze/00000001-0000-0d10-01d1-39b621f894f9/1","link_sensor":"https://cbtests/#/host/1","log_id":1032,"log_message":"EMET detected EAF mitigation and will close the application: EMET_Test64.exe\r\n\r\nEAF check failed:\n  Application \t: C:\\Users\\dan\\Desktop\\EMET_TEST\\EMET_Test64.exe\n  User Name \t: DANWIN764\\dan\n  Session ID \t: 1\n  PID \t\t: 0xD10 (3344)\n  TID \t\t: 0xDB4 (3508)\n  Module \t: N/A\n  Mod Base \t: 0x0000000000000000\n  Mod Address \t: 0x000000000297000D\n  Mem Address \t: 0x0000000000000000\n\r\n","md5":"053EEEE1ABAE53F044F1E386E22AE525","mitigation":"Eaf","pid":3344,"process_guid":"00000001-0000-0d10-01d1-39b621f894f9","sensor_id":1,"timestamp":1450458260,"type":"ingress.event.emetmitigation"}
+{"blocked_event":"ProcessCreate","blocked_reason":"Md5Hash","blocked_result":"ProcessTerminated","cb_server":"cbserver","command_line":"\"C:\\Program Files\\Microsoft Games\\hearts\\hearts.exe\" ","computer_name":"JASON-WIN81-VM","event_type":"blocked_process","md5":"A8524F6C3AFF774911BCA26AB8322602","path":"c:\\program files\\microsoft games\\hearts\\hearts.exe","sensor_id":1,"timestamp":1450470603,"type":"ingress.event.processblock","uid":"S-1-5-21-3382350439-2970772701-2583938045-1000","username":"DANWIN764\\dan"}
+{"cb_server":"cbserver","computer_name":"JASON-WIN81-VM","event_type":"tamper","sensor_id":1,"tamper_type":"CbProcessTerminated","timestamp":1450470455,"type":"ingress.event.tamper"}

packages/carbonback_edr/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: initial release
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/1527

packages/carbonback_edr/data_stream/log/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,11 @@
+dynamic_fields:
+  "event.ingested": ".*"
+numeric_keyword_fields:
+  - carbonblack.edr.actiontype
+  - carbonblack.edr.feed_id
+  - carbonblack.edr.filetype
+  - carbonblack.edr.log_id
+  - carbonblack.edr.protocol
+  - carbonblack.edr.segment_id
+  - carbonblack.edr.sensor_id
+  - rule.id