[m365_defender] Fix log data stream cursor and query (#8492)

* Fix log data stream cursor and query * Remove unnecessary empty check
elastic · Jan 25, 2024 · eba1358 · eba1358
1 parent 8894c9c
commit eba1358
Show file tree

Hide file tree

Showing 8 changed files with 81 additions and 22 deletions.
diff --git a/packages/m365_defender/_dev/deploy/docker/docker-compose.yml b/packages/m365_defender/_dev/deploy/docker/docker-compose.yml
@@ -1,14 +1,28 @@
 version: '2.3'
 services:
-  m365-defender-http:
-    image: docker.elastic.co/observability/stream:v0.8.0
+  m365-defender-log-http:
+    image: docker.elastic.co/observability/stream:v0.13.0
     ports:
       - 8080
     volumes:
-      - ./http-mock-config.yml:/config.yml
+      - ./log-http-mock-config.yml:/config.yml
     environment:
       PORT: 8080
     command:
       - http-server
+      - --exit-on-unmatched-rule
+      - --addr=:8080
+      - --config=/config.yml
+  m365-defender-incident-http:
+    image: docker.elastic.co/observability/stream:v0.13.0
+    ports:
+      - 8080
+    volumes:
+      - ./incident-http-mock-config.yml:/config.yml
+    environment:
+      PORT: 8080
+    command:
+      - http-server
+      - --exit-on-unmatched-rule
       - --addr=:8080
       - --config=/config.yml
diff --git a/packages/m365_defender/_dev/deploy/docker/incident-http-mock-config.yml b/packages/m365_defender/_dev/deploy/docker/incident-http-mock-config.yml
@@ -0,0 +1,27 @@
+rules:
+  - path: /tenant_id/oauth2/v2.0/token
+    methods: [POST]
+    query_params:
+      grant_type: client_credentials
+    request_headers:
+      Content-Type:
+        - "application/x-www-form-urlencoded"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {"access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN","token_type": "Bearer","not_before": 1549647431,"expires_in": 3600}
+  - path: /v1.0/security/incidents
+    methods: [GET]
+    request_headers:
+      Authorization:
+        - "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |
+          {"value":[{"@odata.type":"#microsoft.graph.security.incident","id":"2972395","incidentWebUrl":"https://security.microsoft.com/incidents/2972395?tid=12f988bf-16f1-11af-11ab-1d7cd011db47","redirectIncidentId":null,"tenantId":"b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c","displayName":"Multi-stage incident involving Initial access & Command and control on multiple endpoints reported by multiple sources","createdDateTime":"2021-08-13T08:43:35.5533333Z","lastUpdateDateTime":"2021-09-30T09:35:45.1133333Z","assignedTo":"KaiC@contoso.onmicrosoft.com","classification":"truePositive","determination":"multiStagedAttack","status":"active","severity":"medium","tags":["Demo"],"comments":[{"comment":"Demo incident","createdBy":"DavidS@contoso.onmicrosoft.com","createdTime":"2021-09-30T12:07:37.2756993Z"}],"alerts":[{"@odata.type":"#microsoft.graph.security.alert","id":"da637551227677560813_-961444813","providerAlertId":"da637551227677560813_-961444813","incidentId":"28282","status":"new","severity":"low","classification":"unknown","determination":"unknown","serviceSource":"microsoftDefenderForEndpoint","detectionSource":"antivirus","detectorId":"e0da400f-affd-43ef-b1d5-afc2eb6f2756","tenantId":"b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c","title":"Suspicious execution of hidden file","description":"A hidden file has been launched. This activity could indicate a compromised host. Attackers often hide files associated with malicious tools to evade file system inspection and defenses.","recommendedActions":"Collect artifacts and determine scope\n�\tReview the machine timeline for suspicious activities that may have occurred before and after the time of the alert, and record additional related artifacts (files, IPs/URLs) \n�\tLook for the presence of relevant artifacts on other systems. Identify commonalities and differences between potentially compromised systems.\n�\tSubmit relevant files for deep analysis and review resulting detailed behavioral information.\n�\tSubmit undetected files to the MMPC malware portal\n\nInitiate containment & mitigation \n�\tContact the user to verify intent and initiate local remediation actions as needed.\n�\tUpdate AV signatures and run a full scan. The scan might reveal and remove previously-undetected malware components.\n�\tEnsure that the machine has the latest security updates. In particular, ensure that you have installed the latest software, web browser, and Operating System versions.\n�\tIf credential theft is suspected, reset all relevant users passwords.\n�\tBlock communication with relevant URLs or IPs at the organization�s perimeter.","category":"DefenseEvasion","assignedTo":null,"alertWebUrl":"https://security.microsoft.com/alerts/da637551227677560813_-961444813?tid=b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c","incidentWebUrl":"https://security.microsoft.com/incidents/28282?tid=b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c","actorDisplayName":null,"threatDisplayName":null,"threatFamilyName":null,"mitreTechniques":["T1564.001"],"createdDateTime":"2021-04-27T12:19:27.7211305Z","lastUpdateDateTime":"2021-05-02T14:19:01.3266667Z","resolvedDateTime":null,"firstActivityDateTime":"2021-04-26T07:45:50.116Z","lastActivityDateTime":"2021-05-02T07:56:58.222Z","comments":[],"evidence":[{"@odata.type":"#microsoft.graph.security.deviceEvidence","createdDateTime":"2021-04-27T12:19:27.7211305Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"firstSeenDateTime":"2020-09-12T07:28:32.4321753Z","mdeDeviceId":"73e7e2de709dff64ef64b1d0c30e67fab63279db","azureAdDeviceId":null,"deviceDnsName":"tempDns","osPlatform":"Windows10","osBuild":22424,"version":"Other","healthStatus":"active","riskScore":"medium","rbacGroupId":75,"rbacGroupName":"UnassignedGroup","onboardingStatus":"onboarded","defenderAvStatus":"unknown","loggedOnUsers":[],"roles":["compromised"],"tags":["Test Machine"],"vmMetadata":{"vmId":"ca1b0d41-5a3b-4d95-b48b-f220aed11d78","cloudProvider":"azure","resourceId":"/subscriptions/8700d3a3-3bb7-4fbe-a090-488a1ad04161/resourceGroups/WdatpApi-EUS-STG/providers/Microsoft.Compute/virtualMachines/NirLaviTests","subscriptionId":"8700d3a3-3bb7-4fbe-a090-488a1ad04161"}},{"@odata.type":"#microsoft.graph.security.fileEvidence","createdDateTime":"2021-04-27T12:19:27.7211305Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"detectionStatus":"detected","mdeDeviceId":"73e7e2de709dff64ef64b1d0c30e67fab63279db","roles":[],"tags":[],"fileDetails":{"sha1":"5f1e8acedc065031aad553b710838eb366cfee9a","sha256":"8963a19fb992ad9a76576c5638fd68292cffb9aaac29eb8285f9abf6196a7dec","fileName":"MsSense.exe","filePath":"C:\\Program Files\\temp","fileSize":6136392,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null}},{"@odata.type":"#microsoft.graph.security.processEvidence","createdDateTime":"2021-04-27T12:19:27.7211305Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"processId":4780,"parentProcessId":668,"processCommandLine":"\"MsSense.exe\"","processCreationDateTime":"2021-08-12T12:43:19.0772577Z","parentProcessCreationDateTime":"2021-08-12T07:39:09.0909239Z","detectionStatus":"detected","mdeDeviceId":"73e7e2de709dff64ef64b1d0c30e67fab63279db","roles":[],"tags":[],"imageFile":{"sha1":"5f1e8acedc065031aad553b710838eb366cfee9a","sha256":"8963a19fb992ad9a76576c5638fd68292cffb9aaac29eb8285f9abf6196a7dec","fileName":"MsSense.exe","filePath":"C:\\Program Files\\temp","fileSize":6136392,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null},"parentProcessImageFile":{"sha1":null,"sha256":null,"fileName":"services.exe","filePath":"C:\\Windows\\System32","fileSize":731744,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null},"userAccount":{"accountName":"SYSTEM","domainName":"NT AUTHORITY","userSid":"S-1-5-18","azureAdUserId":null,"userPrincipalName":null}},{"@odata.type":"#microsoft.graph.security.registryKeyEvidence","createdDateTime":"2021-04-27T12:19:27.7211305Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"registryKey":"SYSTEM\\CONTROLSET001\\CONTROL\\WMI\\AUTOLOGGER\\SENSEAUDITLOGGER","registryHive":"HKEY_LOCAL_MACHINE","roles":[],"tags":[]}]}]}]}
diff --git a/...r/_dev/deploy/docker/http-mock-config.yml → ...ev/deploy/docker/log-http-mock-config.yml b/...r/_dev/deploy/docker/http-mock-config.yml → ...ev/deploy/docker/log-http-mock-config.yml
@@ -1,3 +1,4 @@
+as_sequence: true
 rules:
   - path: /tenant_id/oauth2/v2.0/token
     methods: [POST]
@@ -13,22 +14,10 @@ rules:
             - "application/json"
         body: |-
           {"access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN","token_type": "Bearer","not_before": 1549647431,"expires_in": 3600}
-  - path: /v1.0/security/incidents
-    methods: [GET]
-    request_headers:
-      Authorization:
-        - "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN"
-    responses:
-      - status_code: 200
-        headers:
-          Content-Type:
-            - "application/json"
-        body: |
-          {"value":[{"@odata.type":"#microsoft.graph.security.incident","id":"2972395","incidentWebUrl":"https://security.microsoft.com/incidents/2972395?tid=12f988bf-16f1-11af-11ab-1d7cd011db47","redirectIncidentId":null,"tenantId":"b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c","displayName":"Multi-stage incident involving Initial access & Command and control on multiple endpoints reported by multiple sources","createdDateTime":"2021-08-13T08:43:35.5533333Z","lastUpdateDateTime":"2021-09-30T09:35:45.1133333Z","assignedTo":"KaiC@contoso.onmicrosoft.com","classification":"truePositive","determination":"multiStagedAttack","status":"active","severity":"medium","tags":["Demo"],"comments":[{"comment":"Demo incident","createdBy":"DavidS@contoso.onmicrosoft.com","createdTime":"2021-09-30T12:07:37.2756993Z"}],"alerts":[{"@odata.type":"#microsoft.graph.security.alert","id":"da637551227677560813_-961444813","providerAlertId":"da637551227677560813_-961444813","incidentId":"28282","status":"new","severity":"low","classification":"unknown","determination":"unknown","serviceSource":"microsoftDefenderForEndpoint","detectionSource":"antivirus","detectorId":"e0da400f-affd-43ef-b1d5-afc2eb6f2756","tenantId":"b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c","title":"Suspicious execution of hidden file","description":"A hidden file has been launched. This activity could indicate a compromised host. Attackers often hide files associated with malicious tools to evade file system inspection and defenses.","recommendedActions":"Collect artifacts and determine scope\n�\tReview the machine timeline for suspicious activities that may have occurred before and after the time of the alert, and record additional related artifacts (files, IPs/URLs) \n�\tLook for the presence of relevant artifacts on other systems. Identify commonalities and differences between potentially compromised systems.\n�\tSubmit relevant files for deep analysis and review resulting detailed behavioral information.\n�\tSubmit undetected files to the MMPC malware portal\n\nInitiate containment & mitigation \n�\tContact the user to verify intent and initiate local remediation actions as needed.\n�\tUpdate AV signatures and run a full scan. The scan might reveal and remove previously-undetected malware components.\n�\tEnsure that the machine has the latest security updates. In particular, ensure that you have installed the latest software, web browser, and Operating System versions.\n�\tIf credential theft is suspected, reset all relevant users passwords.\n�\tBlock communication with relevant URLs or IPs at the organization�s perimeter.","category":"DefenseEvasion","assignedTo":null,"alertWebUrl":"https://security.microsoft.com/alerts/da637551227677560813_-961444813?tid=b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c","incidentWebUrl":"https://security.microsoft.com/incidents/28282?tid=b3c1b5fc-828c-45fa-a1e1-10d74f6d6e9c","actorDisplayName":null,"threatDisplayName":null,"threatFamilyName":null,"mitreTechniques":["T1564.001"],"createdDateTime":"2021-04-27T12:19:27.7211305Z","lastUpdateDateTime":"2021-05-02T14:19:01.3266667Z","resolvedDateTime":null,"firstActivityDateTime":"2021-04-26T07:45:50.116Z","lastActivityDateTime":"2021-05-02T07:56:58.222Z","comments":[],"evidence":[{"@odata.type":"#microsoft.graph.security.deviceEvidence","createdDateTime":"2021-04-27T12:19:27.7211305Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"firstSeenDateTime":"2020-09-12T07:28:32.4321753Z","mdeDeviceId":"73e7e2de709dff64ef64b1d0c30e67fab63279db","azureAdDeviceId":null,"deviceDnsName":"tempDns","osPlatform":"Windows10","osBuild":22424,"version":"Other","healthStatus":"active","riskScore":"medium","rbacGroupId":75,"rbacGroupName":"UnassignedGroup","onboardingStatus":"onboarded","defenderAvStatus":"unknown","loggedOnUsers":[],"roles":["compromised"],"tags":["Test Machine"],"vmMetadata":{"vmId":"ca1b0d41-5a3b-4d95-b48b-f220aed11d78","cloudProvider":"azure","resourceId":"/subscriptions/8700d3a3-3bb7-4fbe-a090-488a1ad04161/resourceGroups/WdatpApi-EUS-STG/providers/Microsoft.Compute/virtualMachines/NirLaviTests","subscriptionId":"8700d3a3-3bb7-4fbe-a090-488a1ad04161"}},{"@odata.type":"#microsoft.graph.security.fileEvidence","createdDateTime":"2021-04-27T12:19:27.7211305Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"detectionStatus":"detected","mdeDeviceId":"73e7e2de709dff64ef64b1d0c30e67fab63279db","roles":[],"tags":[],"fileDetails":{"sha1":"5f1e8acedc065031aad553b710838eb366cfee9a","sha256":"8963a19fb992ad9a76576c5638fd68292cffb9aaac29eb8285f9abf6196a7dec","fileName":"MsSense.exe","filePath":"C:\\Program Files\\temp","fileSize":6136392,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null}},{"@odata.type":"#microsoft.graph.security.processEvidence","createdDateTime":"2021-04-27T12:19:27.7211305Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"processId":4780,"parentProcessId":668,"processCommandLine":"\"MsSense.exe\"","processCreationDateTime":"2021-08-12T12:43:19.0772577Z","parentProcessCreationDateTime":"2021-08-12T07:39:09.0909239Z","detectionStatus":"detected","mdeDeviceId":"73e7e2de709dff64ef64b1d0c30e67fab63279db","roles":[],"tags":[],"imageFile":{"sha1":"5f1e8acedc065031aad553b710838eb366cfee9a","sha256":"8963a19fb992ad9a76576c5638fd68292cffb9aaac29eb8285f9abf6196a7dec","fileName":"MsSense.exe","filePath":"C:\\Program Files\\temp","fileSize":6136392,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null},"parentProcessImageFile":{"sha1":null,"sha256":null,"fileName":"services.exe","filePath":"C:\\Windows\\System32","fileSize":731744,"filePublisher":"Microsoft Corporation","signer":null,"issuer":null},"userAccount":{"accountName":"SYSTEM","domainName":"NT AUTHORITY","userSid":"S-1-5-18","azureAdUserId":null,"userPrincipalName":null}},{"@odata.type":"#microsoft.graph.security.registryKeyEvidence","createdDateTime":"2021-04-27T12:19:27.7211305Z","verdict":"unknown","remediationStatus":"none","remediationStatusDetails":null,"registryKey":"SYSTEM\\CONTROLSET001\\CONTROL\\WMI\\AUTOLOGGER\\SENSEAUDITLOGGER","registryHive":"HKEY_LOCAL_MACHINE","roles":[],"tags":[]}]}]}]}
   - path: /api/incidents
     methods: [GET]
     query_params:
-      $filter: "{$filter:.*}"
+      $filter: "lastUpdateTime gt 2020-08-06T12:07:55.32Z"
     request_headers:
       Authorization:
         - "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN"
@@ -625,3 +614,19 @@ rules:
               }
             ]
           }
+  - path: /api/incidents
+    methods: [GET]
+    query_params:
+      $filter: "lastUpdateTime gt 2020-09-06T12:07:55.32Z"
+    request_headers:
+      Authorization:
+        - "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ilg1ZVhrN"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |-
+          {
+            "value": []
+          }
diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.6.2"
+  changes:
+    - description: Fix cursor value and query building for log data stream.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8492
 - version: "2.6.1"
   changes:
     - description: Changed owners

diff --git a/packages/m365_defender/data_stream/incident/_dev/test/system/test-default-config.yml b/packages/m365_defender/data_stream/incident/_dev/test/system/test-default-config.yml
@@ -1,5 +1,5 @@
 input: httpjson
-service: m365-defender-http
+service: m365-defender-incident-http
 vars:
   login_url: http://{{Hostname}}:{{Port}}
   client_id: xxxx

diff --git a/packages/m365_defender/data_stream/log/_dev/test/system/test-httpjson-config.yml b/packages/m365_defender/data_stream/log/_dev/test/system/test-httpjson-config.yml
@@ -1,5 +1,5 @@
 input: httpjson
-service: m365-defender-http
+service: m365-defender-log-http
 vars:
   login_url: http://{{Hostname}}:{{Port}}
   client_id: xxxx
@@ -10,5 +10,7 @@ data_stream:
     request_url: http://{{Hostname}}:{{Port}}
     preserve_original_event: true
     enable_request_tracer: true
+    interval: 10s
+    initial_interval: false
 assert:
   hit_count: 11
diff --git a/packages/m365_defender/data_stream/log/agent/stream/httpjson.yml.hbs b/packages/m365_defender/data_stream/log/agent/stream/httpjson.yml.hbs
@@ -20,10 +20,15 @@ request.transforms:
       value: "MdatpPartner-Elastic-Filebeat/1.0.0"
   - set:
       target: "url.params.$filter"
-      value: 'lastUpdateTime gt [[formatDate .cursor.lastUpdateTime "2006-01-02T15:04:05.9999999Z"]]'
-      default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-{{initial_interval}}")) "2006-01-02T15:04:05.9999999Z"]]'
+      value: 'lastUpdateTime gt [[.cursor.lastUpdateTime]]'
+{{#if initial_interval}}
+      default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-{{initial_interval}}")) "2006-01-02T15:04:05.99Z"]]'
+{{else}}
+      default: 'lastUpdateTime gt 2020-08-06T12:07:55.32Z'
+{{/if}}
 response.split:
   target: body.value
+  ignore_empty_value: true
   split:
     target: body.alerts
     keep_parent: true
@@ -32,7 +37,8 @@ response.split:
       keep_parent: true
 cursor:
   lastUpdateTime:
-    value: "[[.last_response.body.lastUpdateTime]]"
+    value: "[[.last_event.lastUpdateTime]]"
+    ignore_empty_value: true
 
 tags:
 {{#if preserve_original_event}}

diff --git a/packages/m365_defender/manifest.yml b/packages/m365_defender/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: m365_defender
 title: Microsoft M365 Defender
-version: "2.6.1"
+version: "2.6.2"
 description: Collect logs from Microsoft M365 Defender with Elastic Agent.
 categories:
   - "security"