[integrations][CrowdStrike] - Fixed Windows NT timestamp parsing issu…

…e and IDP Log pipeline field naming issue (#7548) * fixed windows NT timestamp issue and idp logs pipeline issues * updated changelog * updated comments and implemented PR sugestions * In Windows NT, the timestamp is a 64-bit value representing the number of 100-nanosecond intervals that have elapsed since January 1, 1601 (UTC). The high bit of this value is set to 1 to indicate that it's a Windows NT timestamp. The condition longValue > 0x0100000000000000L checks if the high bit of the 64-bit value is set, which indicates that it's a Windows NT timestamp. If this condition is true, it means the timestamp is a Windows NT timestamp and needs to be converted to a Unix timestamp. The conversion involves dividing the value by 10000000 to convert from 100-nanosecond intervals to seconds and then subtracting the offset 11644473600L to account for the difference between Windows NT and Unix epoch times. * made PR suggested changes * updated comments
elastic · Sep 5, 2023 · 64eea1b · 64eea1b
1 parent d3df265
commit 64eea1b
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 9 deletions.
diff --git a/packages/crowdstrike/changelog.yml b/packages/crowdstrike/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.18.1"
+  changes:
+    - description: Fixed Windows NT timestamp handling.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/7548
 - version: "1.18.0"
   changes:
     - description: Update package to ECS 8.9.0.

diff --git a/...owdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-ipd-summary.log-expected.json b/...owdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-ipd-summary.log-expected.json
@@ -5,10 +5,10 @@
             "crowdstrike": {
                 "event": {
                     "ActivityId": "12345678-1234-1234-1234-123456789000",
-                    "MostRecentActivityTimeStamp": 133313215755670000,
+                    "MostRecentActivityTimeStamp": 1686847975,
                     "Objective": "Gain Access",
                     "PatternId": "51135",
-                    "PrecedingActivityTimeStamp": 133154452345780000,
+                    "PrecedingActivityTimeStamp": 1670971634,
                     "SourceEndpointAccountObjectGuid": "12345678-1234-1234-1234-123456789000",
                     "SourceEndpointAccountObjectSid": "S-1-3-44-55555555-666666666-7777777777-88888",
                     "SourceEndpointSensorId": "12345678901234567890123456789012"
@@ -28,12 +28,13 @@
                 "category": [
                     "malware"
                 ],
-                "created": "+4223582-11-13T00:53:20.000Z",
+                "created": "2023-03-01T05:50:56.000Z",
+                "end": "1992-03-21T19:15:00.000Z",
                 "kind": "alert",
                 "original": "{\n\t\"metadata\": {\n\t\t\"customerIDString\": \"12312312312312321\",\n\t\t\"offset\": 2662765,\n\t\t\"eventType\": \"IdpDetectionSummaryEvent\",\n\t\t\"eventCreationTime\": 1686848064000,\n\t\t\"version\": \"1.0\"\n\t},\n\t\"event\": {\n\t\t\"ContextTimeStamp\": 133221234560000000,\n\t\t\"DetectId\": \"12345678901234567890123456789012:ind:12345678901234567890123456789012:12345678-1234-1234-1234-123456789000\",\n\t\t\"DetectName\": \"Unusual login to an endpoint\",\n\t\t\"DetectDescription\": \"A user logged in to a machine for the first time\",\n\t\t\"FalconHostLink\": \"https://falcon.crowdstrike.com/identity-protection/detections/12345678901234567890123456789012:ind:12345678901234567890123456789012:12345678-1234-1234-1234-123456789000?cid=12345678901234567890123456789012\",\n\t\t\"StartTime\": 123456789000000000,\n\t\t\"EndTime\": 123456789000000000,\n\t\t\"Severity\": 7,\n\t\t\"Tactic\": \"Initial Access\",\n\t\t\"Technique\": \"Valid Accounts\",\n\t\t\"Objective\": \"Gain Access\",\n\t\t\"SourceAccountDomain\": \"DOMAIN.COM\",\n\t\t\"SourceAccountName\": \"johnb\",\n\t\t\"SourceAccountObjectSid\": \"S-1-3-44-55555555-666666666-7777777777-88888\",\n\t\t\"SourceEndpointAccountObjectGuid\": \"12345678-1234-1234-1234-123456789000\",\n\t\t\"SourceEndpointAccountObjectSid\": \"S-1-3-44-55555555-666666666-7777777777-88888\",\n\t\t\"SourceEndpointHostName\": \"pc01.domain.com\",\n\t\t\"SourceEndpointIpAddress\": \"81.2.69.144\",\n\t\t\"SourceEndpointSensorId\": \"12345678901234567890123456789012\",\n\t\t\"PrecedingActivityTimeStamp\": 133154452345780000,\n\t\t\"MostRecentActivityTimeStamp\": 133313215755670000,\n\t\t\"ActivityId\": \"12345678-1234-1234-1234-123456789000\",\n\t\t\"PatternId\": 51135\n\t}\n}",
                 "reference": "https://falcon.crowdstrike.com/identity-protection/detections/12345678901234567890123456789012:ind:12345678901234567890123456789012:12345678-1234-1234-1234-123456789000?cid=12345678901234567890123456789012",
                 "severity": 7,
-                "start": "+3914159-11-26T20:00:00.000Z",
+                "start": "1992-03-21T19:15:00.000Z",
                 "type": [
                     "info"
                 ]

diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/default.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/default.yml
@@ -35,7 +35,49 @@ processors:
       type: string
       tag: convert_pattern_id
       ignore_missing: true
+  # Script to convert windows NT timestamp to unix timestamp
+  - script:
+        tag: convert-nt-timestamp-to-unix
+        description: Convert Windows NT timestamps to UNIX timestamps for multiple fields.
+        lang: painless
+        if : ctx.crowdstrike?.event != null
+        params:
+            values:
+                - 'StartTime'
+                - 'EndTime'
+                - 'ContextTimeStamp'
+                - 'EndTimestamp'
+                - 'IncidentEndTime'
+                - 'IncidentStartTime'
+                - 'ItemPostedTimestamp'
+                - 'MatchedTimestamp'
+                - 'MostRecentActivityTimeStamp'
+                - 'PrecedingActivityTimeStamp'
+                - 'StartTimestamp'
+                - 'UTCTimestamp'
+# Process to convert LDAP/WIN32 FILETIME to Unix timestamp. 
+# More details can be found here https://devblogs.microsoft.com/oldnewthing/20030905-02/?p=42653 and here https://www.epochconverter.com/ldap
+        source: |
+            def convertToUnix(def longValue) {
+                if (longValue > 0x0100000000000000L) {
+                    return (longValue / 10000000) - 11644473600L;
+                }
+                return longValue;
+            }
 
+            for (def field : params.values) {
+                def fieldValue = ctx.crowdstrike.event[field];
+                if (fieldValue != null) {
+                    if (fieldValue instanceof long) {
+                         ctx.crowdstrike.event[field] = convertToUnix(fieldValue);
+                    } else if (fieldValue instanceof String) {
+                        if (!fieldValue.contains('.')) {
+                           def timestamp = Long.parseLong(fieldValue);
+                            ctx.crowdstrike.event[field] = convertToUnix(timestamp);
+                        }
+                    }
+                } 
+            }
   # UTCTimestamp should exist in each event, however on the off-chance it might not be (Like RemoteSession Start/End), then we have to use eventCreation time.
   - date:
       field: crowdstrike.event.UTCTimestamp

diff --git a/...es/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/ipd_detection_summary.yml b/...es/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/ipd_detection_summary.yml
@@ -211,19 +211,19 @@ processors:
       if: "ctx.crowdstrike?.event?.EndTime != null && ctx.crowdstrike.event.EndTime.length() > 18"
   - date:
       field: crowdstrike.event.EndTime
-      target_field: event.start
+      target_field: event.end
       timezone: UTC
       formats:
         - UNIX_MS
-      tag: date_event_start_time_epoch
+      tag: date_event_end_time_epoch
       if: "ctx.crowdstrike?.event?.EndTime != null && ctx.crowdstrike.event.EndTime.length() >= 12"
   - date:
       field: crowdstrike.event.EndTime
-      target_field: event.start
+      target_field: event.end
       timezone: UTC
       formats:
         - UNIX
-      tag: date_event_start_time_epoch
+      tag: date_event_end_time_epoch
       if: 'ctx.crowdstrike?.event?.EndTime != null && ctx.crowdstrike.event.EndTime.length() <= 11'
   - append:
       field: related.hosts

diff --git a/packages/crowdstrike/manifest.yml b/packages/crowdstrike/manifest.yml
@@ -1,6 +1,6 @@
 name: crowdstrike
 title: CrowdStrike
-version: "1.18.0"
+version: "1.18.1"
 description: Collect logs from Crowdstrike with Elastic Agent.
 type: integration
 format_version: 2.7.0