[sei packages] Undefined log.file.* fields breaking tests for filestream inputs

packages/cisco_ise/changelog.yml

@@ -1,11 +1,16 @@
 # newer versions go on top
+- version: 1.17.0
+  changes:
+    - description: Adapt fields for changes in file system info
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8014
 - version: 1.16.0
   changes:
     - description: ECS version updated to 8.10.0.
       type: enhancement
       link: https://github.com/elastic/integrations/pull/7905
 - version: 1.15.0
-  changes: 
+  changes:
     - description: "The format_version in the package manifest changed from 2.11.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest."
       type: enhancement
       link: https://github.com/elastic/integrations/pull/7883

packages/cisco_ise/data_stream/log/_dev/test/system/test-filestream-config.yml

@@ -6,3 +6,9 @@ data_stream:
     preserve_duplicate_custom_fields: true
     paths:
       - '{{SERVICE_LOGS_DIR}}/*.log'
+numeric_keyword_fields:
+  - log.file.device_id
+  - log.file.inode
+  - log.file.idxhi
+  - log.file.idxlo
+  - log.file.vol

packages/cisco_ise/data_stream/log/fields/agent.yml

@@ -175,3 +175,24 @@
 - name: log.offset
   type: long
   description: Log offset
+- name: log.file
+  type: group
+  fields:
+    - name: device_id
+      type: keyword
+      description: ID of the device containing the filesystem where the file resides.
+    - name: fingerprint
+      type: keyword
+      description: The sha256 fingerprint identity of the file when fingerprinting is enabled.
+    - name: inode
+      type: keyword
+      description: Inode number of the log file.
+    - name: idxhi
+      type: keyword
+      description: The high-order part of a unique identifier that is associated with a file. (Windows-only)
+    - name: idxlo
+      type: keyword
+      description: The low-order part of a unique identifier that is associated with a file. (Windows-only)
+    - name: vol
+      type: keyword
+      description: The serial number of the volume that contains a file. (Windows-only)

packages/cisco_ise/data_stream/log/sample_event.json

@@ -1,181 +1,181 @@
 {
-    "@timestamp": "2020-04-27T11:11:47.028-08:00",
+    "@timestamp": "2020-02-21T19:13:08.328Z",
     "agent": {
-        "ephemeral_id": "86f518cd-51e3-4798-9fa5-e8947dc5d209",
-        "id": "5607d6f4-6e45-4c33-a087-2e07de5f0082",
+        "ephemeral_id": "1c70d737-7545-456d-8fb9-7033dca67ed3",
+        "id": "901f4c48-583a-4848-aa7b-89dc8e9c4b76",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.9.1"
+        "version": "8.10.2"
     },
     "cisco_ise": {
         "log": {
             "acct": {
-                "authentic": "RADIUS",
-                "session": {
-                    "id": "00000000/d4:ca:6d:14:87:3b/20879"
-                },
-                "status": {
-                    "type": "Start"
+                "request": {
+                    "flags": "Stop"
                 }
             },
             "acs": {
                 "session": {
-                    "id": "hijk.xyz.com/176956368/1092777"
-                }
-            },
-            "airespace": {
-                "wlan": {
-                    "id": 1
-                }
-            },
-            "allowed_protocol": {
-                "matched": {
-                    "rule": "Default"
+                    "id": "ldnnacpsn1/359344348/952729"
                 }
             },
-            "called_station": {
-                "id": "00-24-97-69-7a-c0"
-            },
-            "calling_station": {
-                "id": "d4-ca-6d-14-87-3b"
+            "authen_method": "TacacsPlus",
+            "avpair": {
+                "priv_lvl": 15,
+                "start_time": "2020-03-26T01:17:12.000Z",
+                "task_id": "2962",
+                "timezone": "GMT"
             },
             "category": {
-                "name": "CISE_RADIUS_Accounting"
+                "name": "CISE_TACACS_Accounting"
             },
-            "class": "CACS:0a2025060001794f52cfa877:hijk.xyz.com/176956368/1092772",
+            "cmdset": "[ CmdAV=show mac-address-table <cr> ]",
             "config_version": {
-                "id": 33
+                "id": 1829
             },
             "cpm": {
                 "session": {
-                    "id": "0a222bc0000000d123e111f0"
+                    "id": "81.2.69.144Accounting306034364"
                 }
             },
-            "event": {
-                "timestamp": "2014-01-10T07:59:55.000Z"
-            },
-            "framed": {
-                "ip": "81.2.69.145"
+            "device": {
+                "type": [
+                    "Device Type#All Device Types#Routers",
+                    "Device Type#All Device Types#Routers"
+                ]
             },
-            "location": "Location#All Locations#SJC#WNBU",
+            "ipsec": [
+                "IPSEC#Is IPSEC Device",
+                "IPSEC#Is IPSEC Device"
+            ],
+            "location": [
+                "Location#All Locations#EMEA",
+                "Location#All Locations#EMEA"
+            ],
             "message": {
-                "code": "3000",
-                "description": "Radius-Accounting: RADIUS Accounting start request",
-                "id": "0000070618"
-            },
-            "nas": {
-                "identifier": "Acme_fe:56:00",
-                "ip": "81.2.69.145",
-                "port": {
-                    "number": 13,
-                    "type": "Wireless - IEEE 802.11"
-                }
+                "code": "3300",
+                "description": "Tacacs-Accounting: TACACS+ Accounting with Command",
+                "id": "0000000001"
+            },
+            "model": {
+                "name": "Unknown"
             },
             "network": {
                 "device": {
                     "groups": [
-                        "Location#All Locations#SJC#WNBU",
-                        "Device Type#All Device Types#Wireless#WLC"
+                        "Location#All Locations#EMEA",
+                        "Device Type#All Device Types#Routers",
+                        "IPSEC#Is IPSEC Device"
                     ],
-                    "name": "WNBU-WLC1"
+                    "name": "wlnwan1",
+                    "profile": [
+                        "Cisco",
+                        "Cisco"
+                    ]
                 }
             },
+            "port": "tty10",
+            "privilege": {
+                "level": 15
+            },
             "request": {
-                "latency": 6
+                "latency": 1
+            },
+            "response": {
+                "AcctReply-Status": "Success"
             },
             "segment": {
                 "number": 0,
-                "total": 1
+                "total": 4
             },
             "selected": {
                 "access": {
-                    "service": "Default Network Access"
+                    "service": "Device Admin - TACACS"
                 }
             },
+            "service": {
+                "argument": "shell",
+                "name": "Login"
+            },
+            "software": {
+                "version": "Unknown"
+            },
             "step": [
-                "11004",
-                "11017",
+                "13006",
                 "15049",
                 "15008",
                 "15048",
-                "15048",
-                "15048",
-                "15004",
-                "15006",
-                "11005"
+                "13035"
             ],
-            "tunnel": {
-                "medium": {
-                    "type": "(tag=0) 802"
-                },
-                "private": {
-                    "group_id": "(tag=0) 70"
-                },
-                "type": "(tag=0) VLAN"
-            }
+            "type": "Accounting"
         }
     },
     "client": {
-        "ip": "81.2.69.145"
+        "ip": "81.2.69.144"
     },
     "data_stream": {
         "dataset": "cisco_ise.log",
         "namespace": "ep",
         "type": "logs"
     },
+    "destination": {
+        "ip": "81.2.69.144"
+    },
     "ecs": {
         "version": "8.10.0"
     },
     "elastic_agent": {
-        "id": "5607d6f4-6e45-4c33-a087-2e07de5f0082",
+        "id": "901f4c48-583a-4848-aa7b-89dc8e9c4b76",
         "snapshot": false,
-        "version": "8.9.1"
+        "version": "8.10.2"
     },
     "event": {
-        "action": "radius-accounting",
+        "action": "tacacs-accounting",
         "agent_id_status": "verified",
         "category": [
             "configuration"
         ],
         "dataset": "cisco_ise.log",
-        "ingested": "2023-08-29T17:11:24Z",
+        "ingested": "2023-10-03T09:31:56Z",
         "kind": "event",
-        "original": "\u003c182\u003eApr 27 11:11:47 hijk.xyz.com CISE_RADIUS_Accounting 0000070618 1 0 2020-04-27 11:11:47.028075 -08:00 0091827141 3000 NOTICE Radius-Accounting: RADIUS Accounting start request, ConfigVersionId=33, Device IP Address=81.2.69.145, RequestLatency=6, NetworkDeviceName=WNBU-WLC1, User-Name=nisehorrrrn, NAS-IP-Address=81.2.69.145, NAS-Port=13, Framed-IP-Address=81.2.69.145, Class=CACS:0a2025060001794f52cfa877:hijk.xyz.com/176956368/1092772, Called-Station-ID=00-24-97-69-7a-c0, Calling-Station-ID=d4-ca-6d-14-87-3b, NAS-Identifier=Acme_fe:56:00, Acct-Status-Type=Start, Acct-Session-Id=00000000/d4:ca:6d:14:87:3b/20879, Acct-Authentic=RADIUS, Event-Timestamp=1389340795, NAS-Port-Type=Wireless - IEEE 802.11, Tunnel-Type=(tag=0) VLAN, Tunnel-Medium-Type=(tag=0) 802, Tunnel-Private-Group-ID=(tag=0) 70, Airespace-Wlan-Id=1, AcsSessionID=hijk.xyz.com/176956368/1092777, SelectedAccessService=Default Network Access, Step=11004, Step=11017, Step=15049, Step=15008, Step=15048, Step=15048, Step=15048, Step=15004, Step=15006, Step=11005, NetworkDeviceGroups=Location#All Locations#SJC#WNBU, NetworkDeviceGroups=Device Type#All Device Types#Wireless#WLC, CPMSessionID=0a222bc0000000d123e111f0, AllowedProtocolMatchedRule=Default, Location=Location#All Locations#SJC#WNBU, Device Type=Device Type#All Device Types#Wireless#WLC",
-        "sequence": 91827141,
-        "timezone": "-08:00",
+        "original": "<182>Feb 21 19:13:08 cisco-ise-host CISE_TACACS_Accounting 0000000001 4 0 2020-02-21 19:13:08.328 +00:00 0018415781 3300 NOTICE Tacacs-Accounting: TACACS+ Accounting with Command, ConfigVersionId=1829, Device IP Address=81.2.69.144, CmdSet=[ CmdAV=show mac-address-table <cr> ], RequestLatency=1, NetworkDeviceName=wlnwan1, Type=Accounting, Privilege-Level=15, Service=Login, User=psxvne, Port=tty10, Remote-Address=81.2.69.144, Authen-Method=TacacsPlus, AVPair=task_id=2962, AVPair=timezone=GMT, AVPair=start_time=1585185432, AVPair=priv-lvl=15, AcctRequest-Flags=Stop, Service-Argument=shell, AcsSessionID=ldnnacpsn1/359344348/952729, SelectedAccessService=Device Admin - TACACS, Step=13006, Step=15049, Step=15008, Step=15048, Step=13035, NetworkDeviceGroups=Location#All Locations#EMEA, NetworkDeviceGroups=Device Type#All Device Types#Routers, NetworkDeviceGroups=IPSEC#Is IPSEC Device, CPMSessionID=81.2.69.144Accounting306034364, Model Name=Unknown, Software Version=Unknown, Network Device Profile=Cisco, Location=Location#All Locations#EMEA, Device Type=Device Type#All Device Types#Routers, IPSEC=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success; }, Network Device Profile=Cisco, Location=Location#All Locations#EMEA, Device Type=Device Type#All Device Types#Routers, IPSEC=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success; }",
+        "sequence": 18415781,
+        "timezone": "+00:00",
         "type": [
             "info"
         ]
     },
     "host": {
-        "hostname": "hijk.xyz.com"
+        "hostname": "cisco-ise-host"
     },
     "input": {
         "type": "filestream"
     },
     "log": {
         "file": {
+            "device_id": 2080,
+            "inode": 88860,
             "path": "/tmp/service_logs/log.log"
         },
         "level": "notice",
-        "offset": 44899,
+        "offset": 71596,
         "syslog": {
             "priority": 182,
             "severity": {
                 "name": "notice"
             }
         }
     },
-    "message": "2020-04-27 11:11:47.028075 -08:00 0091827141 3000 NOTICE Radius-Accounting: RADIUS Accounting start request, ConfigVersionId=33, Device IP Address=81.2.69.145, RequestLatency=6, NetworkDeviceName=WNBU-WLC1, User-Name=nisehorrrrn, NAS-IP-Address=81.2.69.145, NAS-Port=13, Framed-IP-Address=81.2.69.145, Class=CACS:0a2025060001794f52cfa877:hijk.xyz.com/176956368/1092772, Called-Station-ID=00-24-97-69-7a-c0, Calling-Station-ID=d4-ca-6d-14-87-3b, NAS-Identifier=Acme_fe:56:00, Acct-Status-Type=Start, Acct-Session-Id=00000000/d4:ca:6d:14:87:3b/20879, Acct-Authentic=RADIUS, Event-Timestamp=1389340795, NAS-Port-Type=Wireless - IEEE 802.11, Tunnel-Type=(tag=0) VLAN, Tunnel-Medium-Type=(tag=0) 802, Tunnel-Private-Group-ID=(tag=0) 70, Airespace-Wlan-Id=1, AcsSessionID=hijk.xyz.com/176956368/1092777, SelectedAccessService=Default Network Access, Step=11004, Step=11017, Step=15049, Step=15008, Step=15048, Step=15048, Step=15048, Step=15004, Step=15006, Step=11005, NetworkDeviceGroups=Location#All Locations#SJC#WNBU, NetworkDeviceGroups=Device Type#All Device Types#Wireless#WLC, CPMSessionID=0a222bc0000000d123e111f0, AllowedProtocolMatchedRule=Default, Location=Location#All Locations#SJC#WNBU, Device Type=Device Type#All Device Types#Wireless#WLC",
+    "message": "2020-02-21 19:13:08.328 +00:00 0018415781 3300 NOTICE Tacacs-Accounting: TACACS+ Accounting with Command, ConfigVersionId=1829, Device IP Address=81.2.69.144, CmdSet=[ CmdAV=show mac-address-table <cr> ], RequestLatency=1, NetworkDeviceName=wlnwan1, Type=Accounting, Privilege-Level=15, Service=Login, User=psxvne, Port=tty10, Remote-Address=81.2.69.144, Authen-Method=TacacsPlus, AVPair=task_id=2962, AVPair=timezone=GMT, AVPair=start_time=1585185432, AVPair=priv-lvl=15, AcctRequest-Flags=Stop, Service-Argument=shell, AcsSessionID=ldnnacpsn1/359344348/952729, SelectedAccessService=Device Admin - TACACS, Step=13006, Step=15049, Step=15008, Step=15048, Step=13035, NetworkDeviceGroups=Location#All Locations#EMEA, NetworkDeviceGroups=Device Type#All Device Types#Routers, NetworkDeviceGroups=IPSEC#Is IPSEC Device, CPMSessionID=81.2.69.144Accounting306034364, Model Name=Unknown, Software Version=Unknown, Network Device Profile=Cisco, Location=Location#All Locations#EMEA, Device Type=Device Type#All Device Types#Routers, IPSEC=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success; }, Network Device Profile=Cisco, Location=Location#All Locations#EMEA, Device Type=Device Type#All Device Types#Routers, IPSEC=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success; }",
     "related": {
         "hosts": [
-            "hijk.xyz.com"
+            "cisco-ise-host"
         ],
         "ip": [
-            "81.2.69.145"
+            "81.2.69.144"
         ],
         "user": [
-            "nisehorrrrn"
+            "psxvne"
         ]
     },
     "tags": [
@@ -184,6 +184,6 @@
         "cisco_ise-log"
     ],
     "user": {
-        "name": "nisehorrrrn"
+        "name": "psxvne"
     }
 }