Linux Events Update

Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.xif

@@ -1,44 +1,111 @@
 [MODEL: dataset="linux_linux_raw"]
 filter _log_source_file_name in("auth*", "secure")
-| alter current_year = arrayindex(regextract(to_string(current_time()), "\d{4}"), 0),
-    //timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0),
-    //timestamp = format_string("%s %s", current_year, timestamp),
-    hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)),
-    process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z]+)"), 0)),
-    pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)),
-    event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)),
-    src_ip = arrayindex(regextract(_raw_log,"\s(\d{1,3}(?:\.\d{1,3}){3})"),0),
-    user_name1 = arrayindex(regextract(_raw_log ,"\[USER\=([^\]]+)\]"),0),
-    user_name2 = arrayindex(regextract(_raw_log ,"for\suser\s(\S+)\sby"),0),
-    command = arrayindex(regextract(_raw_log ,"\[COMMAND\=([^\]]+)\]"),0)
 | alter
-   // _time = parse_timestamp("%Y %b %d %H:%M:%S", timestamp),
-    xdm.source.host.hostname = hostname,
-    xdm.source.process.name = process_name,
-    xdm.source.process.pid = pid,
-    xdm.event.description = event_description,
-    xdm.source.ipv4 = src_ip,
-    xdm.source.user.username = coalesce(user_name1 , user_name2 ),
-    xdm.target.process.command_line = command;
+    get_hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)),
+    get_process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z]+)"), 0)),
+    get_pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)),
+    get_event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)),
+    get_src_ip = arrayindex(regextract(_raw_log,"\s(\d{1,3}(?:\.\d{1,3}){3})"),0),
+    get_user_name1 = arrayindex(regextract(_raw_log ,"\[USER\=([^\]]+)\]"),0),
+    get_user_name2 = arrayindex(regextract(_raw_log ,"for\suser\s(\S+)\sby"),0),
+    get_user_name3 = arrayindex(regextract(_raw_log ,"USER=([^\s]+)"),0),
+    get_command1 = arrayindex(regextract(_raw_log ,"\[COMMAND\=([^\]]+)\]"),0),
+    get_command2 = arrayindex(regextract(_raw_log ,"\s\[[^\]]+\]:\s(.*)"),0),
+    get_command3 = arrayindex(regextract(_raw_log ,"COMMAND=(.*)"),0),
+    get_cwd = arrayindex(regextract(_raw_log ,"\scwd:([^\s]+)"),0),
+    get_pwd = arrayindex(regextract(_raw_log ,"PWD=([^\s]+)"),0),
+    get_filename = arrayindex(regextract(_raw_log ,"\sfilename:([^]]+)"),0),
+    get_uid = arrayindex(regextract(_raw_log ,"[^\-]uid[:=](\d+)"),0),
+    get_sid = arrayindex(regextract(_raw_log ,"\ssid[:=](\d+)"),0),
+    get_tty1 = arrayindex(regextract(_raw_log ,"\stty[:=]\(([^)]+)\)"),0),
+    get_tty2 = arrayindex(regextract(_raw_log ,"TTY\=([^\s]+)"),0),
+    get_port1 = to_integer(arrayindex(regextract(_raw_log ,"\d+\.\d+\.\d+\.\d+\:(\d+)"),0)),
+    get_port2 = to_integer(arrayindex(regextract(_raw_log ,"port\s(\d+)"),0)),
+    get_sha256 = arrayindex(regextract(_raw_log ,"SHA256:([^\s]+)"),0)
+| alter
+    xdm.source.host.hostname = get_hostname,
+    xdm.source.process.name = get_process_name,
+    xdm.source.process.pid = get_pid,
+    xdm.event.description = get_event_description,
+    xdm.source.ipv4 = get_src_ip,
+    xdm.source.user.username = coalesce(get_user_name1 , get_user_name2, get_user_name3),
+    xdm.target.process.command_line = coalesce(get_command1, get_command2, get_command3),
+    xdm.source.process.executable.path = coalesce(get_cwd, get_pwd),
+    xdm.target.file.path = get_filename,
+    xdm.source.agent.identifier = get_uid,
+    xdm.source.user.identifier = get_sid,
+    xdm.source.interface = coalesce(get_tty1, get_tty2),
+    xdm.source.port = coalesce(get_port1, get_port2),
+    xdm.source.process.executable.sha256 = get_sha256;
+
 
 filter _log_source_file_name in("syslog*", "messages")
-| alter current_year = arrayindex(regextract(to_string(current_time()), "\d{4}"), 0),
-    timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0)
-| alter timestamp = format_string("%s %s", current_year, timestamp),
-    hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)),
-    process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z\_\-]+)"), 0)),
-    pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)),
-    event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)),
-    src_ip = to_string(arrayindex(regextract(_raw_log,"\s(\d{1,3}(?:\.\d{1,3}){3})"),0)),
-    log_level = arrayindex(regextract(_raw_log ,"\:\s\<([a-zA-Z]+)\>"),0),
-    user_name = arrayindex(regextract(_raw_log ,"\:\s\(([a-zA-Z]+)\)\sCMD"),0),
-    command_line = arrayindex(regextract(_raw_log ,"\)\sCMD\s\(([^\)]+)\)"),0)
+| alter 
+    get_hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)),
+    get_process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z\_\-]+)"), 0)),
+    get_tar_pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)),
+    get_src_pid = to_integer(arrayindex(regextract(_raw_log, "\[pid\s+(\d+)\]"),0)),
+    get_event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)),
+    get_log_level = arrayindex(regextract(_raw_log ,"\:\s\<([a-zA-Z]+)\>"),0),
+    get_user_name = arrayindex(regextract(_raw_log ,"\:\s\(([a-zA-Z]+)\)\sCMD"),0),
+    get_command_line = arrayindex(regextract(_raw_log ,"\)\sCMD\s\(([^\)]+)\)"),0),
+    get_error_file = if(_raw_log ~= "\[ERROR\]", arrayindex(regextract(_raw_log ,"File\s\"\"([^\"]+)\"\""),0), null),
+    get_warning_username = if(_raw_log ~= "\[WARNING\]", replex(arrayindex(regextract(_raw_log ,"for user ([^\@]+)"),0), "'", ""), null),
+    get_level_des = if(_raw_log ~= "level=", arrayindex(regextract(_raw_log ,"level=([^\s]+)"),0), null),
+    get_msg_des = if(_raw_log ~= "level=", arrayindex(regextract(_raw_log ,"msg=[\"]+([^\"]+)"),0), null),
+    get_info_url = if(_raw_log ~= "\[INFO\]", arrayindex(regextract(_raw_log ,"URL\:\s+([^\|]+)"),0), null),
+    get_info_issuer = if(_raw_log ~= "\[INFO\]", arrayindex(regextract(_raw_log ,"SSL\:\s+ca\:\s+([^\|]+)"),0), null),
+    get_info_certificate = if(_raw_log ~= "\[INFO\]", arrayindex(regextract(_raw_log ,"SSL\:[^\|]+\|[^\|]+\|certificate:(\s[^\|]+)"),0), null),
+    get_src_ip1 = arrayindex(regextract(_raw_log ,"Source\s+(\d+\.\d+\.\d+\.\d+)\s+replaced\s+with\s+\d+\.\d+\.\d+\.\d+"),0),
+    get_src_ip2 = arrayindex(regextract(_raw_log ,"source\s+\d+\.\d+\.\d+\.\d+\s+from\s+(\d+\.\d+\.\d+\.\d+)"),0),
+    get_src_ip3 = arrayindex(regextract(_raw_log ,"\[(\d+\.\d+\.\d+\.\d+)\]\:\d+\-\>\[\d+\.\d+\.\d+\.\d+\]\:\d+"),0),
+    get_src_ip4 = arrayindex(regextract(_raw_log ,"\[client\s+(\d+\.\d+\.\d+\.\d+)\:\d+\]"),0),
+    get_src_ip5 = arrayindex(regextract(_raw_log ,"from\s+unknown\[([^\]]+)\]"),0),
+    get_changed_ip = arrayindex(regextract(_raw_log ,"Source\s+\d+\.\d+\.\d+\.\d+\s+replaced\s+with\s+(\d+\.\d+\.\d+\.\d+)"),0),
+    get_tar_ip1 = arrayindex(regextract(_raw_log ,"\[\d+\.\d+\.\d+\.\d+\]\:\d+\-\>\[(\d+\.\d+\.\d+\.\d+)\]\:\d+"),0),
+    get_src_port1 = to_integer(arrayindex(regextract(_raw_log ,"\[\d+\.\d+\.\d+\.\d+\]\:(\d+)\-\>\[\d+\.\d+\.\d+\.\d+\]\:\d+"),0)),
+    get_src_port2 = to_integer(arrayindex(regextract(_raw_log ,"\[client\s+\d+\.\d+\.\d+\.\d+\:(\d+)\]"),0)),
+    get_tar_port1 = to_integer(arrayindex(regextract(_raw_log ,"\[\d+\.\d+\.\d+\.\d+\]\:\d+\-\>\[\d+\.\d+\.\d+\.\d+\]\:(\d+)"),0))
+| alter
+    xdm.event.log_level = if(_raw_log ~= "\[ERROR\]", XDM_CONST.LOG_LEVEL_ERROR, _raw_log ~= "\[WARNING\]", XDM_CONST.LOG_LEVEL_WARNING, _raw_log ~= "\[INFO\]", XDM_CONST.LOG_LEVEL_INFORMATIONAL, get_log_level = "error", XDM_CONST.LOG_LEVEL_ERROR, get_log_level = "warning", XDM_CONST.LOG_LEVEL_WARNING, get_log_level = "info", XDM_CONST.LOG_LEVEL_INFORMATIONAL, get_log_level = "debug", XDM_CONST.LOG_LEVEL_DEBUG, get_level_des ~= "error", XDM_CONST.LOG_LEVEL_ERROR, get_level_des ~= "warning", XDM_CONST.LOG_LEVEL_WARNING, get_level_des ~= "info", XDM_CONST.LOG_LEVEL_INFORMATIONAL),
+    xdm.target.host.hostname = get_hostname,
+    xdm.target.process.name = get_process_name,
+    xdm.target.process.pid = get_tar_pid,
+    xdm.event.description = get_event_description,
+    xdm.target.user.username = get_user_name,
+    xdm.target.process.command_line = get_command_line,
+    xdm.target.file.path = get_error_file,
+    xdm.source.user.identifier = get_warning_username,
+    xdm.alert.description = get_msg_des,
+    xdm.target.url = get_info_url,
+    xdm.network.tls.client_certificate.issuer = get_info_issuer,
+    xdm.network.tls.client_certificate.subject = get_info_certificate,
+    xdm.source.ipv4 = coalesce(get_src_ip1, get_src_ip2, get_src_ip3, get_src_ip4, get_src_ip5),
+    xdm.target.ipv4 = coalesce(get_changed_ip, get_tar_ip1),
+    xdm.source.port = coalesce(get_src_port1, get_src_port2),
+    xdm.target.port = get_tar_port1,
+    xdm.source.process.pid = get_src_pid;
+
+
+filter _log_source_file_name = "cron"
+| alter
+    get_hostname = arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0),
+    get_process_name = arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z]+)"), 0),
+    get_pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)),
+    get_event_description = arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0),
+    get_operation_sub = arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+([[:upper:]]+)\s"),0)
+| alter
+    get_task = to_string(arrayindex(regextract(get_event_description, "^\s+\(([^\)]+)\)"),0)),
+    get_info_des = if(get_operation_sub = "INFO", arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+INFO\s+\(([^\)]+)\)"),0), null),
+    get_mail_des = if(get_operation_sub = "MAIL", arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+MAIL\s+\(([^\)]+)\)"),0), null),
+    get_cmd_command = if(get_operation_sub = "CMD", arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+CMD\s+(.*)"),0), null)
 | alter
-   // _time = parse_timestamp("%Y %b %d %H:%M:%S", timestamp),
-    xdm.target.host.hostname = hostname,
-    xdm.target.process.name = process_name,
-    xdm.target.process.pid = to_number(pid),
-    xdm.event.description = event_description,
-    xdm.alert.severity = log_level,
-    xdm.target.user.username = user_name,
-    xdm.target.process.command_line = command_line;
+    xdm.event.log_level = if(get_operation_sub = "INFO", XDM_CONST.LOG_LEVEL_INFORMATIONAL, get_operation_sub = "ERROR", XDM_CONST.LOG_LEVEL_ERROR, get_operation_sub ~= "ALERT", XDM_CONST.LOG_LEVEL_ALERT, get_operation_sub ~= "CRIT", XDM_CONST.LOG_LEVEL_CRITICAL, get_operation_sub ~= "DEBUG", XDM_CONST.LOG_LEVEL_DEBUG, get_operation_sub ~= "EMERG", XDM_CONST.LOG_LEVEL_EMERGENCY, get_operation_sub ~= "NOTICE", XDM_CONST.LOG_LEVEL_NOTICE, get_operation_sub ~= "WARNI", XDM_CONST.LOG_LEVEL_WARNING),
+    xdm.source.host.hostname = get_hostname,
+    xdm.source.process.parent_id = get_process_name,
+    xdm.event.description = get_event_description,
+    xdm.source.process.pid = get_pid,
+    xdm.source.process.executable.file_type = get_task,
+    xdm.event.operation_sub_type = get_operation_sub,
+    xdm.alert.description = coalesce(get_info_des, get_mail_des),
+    xdm.source.process.command_line = get_cmd_command;

Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.yml

@@ -1,6 +1,6 @@
 fromversion: 6.10.0
-id: linux_events_collection
-name: Linux Events Collection
+id: Linux_Events_Collection_ModelingRule
+name: Linux Events Collection Modeling Rule
 rules: ''
 schema: ''
 tags: Linux Events Collection Redhat Ubuntu

Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.xif

@@ -1,6 +1,29 @@
 [INGEST:vendor="linux", product="linux", target_dataset="linux_linux_raw", no_hit=keep]
-alter tmp_current_year = arrayindex(regextract(to_string(current_time()), "\d{4}"), 0),
-tmp_timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0)
-|alter tmp_timestamp = format_string("%s %s", tmp_current_year, tmp_timestamp)
-|alter _time = parse_timestamp("%Y %b %d %H:%M:%S", tmp_timestamp)
-|fields - tmp_current_year,tmp_timestamp;
+alter
+// Get the current year and timestamp.
+    tmp_get_current_year = arrayindex(regextract(to_string(_insert_time), "\d{4}"), 0),
+    tmp_get_timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0)
+|alter 
+// Unifies the year and timestamp as String.
+    tmp_timestamp1 = concat(tmp_get_current_year, " ", tmp_get_timestamp)
+|alter 
+// Converts the full timestamp to datetime format (First option).
+    tmp_timestamp_format1 = parse_timestamp("%Y %b %d %H:%M:%S", tmp_timestamp1)
+| alter 
+// Check the days difference between the current and extracted time.
+    tmp_timeDiff = timestamp_diff(tmp_timestamp_format1, current_time(), "DAY")
+| alter 
+// If the number of days between extracted and current time is positive, reduce the current year by 1.
+    tmp_verify_year = if(tmp_timeDiff > 0, to_string(subtract(to_integer(tmp_get_current_year),1)),null)
+| alter
+// If the year was reduced by 1, unifies the reduced year and extracted timestamp as String.
+    tmp_timestamp2 = if(tmp_verify_year != null, concat(tmp_verify_year, " ", tmp_get_timestamp), null)
+| alter
+// Converts the full timestamp to datetime format (Second option).
+    tmp_timestamp_format2 = if(tmp_timestamp2 != null, parse_timestamp("%Y %b %d %H:%M:%S", tmp_timestamp2), null)
+| alter
+    tmp_check_which_timestamp = coalesce(tmp_timestamp_format2, tmp_timestamp_format1, _insert_time) 
+| alter
+// Check if the second option is null, if not, use the first option.
+    _time = tmp_check_which_timestamp
+| fields -tmp_get_current_year, tmp_get_timestamp, tmp_timestamp1, tmp_timestamp_format1, tmp_timeDiff, tmp_verify_year, tmp_timestamp2, tmp_timestamp_format2, tmp_check_which_timestamp;

Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.yml

@@ -1,5 +1,5 @@
-name: LinuxEventsCollection Parsing Rule
-id: LinuxEventsCollection_Parsing_Rule
+name: Linux Events Collection Parsing Rule
+id: Linux_Events_Collection_ParsingRule
 fromversion: 6.10.0
 tags: []
 rules: ''

Packs/LinuxEventsCollection/ReleaseNotes/1_0_5.md

@@ -0,0 +1,14 @@
+
+#### Parsing Rules
+
+##### Linux Events Collection Parsing Rule
+
+- Updated the Parsing Rule logic.
+
+
+#### Modeling Rules
+
+##### Linux Events Collection Modeling Rule
+
+- Updated the Modeling Rule with additional fields.
+

Packs/LinuxEventsCollection/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Linux Events Collection",
     "description": "Linux is an operating system for servers, desktops, cloud, and IoTs",
     "support": "xsoar",
-    "currentVersion": "1.0.4",
+    "currentVersion": "1.0.5",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",