diff --git a/Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.xif b/Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.xif index 2ac825c687cb..9c48db2a33e5 100644 --- a/Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.xif +++ b/Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.xif @@ -1,44 +1,111 @@ [MODEL: dataset="linux_linux_raw"] filter _log_source_file_name in("auth*", "secure") -| alter current_year = arrayindex(regextract(to_string(current_time()), "\d{4}"), 0), - //timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0), - //timestamp = format_string("%s %s", current_year, timestamp), - hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)), - process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z]+)"), 0)), - pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), - event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)), - src_ip = arrayindex(regextract(_raw_log,"\s(\d{1,3}(?:\.\d{1,3}){3})"),0), - user_name1 = arrayindex(regextract(_raw_log ,"\[USER\=([^\]]+)\]"),0), - user_name2 = arrayindex(regextract(_raw_log ,"for\suser\s(\S+)\sby"),0), - command = arrayindex(regextract(_raw_log ,"\[COMMAND\=([^\]]+)\]"),0) | alter - // _time = parse_timestamp("%Y %b %d %H:%M:%S", timestamp), - xdm.source.host.hostname = hostname, - xdm.source.process.name = process_name, - xdm.source.process.pid = pid, - xdm.event.description = event_description, - xdm.source.ipv4 = src_ip, - xdm.source.user.username = coalesce(user_name1 , user_name2 ), - xdm.target.process.command_line = command; + get_hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)), + get_process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z]+)"), 0)), + get_pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), + get_event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)), + get_src_ip = arrayindex(regextract(_raw_log,"\s(\d{1,3}(?:\.\d{1,3}){3})"),0), + get_user_name1 = arrayindex(regextract(_raw_log ,"\[USER\=([^\]]+)\]"),0), + get_user_name2 = arrayindex(regextract(_raw_log ,"for\suser\s(\S+)\sby"),0), + get_user_name3 = arrayindex(regextract(_raw_log ,"USER=([^\s]+)"),0), + get_command1 = arrayindex(regextract(_raw_log ,"\[COMMAND\=([^\]]+)\]"),0), + get_command2 = arrayindex(regextract(_raw_log ,"\s\[[^\]]+\]:\s(.*)"),0), + get_command3 = arrayindex(regextract(_raw_log ,"COMMAND=(.*)"),0), + get_cwd = arrayindex(regextract(_raw_log ,"\scwd:([^\s]+)"),0), + get_pwd = arrayindex(regextract(_raw_log ,"PWD=([^\s]+)"),0), + get_filename = arrayindex(regextract(_raw_log ,"\sfilename:([^]]+)"),0), + get_uid = arrayindex(regextract(_raw_log ,"[^\-]uid[:=](\d+)"),0), + get_sid = arrayindex(regextract(_raw_log ,"\ssid[:=](\d+)"),0), + get_tty1 = arrayindex(regextract(_raw_log ,"\stty[:=]\(([^)]+)\)"),0), + get_tty2 = arrayindex(regextract(_raw_log ,"TTY\=([^\s]+)"),0), + get_port1 = to_integer(arrayindex(regextract(_raw_log ,"\d+\.\d+\.\d+\.\d+\:(\d+)"),0)), + get_port2 = to_integer(arrayindex(regextract(_raw_log ,"port\s(\d+)"),0)), + get_sha256 = arrayindex(regextract(_raw_log ,"SHA256:([^\s]+)"),0) +| alter + xdm.source.host.hostname = get_hostname, + xdm.source.process.name = get_process_name, + xdm.source.process.pid = get_pid, + xdm.event.description = get_event_description, + xdm.source.ipv4 = get_src_ip, + xdm.source.user.username = coalesce(get_user_name1 , get_user_name2, get_user_name3), + xdm.target.process.command_line = coalesce(get_command1, get_command2, get_command3), + xdm.source.process.executable.path = coalesce(get_cwd, get_pwd), + xdm.target.file.path = get_filename, + xdm.source.agent.identifier = get_uid, + xdm.source.user.identifier = get_sid, + xdm.source.interface = coalesce(get_tty1, get_tty2), + xdm.source.port = coalesce(get_port1, get_port2), + xdm.source.process.executable.sha256 = get_sha256; + filter _log_source_file_name in("syslog*", "messages") -| alter current_year = arrayindex(regextract(to_string(current_time()), "\d{4}"), 0), - timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0) -| alter timestamp = format_string("%s %s", current_year, timestamp), - hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)), - process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z\_\-]+)"), 0)), - pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), - event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)), - src_ip = to_string(arrayindex(regextract(_raw_log,"\s(\d{1,3}(?:\.\d{1,3}){3})"),0)), - log_level = arrayindex(regextract(_raw_log ,"\:\s\<([a-zA-Z]+)\>"),0), - user_name = arrayindex(regextract(_raw_log ,"\:\s\(([a-zA-Z]+)\)\sCMD"),0), - command_line = arrayindex(regextract(_raw_log ,"\)\sCMD\s\(([^\)]+)\)"),0) +| alter + get_hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)), + get_process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z\_\-]+)"), 0)), + get_tar_pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), + get_src_pid = to_integer(arrayindex(regextract(_raw_log, "\[pid\s+(\d+)\]"),0)), + get_event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)), + get_log_level = arrayindex(regextract(_raw_log ,"\:\s\<([a-zA-Z]+)\>"),0), + get_user_name = arrayindex(regextract(_raw_log ,"\:\s\(([a-zA-Z]+)\)\sCMD"),0), + get_command_line = arrayindex(regextract(_raw_log ,"\)\sCMD\s\(([^\)]+)\)"),0), + get_error_file = if(_raw_log ~= "\[ERROR\]", arrayindex(regextract(_raw_log ,"File\s\"\"([^\"]+)\"\""),0), null), + get_warning_username = if(_raw_log ~= "\[WARNING\]", replex(arrayindex(regextract(_raw_log ,"for user ([^\@]+)"),0), "'", ""), null), + get_level_des = if(_raw_log ~= "level=", arrayindex(regextract(_raw_log ,"level=([^\s]+)"),0), null), + get_msg_des = if(_raw_log ~= "level=", arrayindex(regextract(_raw_log ,"msg=[\"]+([^\"]+)"),0), null), + get_info_url = if(_raw_log ~= "\[INFO\]", arrayindex(regextract(_raw_log ,"URL\:\s+([^\|]+)"),0), null), + get_info_issuer = if(_raw_log ~= "\[INFO\]", arrayindex(regextract(_raw_log ,"SSL\:\s+ca\:\s+([^\|]+)"),0), null), + get_info_certificate = if(_raw_log ~= "\[INFO\]", arrayindex(regextract(_raw_log ,"SSL\:[^\|]+\|[^\|]+\|certificate:(\s[^\|]+)"),0), null), + get_src_ip1 = arrayindex(regextract(_raw_log ,"Source\s+(\d+\.\d+\.\d+\.\d+)\s+replaced\s+with\s+\d+\.\d+\.\d+\.\d+"),0), + get_src_ip2 = arrayindex(regextract(_raw_log ,"source\s+\d+\.\d+\.\d+\.\d+\s+from\s+(\d+\.\d+\.\d+\.\d+)"),0), + get_src_ip3 = arrayindex(regextract(_raw_log ,"\[(\d+\.\d+\.\d+\.\d+)\]\:\d+\-\>\[\d+\.\d+\.\d+\.\d+\]\:\d+"),0), + get_src_ip4 = arrayindex(regextract(_raw_log ,"\[client\s+(\d+\.\d+\.\d+\.\d+)\:\d+\]"),0), + get_src_ip5 = arrayindex(regextract(_raw_log ,"from\s+unknown\[([^\]]+)\]"),0), + get_changed_ip = arrayindex(regextract(_raw_log ,"Source\s+\d+\.\d+\.\d+\.\d+\s+replaced\s+with\s+(\d+\.\d+\.\d+\.\d+)"),0), + get_tar_ip1 = arrayindex(regextract(_raw_log ,"\[\d+\.\d+\.\d+\.\d+\]\:\d+\-\>\[(\d+\.\d+\.\d+\.\d+)\]\:\d+"),0), + get_src_port1 = to_integer(arrayindex(regextract(_raw_log ,"\[\d+\.\d+\.\d+\.\d+\]\:(\d+)\-\>\[\d+\.\d+\.\d+\.\d+\]\:\d+"),0)), + get_src_port2 = to_integer(arrayindex(regextract(_raw_log ,"\[client\s+\d+\.\d+\.\d+\.\d+\:(\d+)\]"),0)), + get_tar_port1 = to_integer(arrayindex(regextract(_raw_log ,"\[\d+\.\d+\.\d+\.\d+\]\:\d+\-\>\[\d+\.\d+\.\d+\.\d+\]\:(\d+)"),0)) +| alter + xdm.event.log_level = if(_raw_log ~= "\[ERROR\]", XDM_CONST.LOG_LEVEL_ERROR, _raw_log ~= "\[WARNING\]", XDM_CONST.LOG_LEVEL_WARNING, _raw_log ~= "\[INFO\]", XDM_CONST.LOG_LEVEL_INFORMATIONAL, get_log_level = "error", XDM_CONST.LOG_LEVEL_ERROR, get_log_level = "warning", XDM_CONST.LOG_LEVEL_WARNING, get_log_level = "info", XDM_CONST.LOG_LEVEL_INFORMATIONAL, get_log_level = "debug", XDM_CONST.LOG_LEVEL_DEBUG, get_level_des ~= "error", XDM_CONST.LOG_LEVEL_ERROR, get_level_des ~= "warning", XDM_CONST.LOG_LEVEL_WARNING, get_level_des ~= "info", XDM_CONST.LOG_LEVEL_INFORMATIONAL), + xdm.target.host.hostname = get_hostname, + xdm.target.process.name = get_process_name, + xdm.target.process.pid = get_tar_pid, + xdm.event.description = get_event_description, + xdm.target.user.username = get_user_name, + xdm.target.process.command_line = get_command_line, + xdm.target.file.path = get_error_file, + xdm.source.user.identifier = get_warning_username, + xdm.alert.description = get_msg_des, + xdm.target.url = get_info_url, + xdm.network.tls.client_certificate.issuer = get_info_issuer, + xdm.network.tls.client_certificate.subject = get_info_certificate, + xdm.source.ipv4 = coalesce(get_src_ip1, get_src_ip2, get_src_ip3, get_src_ip4, get_src_ip5), + xdm.target.ipv4 = coalesce(get_changed_ip, get_tar_ip1), + xdm.source.port = coalesce(get_src_port1, get_src_port2), + xdm.target.port = get_tar_port1, + xdm.source.process.pid = get_src_pid; + + +filter _log_source_file_name = "cron" +| alter + get_hostname = arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0), + get_process_name = arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z]+)"), 0), + get_pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), + get_event_description = arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0), + get_operation_sub = arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+([[:upper:]]+)\s"),0) +| alter + get_task = to_string(arrayindex(regextract(get_event_description, "^\s+\(([^\)]+)\)"),0)), + get_info_des = if(get_operation_sub = "INFO", arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+INFO\s+\(([^\)]+)\)"),0), null), + get_mail_des = if(get_operation_sub = "MAIL", arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+MAIL\s+\(([^\)]+)\)"),0), null), + get_cmd_command = if(get_operation_sub = "CMD", arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+CMD\s+(.*)"),0), null) | alter - // _time = parse_timestamp("%Y %b %d %H:%M:%S", timestamp), - xdm.target.host.hostname = hostname, - xdm.target.process.name = process_name, - xdm.target.process.pid = to_number(pid), - xdm.event.description = event_description, - xdm.alert.severity = log_level, - xdm.target.user.username = user_name, - xdm.target.process.command_line = command_line; + xdm.event.log_level = if(get_operation_sub = "INFO", XDM_CONST.LOG_LEVEL_INFORMATIONAL, get_operation_sub = "ERROR", XDM_CONST.LOG_LEVEL_ERROR, get_operation_sub ~= "ALERT", XDM_CONST.LOG_LEVEL_ALERT, get_operation_sub ~= "CRIT", XDM_CONST.LOG_LEVEL_CRITICAL, get_operation_sub ~= "DEBUG", XDM_CONST.LOG_LEVEL_DEBUG, get_operation_sub ~= "EMERG", XDM_CONST.LOG_LEVEL_EMERGENCY, get_operation_sub ~= "NOTICE", XDM_CONST.LOG_LEVEL_NOTICE, get_operation_sub ~= "WARNI", XDM_CONST.LOG_LEVEL_WARNING), + xdm.source.host.hostname = get_hostname, + xdm.source.process.parent_id = get_process_name, + xdm.event.description = get_event_description, + xdm.source.process.pid = get_pid, + xdm.source.process.executable.file_type = get_task, + xdm.event.operation_sub_type = get_operation_sub, + xdm.alert.description = coalesce(get_info_des, get_mail_des), + xdm.source.process.command_line = get_cmd_command; \ No newline at end of file diff --git a/Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.yml b/Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.yml index 9b523298a3c6..583deaa5754e 100644 --- a/Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.yml +++ b/Packs/LinuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.yml @@ -1,6 +1,6 @@ fromversion: 6.10.0 -id: linux_events_collection -name: Linux Events Collection +id: Linux_Events_Collection_ModelingRule +name: Linux Events Collection Modeling Rule rules: '' schema: '' tags: Linux Events Collection Redhat Ubuntu \ No newline at end of file diff --git a/Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.xif b/Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.xif index 0fb1b9a254aa..43272d0a476b 100644 --- a/Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.xif +++ b/Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.xif @@ -1,6 +1,29 @@ [INGEST:vendor="linux", product="linux", target_dataset="linux_linux_raw", no_hit=keep] -alter tmp_current_year = arrayindex(regextract(to_string(current_time()), "\d{4}"), 0), -tmp_timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0) -|alter tmp_timestamp = format_string("%s %s", tmp_current_year, tmp_timestamp) -|alter _time = parse_timestamp("%Y %b %d %H:%M:%S", tmp_timestamp) -|fields - tmp_current_year,tmp_timestamp; \ No newline at end of file +alter +// Get the current year and timestamp. + tmp_get_current_year = arrayindex(regextract(to_string(_insert_time), "\d{4}"), 0), + tmp_get_timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0) +|alter +// Unifies the year and timestamp as String. + tmp_timestamp1 = concat(tmp_get_current_year, " ", tmp_get_timestamp) +|alter +// Converts the full timestamp to datetime format (First option). + tmp_timestamp_format1 = parse_timestamp("%Y %b %d %H:%M:%S", tmp_timestamp1) +| alter +// Check the days difference between the current and extracted time. + tmp_timeDiff = timestamp_diff(tmp_timestamp_format1, current_time(), "DAY") +| alter +// If the number of days between extracted and current time is positive, reduce the current year by 1. + tmp_verify_year = if(tmp_timeDiff > 0, to_string(subtract(to_integer(tmp_get_current_year),1)),null) +| alter +// If the year was reduced by 1, unifies the reduced year and extracted timestamp as String. + tmp_timestamp2 = if(tmp_verify_year != null, concat(tmp_verify_year, " ", tmp_get_timestamp), null) +| alter +// Converts the full timestamp to datetime format (Second option). + tmp_timestamp_format2 = if(tmp_timestamp2 != null, parse_timestamp("%Y %b %d %H:%M:%S", tmp_timestamp2), null) +| alter + tmp_check_which_timestamp = coalesce(tmp_timestamp_format2, tmp_timestamp_format1, _insert_time) +| alter +// Check if the second option is null, if not, use the first option. + _time = tmp_check_which_timestamp +| fields -tmp_get_current_year, tmp_get_timestamp, tmp_timestamp1, tmp_timestamp_format1, tmp_timeDiff, tmp_verify_year, tmp_timestamp2, tmp_timestamp_format2, tmp_check_which_timestamp; \ No newline at end of file diff --git a/Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.yml b/Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.yml index cb2063f92cca..bf61a5a7a28c 100644 --- a/Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.yml +++ b/Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.yml @@ -1,5 +1,5 @@ -name: LinuxEventsCollection Parsing Rule -id: LinuxEventsCollection_Parsing_Rule +name: Linux Events Collection Parsing Rule +id: Linux_Events_Collection_ParsingRule fromversion: 6.10.0 tags: [] rules: '' diff --git a/Packs/LinuxEventsCollection/ReleaseNotes/1_0_5.md b/Packs/LinuxEventsCollection/ReleaseNotes/1_0_5.md new file mode 100644 index 000000000000..8b043714bd21 --- /dev/null +++ b/Packs/LinuxEventsCollection/ReleaseNotes/1_0_5.md @@ -0,0 +1,14 @@ + +#### Parsing Rules + +##### Linux Events Collection Parsing Rule + +- Updated the Parsing Rule logic. + + +#### Modeling Rules + +##### Linux Events Collection Modeling Rule + +- Updated the Modeling Rule with additional fields. + diff --git a/Packs/LinuxEventsCollection/pack_metadata.json b/Packs/LinuxEventsCollection/pack_metadata.json index 1438ce9563b3..52b0db2384c3 100644 --- a/Packs/LinuxEventsCollection/pack_metadata.json +++ b/Packs/LinuxEventsCollection/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Linux Events Collection", "description": "Linux is an operating system for servers, desktops, cloud, and IoTs", "support": "xsoar", - "currentVersion": "1.0.4", + "currentVersion": "1.0.5", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",