forked from demisto/content
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Updated ModelingRules * Updated ReleaseNotes * Updated ReleaseNotes * Reverted fromversion * Updated ParsingRules logic * Updated ParsingRules * Updated ReleaseNotes * Updated ModelingRules .yml * Updated ReleaseNotes * Updated ParsingRules .yml * Updated .yml configs * Updated ReleaseNotes * Updated ParsingRules * Updated ModelingRules
- Loading branch information
Showing
6 changed files
with
152 additions
and
48 deletions.
There are no files selected for viewing
143 changes: 105 additions & 38 deletions
143
...nuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.xif
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,44 +1,111 @@ | ||
| [MODEL: dataset="linux_linux_raw"] | ||
| filter _log_source_file_name in("auth*", "secure") | ||
| | alter current_year = arrayindex(regextract(to_string(current_time()), "\d{4}"), 0), | ||
| //timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0), | ||
| //timestamp = format_string("%s %s", current_year, timestamp), | ||
| hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)), | ||
| process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z]+)"), 0)), | ||
| pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), | ||
| event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)), | ||
| src_ip = arrayindex(regextract(_raw_log,"\s(\d{1,3}(?:\.\d{1,3}){3})"),0), | ||
| user_name1 = arrayindex(regextract(_raw_log ,"\[USER\=([^\]]+)\]"),0), | ||
| user_name2 = arrayindex(regextract(_raw_log ,"for\suser\s(\S+)\sby"),0), | ||
| command = arrayindex(regextract(_raw_log ,"\[COMMAND\=([^\]]+)\]"),0) | ||
| | alter | ||
| // _time = parse_timestamp("%Y %b %d %H:%M:%S", timestamp), | ||
| xdm.source.host.hostname = hostname, | ||
| xdm.source.process.name = process_name, | ||
| xdm.source.process.pid = pid, | ||
| xdm.event.description = event_description, | ||
| xdm.source.ipv4 = src_ip, | ||
| xdm.source.user.username = coalesce(user_name1 , user_name2 ), | ||
| xdm.target.process.command_line = command; | ||
| get_hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)), | ||
| get_process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z]+)"), 0)), | ||
| get_pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), | ||
| get_event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)), | ||
| get_src_ip = arrayindex(regextract(_raw_log,"\s(\d{1,3}(?:\.\d{1,3}){3})"),0), | ||
| get_user_name1 = arrayindex(regextract(_raw_log ,"\[USER\=([^\]]+)\]"),0), | ||
| get_user_name2 = arrayindex(regextract(_raw_log ,"for\suser\s(\S+)\sby"),0), | ||
| get_user_name3 = arrayindex(regextract(_raw_log ,"USER=([^\s]+)"),0), | ||
| get_command1 = arrayindex(regextract(_raw_log ,"\[COMMAND\=([^\]]+)\]"),0), | ||
| get_command2 = arrayindex(regextract(_raw_log ,"\s\[[^\]]+\]:\s(.*)"),0), | ||
| get_command3 = arrayindex(regextract(_raw_log ,"COMMAND=(.*)"),0), | ||
| get_cwd = arrayindex(regextract(_raw_log ,"\scwd:([^\s]+)"),0), | ||
| get_pwd = arrayindex(regextract(_raw_log ,"PWD=([^\s]+)"),0), | ||
| get_filename = arrayindex(regextract(_raw_log ,"\sfilename:([^]]+)"),0), | ||
| get_uid = arrayindex(regextract(_raw_log ,"[^\-]uid[:=](\d+)"),0), | ||
| get_sid = arrayindex(regextract(_raw_log ,"\ssid[:=](\d+)"),0), | ||
| get_tty1 = arrayindex(regextract(_raw_log ,"\stty[:=]\(([^)]+)\)"),0), | ||
| get_tty2 = arrayindex(regextract(_raw_log ,"TTY\=([^\s]+)"),0), | ||
| get_port1 = to_integer(arrayindex(regextract(_raw_log ,"\d+\.\d+\.\d+\.\d+\:(\d+)"),0)), | ||
| get_port2 = to_integer(arrayindex(regextract(_raw_log ,"port\s(\d+)"),0)), | ||
| get_sha256 = arrayindex(regextract(_raw_log ,"SHA256:([^\s]+)"),0) | ||
| | alter | ||
| xdm.source.host.hostname = get_hostname, | ||
| xdm.source.process.name = get_process_name, | ||
| xdm.source.process.pid = get_pid, | ||
| xdm.event.description = get_event_description, | ||
| xdm.source.ipv4 = get_src_ip, | ||
| xdm.source.user.username = coalesce(get_user_name1 , get_user_name2, get_user_name3), | ||
| xdm.target.process.command_line = coalesce(get_command1, get_command2, get_command3), | ||
| xdm.source.process.executable.path = coalesce(get_cwd, get_pwd), | ||
| xdm.target.file.path = get_filename, | ||
| xdm.source.agent.identifier = get_uid, | ||
| xdm.source.user.identifier = get_sid, | ||
| xdm.source.interface = coalesce(get_tty1, get_tty2), | ||
| xdm.source.port = coalesce(get_port1, get_port2), | ||
| xdm.source.process.executable.sha256 = get_sha256; | ||
|
|
||
|
|
||
| filter _log_source_file_name in("syslog*", "messages") | ||
| | alter current_year = arrayindex(regextract(to_string(current_time()), "\d{4}"), 0), | ||
| timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0) | ||
| | alter timestamp = format_string("%s %s", current_year, timestamp), | ||
| hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)), | ||
| process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z\_\-]+)"), 0)), | ||
| pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), | ||
| event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)), | ||
| src_ip = to_string(arrayindex(regextract(_raw_log,"\s(\d{1,3}(?:\.\d{1,3}){3})"),0)), | ||
| log_level = arrayindex(regextract(_raw_log ,"\:\s\<([a-zA-Z]+)\>"),0), | ||
| user_name = arrayindex(regextract(_raw_log ,"\:\s\(([a-zA-Z]+)\)\sCMD"),0), | ||
| command_line = arrayindex(regextract(_raw_log ,"\)\sCMD\s\(([^\)]+)\)"),0) | ||
| | alter | ||
| get_hostname = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0)), | ||
| get_process_name = to_string(arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z\_\-]+)"), 0)), | ||
| get_tar_pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), | ||
| get_src_pid = to_integer(arrayindex(regextract(_raw_log, "\[pid\s+(\d+)\]"),0)), | ||
| get_event_description = to_string(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0)), | ||
| get_log_level = arrayindex(regextract(_raw_log ,"\:\s\<([a-zA-Z]+)\>"),0), | ||
| get_user_name = arrayindex(regextract(_raw_log ,"\:\s\(([a-zA-Z]+)\)\sCMD"),0), | ||
| get_command_line = arrayindex(regextract(_raw_log ,"\)\sCMD\s\(([^\)]+)\)"),0), | ||
| get_error_file = if(_raw_log ~= "\[ERROR\]", arrayindex(regextract(_raw_log ,"File\s\"\"([^\"]+)\"\""),0), null), | ||
| get_warning_username = if(_raw_log ~= "\[WARNING\]", replex(arrayindex(regextract(_raw_log ,"for user ([^\@]+)"),0), "'", ""), null), | ||
| get_level_des = if(_raw_log ~= "level=", arrayindex(regextract(_raw_log ,"level=([^\s]+)"),0), null), | ||
| get_msg_des = if(_raw_log ~= "level=", arrayindex(regextract(_raw_log ,"msg=[\"]+([^\"]+)"),0), null), | ||
| get_info_url = if(_raw_log ~= "\[INFO\]", arrayindex(regextract(_raw_log ,"URL\:\s+([^\|]+)"),0), null), | ||
| get_info_issuer = if(_raw_log ~= "\[INFO\]", arrayindex(regextract(_raw_log ,"SSL\:\s+ca\:\s+([^\|]+)"),0), null), | ||
| get_info_certificate = if(_raw_log ~= "\[INFO\]", arrayindex(regextract(_raw_log ,"SSL\:[^\|]+\|[^\|]+\|certificate:(\s[^\|]+)"),0), null), | ||
| get_src_ip1 = arrayindex(regextract(_raw_log ,"Source\s+(\d+\.\d+\.\d+\.\d+)\s+replaced\s+with\s+\d+\.\d+\.\d+\.\d+"),0), | ||
| get_src_ip2 = arrayindex(regextract(_raw_log ,"source\s+\d+\.\d+\.\d+\.\d+\s+from\s+(\d+\.\d+\.\d+\.\d+)"),0), | ||
| get_src_ip3 = arrayindex(regextract(_raw_log ,"\[(\d+\.\d+\.\d+\.\d+)\]\:\d+\-\>\[\d+\.\d+\.\d+\.\d+\]\:\d+"),0), | ||
| get_src_ip4 = arrayindex(regextract(_raw_log ,"\[client\s+(\d+\.\d+\.\d+\.\d+)\:\d+\]"),0), | ||
| get_src_ip5 = arrayindex(regextract(_raw_log ,"from\s+unknown\[([^\]]+)\]"),0), | ||
| get_changed_ip = arrayindex(regextract(_raw_log ,"Source\s+\d+\.\d+\.\d+\.\d+\s+replaced\s+with\s+(\d+\.\d+\.\d+\.\d+)"),0), | ||
| get_tar_ip1 = arrayindex(regextract(_raw_log ,"\[\d+\.\d+\.\d+\.\d+\]\:\d+\-\>\[(\d+\.\d+\.\d+\.\d+)\]\:\d+"),0), | ||
| get_src_port1 = to_integer(arrayindex(regextract(_raw_log ,"\[\d+\.\d+\.\d+\.\d+\]\:(\d+)\-\>\[\d+\.\d+\.\d+\.\d+\]\:\d+"),0)), | ||
| get_src_port2 = to_integer(arrayindex(regextract(_raw_log ,"\[client\s+\d+\.\d+\.\d+\.\d+\:(\d+)\]"),0)), | ||
| get_tar_port1 = to_integer(arrayindex(regextract(_raw_log ,"\[\d+\.\d+\.\d+\.\d+\]\:\d+\-\>\[\d+\.\d+\.\d+\.\d+\]\:(\d+)"),0)) | ||
| | alter | ||
| xdm.event.log_level = if(_raw_log ~= "\[ERROR\]", XDM_CONST.LOG_LEVEL_ERROR, _raw_log ~= "\[WARNING\]", XDM_CONST.LOG_LEVEL_WARNING, _raw_log ~= "\[INFO\]", XDM_CONST.LOG_LEVEL_INFORMATIONAL, get_log_level = "error", XDM_CONST.LOG_LEVEL_ERROR, get_log_level = "warning", XDM_CONST.LOG_LEVEL_WARNING, get_log_level = "info", XDM_CONST.LOG_LEVEL_INFORMATIONAL, get_log_level = "debug", XDM_CONST.LOG_LEVEL_DEBUG, get_level_des ~= "error", XDM_CONST.LOG_LEVEL_ERROR, get_level_des ~= "warning", XDM_CONST.LOG_LEVEL_WARNING, get_level_des ~= "info", XDM_CONST.LOG_LEVEL_INFORMATIONAL), | ||
| xdm.target.host.hostname = get_hostname, | ||
| xdm.target.process.name = get_process_name, | ||
| xdm.target.process.pid = get_tar_pid, | ||
| xdm.event.description = get_event_description, | ||
| xdm.target.user.username = get_user_name, | ||
| xdm.target.process.command_line = get_command_line, | ||
| xdm.target.file.path = get_error_file, | ||
| xdm.source.user.identifier = get_warning_username, | ||
| xdm.alert.description = get_msg_des, | ||
| xdm.target.url = get_info_url, | ||
| xdm.network.tls.client_certificate.issuer = get_info_issuer, | ||
| xdm.network.tls.client_certificate.subject = get_info_certificate, | ||
| xdm.source.ipv4 = coalesce(get_src_ip1, get_src_ip2, get_src_ip3, get_src_ip4, get_src_ip5), | ||
| xdm.target.ipv4 = coalesce(get_changed_ip, get_tar_ip1), | ||
| xdm.source.port = coalesce(get_src_port1, get_src_port2), | ||
| xdm.target.port = get_tar_port1, | ||
| xdm.source.process.pid = get_src_pid; | ||
|
|
||
|
|
||
| filter _log_source_file_name = "cron" | ||
| | alter | ||
| get_hostname = arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+([^\s]+)"), 0), | ||
| get_process_name = arrayindex(regextract(_raw_log, "\d+\:\d+\:\d+\s\S+\s([a-zA-Z]+)"), 0), | ||
| get_pid = to_integer(arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s[^\s]+\[(\d+)\]"),0)), | ||
| get_event_description = arrayindex(regextract(_raw_log, "\d{2}\S\d{2}\S\d{2}\s+[^\s]+\s+[^\:]+\:+(.*)"),0), | ||
| get_operation_sub = arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+([[:upper:]]+)\s"),0) | ||
| | alter | ||
| get_task = to_string(arrayindex(regextract(get_event_description, "^\s+\(([^\)]+)\)"),0)), | ||
| get_info_des = if(get_operation_sub = "INFO", arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+INFO\s+\(([^\)]+)\)"),0), null), | ||
| get_mail_des = if(get_operation_sub = "MAIL", arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+MAIL\s+\(([^\)]+)\)"),0), null), | ||
| get_cmd_command = if(get_operation_sub = "CMD", arrayindex(regextract(_raw_log, "\:\s+\(\S+\s+CMD\s+(.*)"),0), null) | ||
| | alter | ||
| // _time = parse_timestamp("%Y %b %d %H:%M:%S", timestamp), | ||
| xdm.target.host.hostname = hostname, | ||
| xdm.target.process.name = process_name, | ||
| xdm.target.process.pid = to_number(pid), | ||
| xdm.event.description = event_description, | ||
| xdm.alert.severity = log_level, | ||
| xdm.target.user.username = user_name, | ||
| xdm.target.process.command_line = command_line; | ||
| xdm.event.log_level = if(get_operation_sub = "INFO", XDM_CONST.LOG_LEVEL_INFORMATIONAL, get_operation_sub = "ERROR", XDM_CONST.LOG_LEVEL_ERROR, get_operation_sub ~= "ALERT", XDM_CONST.LOG_LEVEL_ALERT, get_operation_sub ~= "CRIT", XDM_CONST.LOG_LEVEL_CRITICAL, get_operation_sub ~= "DEBUG", XDM_CONST.LOG_LEVEL_DEBUG, get_operation_sub ~= "EMERG", XDM_CONST.LOG_LEVEL_EMERGENCY, get_operation_sub ~= "NOTICE", XDM_CONST.LOG_LEVEL_NOTICE, get_operation_sub ~= "WARNI", XDM_CONST.LOG_LEVEL_WARNING), | ||
| xdm.source.host.hostname = get_hostname, | ||
| xdm.source.process.parent_id = get_process_name, | ||
| xdm.event.description = get_event_description, | ||
| xdm.source.process.pid = get_pid, | ||
| xdm.source.process.executable.file_type = get_task, | ||
| xdm.event.operation_sub_type = get_operation_sub, | ||
| xdm.alert.description = coalesce(get_info_des, get_mail_des), | ||
| xdm.source.process.command_line = get_cmd_command; |
4 changes: 2 additions & 2 deletions
4
...nuxEventsCollection/ModelingRules/LinuxEventsCollection_1_3/LinuxEventsCollection_1_3.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| fromversion: 6.10.0 | ||
| id: linux_events_collection | ||
| name: Linux Events Collection | ||
| id: Linux_Events_Collection_ModelingRule | ||
| name: Linux Events Collection Modeling Rule | ||
| rules: '' | ||
| schema: '' | ||
| tags: Linux Events Collection Redhat Ubuntu |
33 changes: 28 additions & 5 deletions
33
...tion/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.xif
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,29 @@ | ||
| [INGEST:vendor="linux", product="linux", target_dataset="linux_linux_raw", no_hit=keep] | ||
| alter tmp_current_year = arrayindex(regextract(to_string(current_time()), "\d{4}"), 0), | ||
| tmp_timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0) | ||
| |alter tmp_timestamp = format_string("%s %s", tmp_current_year, tmp_timestamp) | ||
| |alter _time = parse_timestamp("%Y %b %d %H:%M:%S", tmp_timestamp) | ||
| |fields - tmp_current_year,tmp_timestamp; | ||
| alter | ||
| // Get the current year and timestamp. | ||
| tmp_get_current_year = arrayindex(regextract(to_string(_insert_time), "\d{4}"), 0), | ||
| tmp_get_timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0) | ||
| |alter | ||
| // Unifies the year and timestamp as String. | ||
| tmp_timestamp1 = concat(tmp_get_current_year, " ", tmp_get_timestamp) | ||
| |alter | ||
| // Converts the full timestamp to datetime format (First option). | ||
| tmp_timestamp_format1 = parse_timestamp("%Y %b %d %H:%M:%S", tmp_timestamp1) | ||
| | alter | ||
| // Check the days difference between the current and extracted time. | ||
| tmp_timeDiff = timestamp_diff(tmp_timestamp_format1, current_time(), "DAY") | ||
| | alter | ||
| // If the number of days between extracted and current time is positive, reduce the current year by 1. | ||
| tmp_verify_year = if(tmp_timeDiff > 0, to_string(subtract(to_integer(tmp_get_current_year),1)),null) | ||
| | alter | ||
| // If the year was reduced by 1, unifies the reduced year and extracted timestamp as String. | ||
| tmp_timestamp2 = if(tmp_verify_year != null, concat(tmp_verify_year, " ", tmp_get_timestamp), null) | ||
| | alter | ||
| // Converts the full timestamp to datetime format (Second option). | ||
| tmp_timestamp_format2 = if(tmp_timestamp2 != null, parse_timestamp("%Y %b %d %H:%M:%S", tmp_timestamp2), null) | ||
| | alter | ||
| tmp_check_which_timestamp = coalesce(tmp_timestamp_format2, tmp_timestamp_format1, _insert_time) | ||
| | alter | ||
| // Check if the second option is null, if not, use the first option. | ||
| _time = tmp_check_which_timestamp | ||
| | fields -tmp_get_current_year, tmp_get_timestamp, tmp_timestamp1, tmp_timestamp_format1, tmp_timeDiff, tmp_verify_year, tmp_timestamp2, tmp_timestamp_format2, tmp_check_which_timestamp; |
4 changes: 2 additions & 2 deletions
4
...tion/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
|
|
||
| #### Parsing Rules | ||
|
|
||
| ##### Linux Events Collection Parsing Rule | ||
|
|
||
| - Updated the Parsing Rule logic. | ||
|
|
||
|
|
||
| #### Modeling Rules | ||
|
|
||
| ##### Linux Events Collection Modeling Rule | ||
|
|
||
| - Updated the Modeling Rule with additional fields. | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters