Kiteworks.xif

[MODEL: dataset="kiteworks_kiteworks_raw"]

// JSON Format 
filter _raw_log ~= "^(?:\S+\s+){5}\{" 
| alter 
    syslog_priority = to_integer(arrayindex(regextract(_raw_log, "^\<(\d{1,3})\>\s*\w+"), 0)),
    syslog_hostname = arrayindex(regextract(_raw_log, "(?:\S+\s+){3}(\S+)"), 0),
    syslog_process_name = rtrim(arrayindex(regextract(_raw_log, "(?:\S+\s+){4}(\S+)"), 0), ":"),
    json_payload = arrayindex(regextract(_raw_log, "(?:\S+\s+){5}(\{.+)"), 0)
| alter syslog_facility = floor(divide(syslog_priority, 8))
| alter syslog_severity = to_string(subtract(syslog_priority, multiply(syslog_facility, 8)))
| alter
    agent_id = to_string(json_payload -> agent.id),
    agent_name = json_payload -> agent.name,
    app_host = json_payload -> app_host,
    client_device = json_payload -> client_device,
    client_id = to_string(json_payload -> client_id),
    client_name = json_payload -> client_name,
    context = to_string(json_payload -> data.context),
    data_type = json_payload -> data.type,
    description = json_payload -> description,
    email = json_payload -> data.email,
    email_attachments = json_payload -> data.attachments[],
    email_id = to_string(json_payload -> data.mail.id),
    email_recipients = arraymap(json_payload -> data.recipients[], "@element" -> name),
    email_sender = json_payload -> data.mail.sender,
    email_subject = json_payload -> data.mail.subject,
    error_msg = json_payload -> data.error_msg,
    event = json_payload -> event,
    file_file_id = to_string(json_payload -> data.file.file_id),
    file_hash = json_payload -> data.file.hash,
    file_hash_algo = json_payload -> data.file.hash_algo,
    file_id = to_string(json_payload -> data.file.id),
    file_mime_type = coalesce(json_payload -> data.mime, json_payload -> data.file.mime),
    file_name = coalesce(json_payload -> data.file.name, json_payload -> data.source_file.name),
    file_owner_id = to_string(json_payload -> data.file_owner.id),
    file_owner_name = json_payload -> data.file_owner.name,
    file_path = coalesce(json_payload -> data.file.path, json_payload -> data.source_file.path),
    file_size = json_payload -> data.file.size,
    file_fingerprints = json_payload -> data.file.fingerprints[],
    folder_name = coalesce(json_payload -> data.folder.name, json_payload -> data.dest_folder.name, json_payload -> data.source_folder.name, json_payload -> data.parent_folder.name),
    folder_path = coalesce(json_payload -> data.folder.path, json_payload -> data.dest_folder.path, json_payload -> data.source_folder.path, json_payload -> data.parent_folder.path),
    full_log = json_payload -> full_log,
    host_id = to_string(json_payload -> data.host_id),
    id = to_string(json_payload -> id),
    is_folder_upload = to_string(json_payload -> data.is_folder_upload),
    is_malicious = to_string(json_payload -> data.malicious),
    is_prohibited = to_string(json_payload -> data.prohibited),
    is_quarantined = to_string(json_payload -> data.quarantined),
    is_skipped = to_string(json_payload -> data.skipped),
    is_successful = to_string(json_payload -> successful),
    node_ip = json_payload -> data.node_ip,
    report_name = json_payload -> data.report_name,
    resource_name = json_payload -> data.name,
    role_name = json_payload -> data.role_name,
    rule_id = to_string(json_payload -> rule.id),
    rule_name = json_payload -> rule.name,
    scanning_type = json_payload -> data.scanning_type,
    session_id = to_string(json_payload -> data.session),
    src_ip = coalesce(json_payload -> data.srcip, json_payload -> user_ip),
    status_reason = json_payload -> data.status_reason,
    sw_version = json_payload -> data.sw_version,
    syscheck_gname_after = json_payload -> syscheck.gname_after,
    syscheck_path = json_payload -> syscheck.path,
    syscheck_symbolic_path = json_payload -> syscheck.symbolic_path,
    syscheck_uid_after = to_string(json_payload -> syscheck.uid_after),
    syscheck_uname_after = json_payload -> syscheck.uname_after,
    target_client_id = to_string(json_payload -> data.client_id),
    target_user_guid = to_string(json_payload -> data.user.guid),
    target_user_id = to_string(json_payload -> data.user.id),
    target_user_name = json_payload -> data.user.name,
    target_users_names = arraystring(arraymap(json_payload -> data.users[], "@element" -> name), ","),
    tenant_id = to_string(json_payload -> tenant_id),
    token = json_payload -> token,
    url_host = json_payload -> url_host,
    user_agent = json_payload -> user_agent,
    user_id = to_string(json_payload -> user_id),
    user_name = coalesce(json_payload -> user_name, json_payload -> data.srcuser, json_payload -> data.dstuser),
    user_type = json_payload -> user_type
| alter 
    email_attachments_filename = arraystring(arraymap(email_attachments, "@element" -> name), ","),
    email_attachments_extension = arraystring(arraymap(email_attachments, arrayindex(regextract("@element" -> name, "\.(\w+)"), 0)), ","),
    email_attachments_mime_type = arraystring(arraymap(email_attachments, "@element" -> mime), ","),
    email_attachments_md5 = arraymap(email_attachments, if("@element" -> hash_algo = "md5", "@element" -> hash)),
    email_attachments_sha256 = arraymap(email_attachments, if("@element" -> hash_algo ~= "(?i)sha.+256", "@element" -> hash)),
    email_attachments_size = to_integer(arrayindex(email_attachments, 0) -> size),
    file_hash_md5 = if(file_hash_algo ~= "(?i)md5", file_hash),
    file_hash_sha256 = if(file_hash_algo ~= "(?i)sha.+256", file_hash),
    file_fingerprints_md5 = arrayindex(arraymap(file_fingerprints , if("@element" -> algo = "md5", "@element" -> hash)), 0),
    file_fingerprints_sha256 = arrayindex(arraymap(file_fingerprints , if("@element" -> algo ~= "(?i)sha.+256", "@element" -> hash)), 0),
    node_ipv4 = if(node_ip ~= "(?:\d{1,3}\.){3}\d{1,3}", node_ip),
    node_ipv6 = if(node_ip ~= "(?:[a-fA-F\d]{0,4}\:){7}[\wa-fA-F]{0,4}", node_ip),
    src_ipv4 = if(src_ip ~= "(?:\d{1,3}\.){3}\d{1,3}", src_ip),
    src_ipv6 = if(src_ip ~= "(?:[a-fA-F\d]{0,4}\:){7}[\wa-fA-F]{0,4}", src_ip),
    target_client_name = if(target_client_id != null, resource_name),
    verdict = if(is_malicious = "1", "MALICIOUS", is_prohibited = "1", "PROHIBITED", is_quarantined = "1", "QUARANTINED", is_skipped = "1", "SKIPPED")
| alter attachments_md5_fingerprints = arraymap(email_attachments, arraystring(arrayfilter("@element" -> fingerprints[], "@element" -> algo = "md5"), ","))
| alter attachments_sha256_fingerprints = arraymap(email_attachments, arraystring(arrayfilter("@element" -> fingerprints[], "@element" -> algo ~= "sha.*256"), ","))
| alter attachments_md5_hashes = arraystring(arraymap(attachments_md5_fingerprints, "@element" -> hash), ",")
| alter attachments_sha256_hashes = arraystring(arraymap(attachments_sha256_fingerprints, "@element" -> hash), ",")
| alter 
    xdm.alert.severity = syslog_severity,
    xdm.email.attachment.extension = email_attachments_extension, 
    xdm.email.attachment.filename = email_attachments_filename,
    xdm.email.attachment.file_type = email_attachments_mime_type, 
    xdm.email.attachment.md5 = arraystring(email_attachments_md5, ","),
    xdm.email.attachment.sha256 = arraystring(email_attachments_sha256, ","),
    xdm.email.attachment.size = email_attachments_size,
    xdm.email.message_id = email_id,
    xdm.email.recipients = if(array_length(email_recipients) > 0, email_recipients),
    xdm.email.sender = email_sender,
    xdm.email.subject = email_subject,
    xdm.event.description = coalesce(full_log, description),
    xdm.event.log_level = if(syslog_severity = "0", XDM_CONST.LOG_LEVEL_EMERGENCY, syslog_severity = "1", XDM_CONST.LOG_LEVEL_ALERT, syslog_severity = "2", XDM_CONST.LOG_LEVEL_CRITICAL, syslog_severity = "3", XDM_CONST.LOG_LEVEL_ERROR, syslog_severity = "4", XDM_CONST.LOG_LEVEL_WARNING, syslog_severity = "5", XDM_CONST.LOG_LEVEL_NOTICE, syslog_severity = "6", XDM_CONST.LOG_LEVEL_INFORMATIONAL, syslog_severity = "7", XDM_CONST.LOG_LEVEL_DEBUG, syslog_severity),
    xdm.event.operation = if(is_folder_upload = "1", XDM_CONST.OPERATION_TYPE_DIR_CREATE), // enrich
    xdm.event.outcome = if(is_successful = "1", XDM_CONST.OUTCOME_SUCCESS, is_successful = "0", XDM_CONST.OUTCOME_FAILED),
    xdm.event.outcome_reason = coalesce(error_msg, status_reason),
    xdm.event.type = event,
    xdm.network.rule = concat(rule_id, " (", rule_name, ")"),
    xdm.network.session_id = coalesce(session_id, context),
    xdm.observer.action = verdict,
    xdm.observer.name = syslog_hostname,
    xdm.observer.version = sw_version,
    xdm.session_context_id = context,
    xdm.source.agent.identifier = concat(agent_id, " (", agent_name, ")"),
    xdm.source.agent.type = scanning_type,
    xdm.source.application.name = concat(client_id, " (", client_name, ")"),
    xdm.source.host.hostname = if(client_device != "None", client_device),
    xdm.source.ipv4 = src_ipv4,
    xdm.source.ipv6 = src_ipv6,
    xdm.source.process.name = syslog_process_name,
    xdm.source.user_agent = user_agent,
    xdm.source.user.groups = if(syscheck_gname_after != null, arraycreate(syscheck_gname_after)),
    xdm.source.user.identifier = coalesce(user_id, syscheck_uid_after),
    xdm.source.user.user_type = user_type,
    xdm.source.user.username = coalesce(user_name, syscheck_uname_after),
    xdm.target.application.name = concat(target_client_id, " (", target_client_name, ")"),
    xdm.target.application.version = sw_version,
    xdm.target.file.directory = folder_name,
    xdm.target.file.file_type = file_mime_type,
    xdm.target.file.filename = coalesce(file_name, if(event ~= "file|upload" or description ~= "file", resource_name)),
    xdm.target.file.md5 = coalesce(file_hash_md5, file_fingerprints_md5),
    xdm.target.file.path = coalesce(file_path, folder_path, syscheck_path, syscheck_symbolic_path),
    xdm.target.file.sha256 = coalesce(file_hash_sha256, file_fingerprints_sha256),
    xdm.target.file.size = to_integer(file_size),
    xdm.target.host.device_id = coalesce(host_id, tenant_id),
    xdm.target.host.hostname = coalesce(app_host, url_host),
    xdm.target.ipv4 = node_ipv4,
    xdm.target.ipv6 = node_ipv6,
    xdm.target.resource.id = coalesce(id, target_user_guid, file_file_id, file_id),
    xdm.target.resource.name = coalesce(name, rule_name, role_name, report_name),
    xdm.target.resource.parent_id = coalesce(file_owner_id, file_owner_name),
    xdm.target.resource.type = data_type,
    xdm.target.resource.value = token,
    xdm.target.user.identifier = target_user_id,
    xdm.target.user.username = coalesce(target_user_name, target_users_names, email, file_owner_name);

// Single-Line Format
filter _raw_log !~= "^(?:\S+\s+){5}\{" and _raw_log not contains "Activity Type:"
| alter 
    syslog_priority = to_integer(arrayindex(regextract(_raw_log, "^\<(\d{1,3})\>\s*\w+"), 0)),
    syslog_hostname = arrayindex(regextract(_raw_log, "(?:\S+\s+){3}(\S+)"), 0),
    syslog_process_name = rtrim(arrayindex(regextract(_raw_log, "(?:\S+\s+){4}(\S+)"), 0), ":"),
    syslog_payload = arrayindex(regextract(_raw_log, "(?:\S+\s+){5}(.+)"), 0)
| alter syslog_facility = floor(divide(syslog_priority, 8))
| alter syslog_severity = to_string(subtract(syslog_priority, multiply(syslog_facility, 8)))
| alter 
    activity_type = arrayindex(regextract(syslog_payload, "type=(\S+)"), 0),
    audit_uid = arrayindex(regextract(syslog_payload, "auid=(\d+)"), 0),
    auth_service = arrayindex(regextract(syslog_payload, "grantors=(\S+)"), 0),
    command_name = trim(arrayindex(regextract(syslog_payload, "comm=(\S+)"), 0), "\""),
    effective_group_id = arrayindex(regextract(syslog_payload, "egid=(\d+)"), 0),
    effective_uid = arrayindex(regextract(syslog_payload, "euid=(\d+)"), 0),
    event_id = arrayindex(regextract(syslog_payload, "audit\(\d+[^\:]+\:(\d+)"), 0),
    executable = trim(arrayindex(regextract(syslog_payload, "(?:comm|exe|cmd|proctitle)=(\S+)"), 0), "\""),
    exec_args = arraystring(regextract(_raw_log, "a\d+=\"([^\"]+)"), " "),
    group_id = arrayindex(regextract(syslog_payload, "\s+gid=(\d+)"), 0),
    hostname = arrayindex(regextract(syslog_payload, "hostname=([\w\.\-]+)"), 0),
    ip_address = arrayindex(regextract(syslog_payload, "addr=([\d\.\:a-fA-F]+)"), 0),
    is_successful = arrayindex(regextract(syslog_payload, "success=(\w+)"), 0), // yes/no
    msg = rtrim(arrayindex(regextract(syslog_payload, "msg=\'(.+)"), 0), "\'"),
    name = trim(arrayindex(regextract(syslog_payload, "name=(\S+)"), 0), "\""),
    node = arrayindex(regextract(syslog_payload, "node=(\S+)"), 0),
    o_uid = arrayindex(regextract(syslog_payload, "ouid=(\d+)"), 0),
    objtype = arrayindex(regextract(syslog_payload, "objtype=(\S+)"), 0),
    operation = arrayindex(regextract(syslog_payload, "op=(\S+)"), 0),
    pid = arrayindex(regextract(syslog_payload, "\s+pid=(\d+)"), 0),
    ppid = arrayindex(regextract(syslog_payload, "ppid=(\d+)"), 0),
    result = arrayindex(regextract(syslog_payload, "res=(\w+)"), 0),
    session_id = arrayindex(regextract(syslog_payload, "ses=(\S+)"), 0),
    set_group_id = arrayindex(regextract(syslog_payload, "sgid=(\d+)"), 0),
    syscall_number = arrayindex(regextract(syslog_payload, "syscall=(\d+)"), 0),
    user_id = arrayindex(regextract(syslog_payload, "\s+uid=(\d+)"), 0),
    user_account = arrayindex(regextract(syslog_payload, "acct=\"?([^\"]+)"), 0),
    working_directory = arrayindex(regextract(syslog_payload, "cwd=\"?([^\"]+)"), 0)
| alter 
    process_cmd = coalesce(executable, exec_args, command_name),
    src_ipv4 = if(ip_address ~= "(?:\d{1,3}\.){3}\d{1,3}", ip_address),
    src_ipv6 = if(ip_address ~= "(?:[a-fA-F\d]{0,4}\:){7}[\wa-fA-F]{0,4}", ip_address)
| alter 
    xdm.alert.severity = syslog_severity,
    xdm.auth.service = auth_service,
    xdm.event.description = coalesce(msg, syslog_payload),
    xdm.event.id = event_id,
    xdm.event.log_level = if(syslog_severity = "0", XDM_CONST.LOG_LEVEL_EMERGENCY, syslog_severity = "1", XDM_CONST.LOG_LEVEL_ALERT, syslog_severity = "2", XDM_CONST.LOG_LEVEL_CRITICAL, syslog_severity = "3", XDM_CONST.LOG_LEVEL_ERROR, syslog_severity = "4", XDM_CONST.LOG_LEVEL_WARNING, syslog_severity = "5", XDM_CONST.LOG_LEVEL_NOTICE, syslog_severity = "6", XDM_CONST.LOG_LEVEL_INFORMATIONAL, syslog_severity = "7", XDM_CONST.LOG_LEVEL_DEBUG, syslog_severity),
    xdm.event.operation = operation,
    xdm.event.outcome = if(is_successful = "yes" or result = "success", XDM_CONST.OUTCOME_SUCCESS, is_successful = "no" or result ~= "fail", XDM_CONST.OUTCOME_FAILED),
    xdm.event.type = coalesce(activity_type, syslog_process_name),
    xdm.network.session_id = session_id,
    xdm.observer.name = syslog_hostname,
    xdm.session_context_id = session_id,
    xdm.source.host.hostname = coalesce(node, syslog_hostname),
    xdm.source.ipv4 = src_ipv4,
    xdm.source.ipv6 = src_ipv6,
    xdm.source.process.name = syslog_process_name,
    xdm.source.user.groups = if(group_id != null, arraycreate(group_id)),
    xdm.source.user.identifier = coalesce(audit_uid, user_id),
    xdm.source.user.username = user_account,
    xdm.target.host.hostname = hostname, 
    xdm.target.process.command_line = process_cmd,
    xdm.target.process.name = coalesce(command_name, syscall_number), 
    xdm.target.process.pid = to_integer(pid),
    xdm.target.process.parent_id = ppid, 
    xdm.target.process.executable.path = working_directory,
    xdm.target.resource.name = if(name != "?", name),
    xdm.target.resource.type = objtype,
    xdm.target.user.groups = arraydistinct(arraycreate(set_group_id, effective_group_id)),
    xdm.target.user.identifier = coalesce(effective_uid, o_uid, user_id, audit_uid),
    xdm.target.user.username = user_account;

// Activity-Group & Activity-Type Format
filter _raw_log !~= "^(?:\S+\s+){5}\{" and _raw_log contains "Activity Type:"
| alter 
    syslog_priority = to_integer(arrayindex(regextract(_raw_log, "^\<(\d{1,3})\>\s*\w+"), 0)),
    syslog_hostname = arrayindex(regextract(_raw_log, "(?:\S+\s+){3}(\S+)"), 0),
    syslog_process_name = rtrim(arrayindex(regextract(_raw_log, "(?:\S+\s+){4}(\S+)"), 0), ":"),
    syslog_payload = arrayindex(regextract(_raw_log, "(?:\S+\s+){5}(.+)"), 0)
| alter syslog_facility = floor(divide(syslog_priority, 8))
| alter syslog_severity = to_string(subtract(syslog_priority, multiply(syslog_facility, 8)))
| alter 
    activity_type = arrayindex(regextract(syslog_payload, "Activity Type:\s*([^,]+)"), 0),
    activity_group = arrayindex(regextract(syslog_payload, "Activity Group:\s*([^,]+)"), 0),
    activity = arrayindex(regextract(syslog_payload, "Activity:\s*([^,]+)"), 0),
    server_ip = arrayindex(regextract(syslog_payload, "[:,]\s*([a-fA-F\d\.]+),\s*Activity"), 0),
    user_id = arrayindex(regextract(syslog_payload, "^\S+\s+id=(\d+)\s+\("), 0),
    username = arrayindex(regextract(syslog_payload, "^(\S+)\s+id=\d+\s+\("), 0)
| alter 
    src_ipv4 = if(server_ip ~= "(?:\d{1,3}\.){3}\d{1,3}", server_ip),
    src_ipv6 = if(server_ip ~= "(?:[a-fA-F\d]{0,4}\:){7}[\wa-fA-F]{0,4}", server_ip)
| alter 
    xdm.alert.severity = syslog_severity,
    xdm.event.description = coalesce(activity, syslog_payload),
    xdm.event.log_level = if(syslog_severity = "0", XDM_CONST.LOG_LEVEL_EMERGENCY, syslog_severity = "1", XDM_CONST.LOG_LEVEL_ALERT, syslog_severity = "2", XDM_CONST.LOG_LEVEL_CRITICAL, syslog_severity = "3", XDM_CONST.LOG_LEVEL_ERROR, syslog_severity = "4", XDM_CONST.LOG_LEVEL_WARNING, syslog_severity = "5", XDM_CONST.LOG_LEVEL_NOTICE, syslog_severity = "6", XDM_CONST.LOG_LEVEL_INFORMATIONAL, syslog_severity = "7", XDM_CONST.LOG_LEVEL_DEBUG, syslog_severity),
    xdm.event.type = coalesce(activity_type, syslog_process_name),
    xdm.observer.name = syslog_hostname,
    xdm.observer.type = activity_group,
    xdm.source.host.hostname = syslog_hostname,
    xdm.source.ipv4 = src_ipv4,
    xdm.source.ipv6 = src_ipv6,
    xdm.source.process.name = syslog_process_name,
    xdm.source.user.identifier = user_id,
    xdm.source.user.username = username;