From 1f98f047c97e4b8badca136fc9bfe0e0055e2866 Mon Sep 17 00:00:00 2001 From: DGabri Date: Mon, 1 Jul 2024 10:21:30 +0200 Subject: [PATCH 1/6] Added verbose parameter to add historical flow info to the response --- .../modules/alert_store/flow_alert_store.lua | 167 ++++++++++-------- scripts/lua/rest/v2/get/flow/alert/list.lua | 3 +- 2 files changed, 100 insertions(+), 70 deletions(-) diff --git a/scripts/lua/modules/alert_store/flow_alert_store.lua b/scripts/lua/modules/alert_store/flow_alert_store.lua index 009ccad6d947..1bbcb6938bbe 100644 --- a/scripts/lua/modules/alert_store/flow_alert_store.lua +++ b/scripts/lua/modules/alert_store/flow_alert_store.lua @@ -3,9 +3,12 @@ -- local dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/alert_store/?.lua;" .. package.path +package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path +package.path = dirs.installdir .. "/scripts/lua/pro/modules/flow_db/?.lua;" .. package.path -- Import the classes library. local classes = require "classes" +require "http_lint" require "lua_utils" local alert_store = require "alert_store" @@ -20,6 +23,9 @@ local pools = require "pools" local historical_flow_utils = require "historical_flow_utils" local flow_alert_keys = require "flow_alert_keys" local href_icon = "" +local rest_utils = require "rest_utils" +local historical_flow_details_formatter = require "historical_flow_details_formatter" +local db_search_manager = require "db_search_manager" -- ############################################## @@ -230,7 +236,7 @@ function flow_alert_store:insert(alert) else hex_prefix = "X" end - --tprint(alert) + -- tprint(alert) local alert_key = alert_consts.getAlertType(alert.alert_id, alert.entity_id) local mitre_info = alert_consts.getAlertMitreInfo(alert_key) local alert_json = json.decode(alert.json) @@ -248,67 +254,34 @@ function flow_alert_store:insert(alert) -- io.write("---------------------------\n") tprint(debug.traceback()) tprint(alert.flow_risk_bitmap) io.write("---------------------------\n") local fmt = "INSERT INTO %s " .. - "(%salert_id, alert_status, alert_category, interface_id, tstamp, tstamp_end, severity, ip_version, cli_ip, srv_ip, cli_port, srv_port, vlan_id, " .. - "is_cli_attacker, is_cli_victim, is_srv_attacker, is_srv_victim, proto, l7_proto, l7_master_proto, l7_cat, " .. - "cli_name, srv_name, cli_country, srv_country, cli_blacklisted, srv_blacklisted, cli_location, srv_location, " .. - "cli2srv_bytes, srv2cli_bytes, cli2srv_pkts, srv2cli_pkts, first_seen, community_id, score, " .. - "flow_risk_bitmap, alerts_map, cli_host_pool_id, srv_host_pool_id, cli_network, srv_network, probe_ip, input_snmp, output_snmp, " .. - "json, info) " .. - "VALUES (%s%u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', '%s', " .. - "'%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', %u, %u, %s'%s', %u, %u, %u, %u, '%s', %u, %u, '%s', '%s'); " - - local insert_stmt = string.format(fmt, - self:get_write_table_name(), - extra_columns, - extra_values, - alert.alert_id, - ternary(alert.acknowledged, alert_consts.alert_status.acknowledged.alert_status_id, 0), - alert.alert_category, - self:_convert_ifid(interface.getId()), - alert.first_seen, - alert.tstamp, -- 10 - map_score_to_severity(alert.score), - alert.ip_version, - alert.cli_ip, - alert.srv_ip, - alert.cli_port, - alert.srv_port, - alert.vlan_id, - ternary(alert.is_cli_attacker, 1, 0), - ternary(alert.is_cli_victim, 1, 0), - ternary(alert.is_srv_attacker, 1, 0), -- 20 - ternary(alert.is_srv_victim, 1, 0), - alert.proto, - alert.l7_proto, - alert.l7_master_proto, - alert.l7_cat, - self:_escape(alert.cli_name), - self:_escape(alert.srv_name), - alert.cli_country_name, - alert.srv_country_name, - ternary(alert.cli_blacklisted, 1, 0), -- 30 - ternary(alert.srv_blacklisted, 1, 0), - alert.cli_location or 0, - alert.srv_location or 0, - alert.cli2srv_bytes, - alert.srv2cli_bytes, - alert.cli2srv_packets, alert.srv2cli_packets, - alert.first_seen, - alert.community_id, - alert.score, - alert.flow_risk_bitmap or 0, -- 40 - hex_prefix, - alert.alerts_map, - tonumber(alert.cli_host_pool_id or pools.DEFAULT_POOL_ID), - tonumber(alert.srv_host_pool_id or pools.DEFAULT_POOL_ID), - tonumber(alert.cli_network or network_consts.UNKNOWN_NETWORK), - tonumber(alert.srv_network or network_consts.UNKNOWN_NETWORK), - alert.probe_ip, - alert.input_snmp, - alert.output_snmp, - self:_escape(alert.json), -- 50 - self:_escape(alert.info or '')) - + "(%salert_id, alert_status, alert_category, interface_id, tstamp, tstamp_end, severity, ip_version, cli_ip, srv_ip, cli_port, srv_port, vlan_id, " .. + "is_cli_attacker, is_cli_victim, is_srv_attacker, is_srv_victim, proto, l7_proto, l7_master_proto, l7_cat, " .. + "cli_name, srv_name, cli_country, srv_country, cli_blacklisted, srv_blacklisted, cli_location, srv_location, " .. + "cli2srv_bytes, srv2cli_bytes, cli2srv_pkts, srv2cli_pkts, first_seen, community_id, score, " .. + "flow_risk_bitmap, alerts_map, cli_host_pool_id, srv_host_pool_id, cli_network, srv_network, probe_ip, input_snmp, output_snmp, " .. + "json, info) " .. + "VALUES (%s%u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', '%s', " .. + "'%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', %u, %u, %s'%s', %u, %u, %u, %u, '%s', %u, %u, '%s', '%s'); " + + local insert_stmt = string.format(fmt, self:get_write_table_name(), extra_columns, extra_values, alert.alert_id, + ternary(alert.acknowledged, alert_consts.alert_status.acknowledged.alert_status_id, 0), alert.alert_category, + self:_convert_ifid(interface.getId()), alert.first_seen, alert.tstamp, -- 10 + map_score_to_severity(alert.score), alert.ip_version, alert.cli_ip, alert.srv_ip, alert.cli_port, + alert.srv_port, alert.vlan_id, ternary(alert.is_cli_attacker, 1, 0), ternary(alert.is_cli_victim, 1, 0), + ternary(alert.is_srv_attacker, 1, 0), -- 20 + ternary(alert.is_srv_victim, 1, 0), alert.proto, alert.l7_proto, alert.l7_master_proto, alert.l7_cat, + self:_escape(alert.cli_name), self:_escape(alert.srv_name), alert.cli_country_name, alert.srv_country_name, + ternary(alert.cli_blacklisted, 1, 0), -- 30 + ternary(alert.srv_blacklisted, 1, 0), alert.cli_location or 0, alert.srv_location or 0, alert.cli2srv_bytes, + alert.srv2cli_bytes, alert.cli2srv_packets, alert.srv2cli_packets, alert.first_seen, alert.community_id, + alert.score, alert.flow_risk_bitmap or 0, -- 40 + hex_prefix, alert.alerts_map, tonumber(alert.cli_host_pool_id or pools.DEFAULT_POOL_ID), + tonumber(alert.srv_host_pool_id or pools.DEFAULT_POOL_ID), + tonumber(alert.cli_network or network_consts.UNKNOWN_NETWORK), + tonumber(alert.srv_network or network_consts.UNKNOWN_NETWORK), alert.probe_ip, alert.input_snmp, + alert.output_snmp, self:_escape(alert.json), -- 50 + self:_escape(alert.info or '')) + -- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt) return interface.alert_store_query(insert_stmt) @@ -654,7 +627,7 @@ function flow_alert_store:_add_additional_request_filters() self:add_filter_condition_list('output_snmp', output_snmp) self:add_filter_condition_list('snmp_interface', snmp_interface) self:add_filter_condition_list('community_id', community_id) - + self:add_filter_condition_list('cli2srv_bytes', cli2srv_bytes) self:add_filter_condition_list('srv2cli_bytes', srv2cli_bytes) @@ -815,7 +788,23 @@ local RNAME = { INFO = { name = "info", export = true - } + }, + + HIST_FLOW_CLI_INFO = { + name = "hist_flow_cli_info", + export = true + }, + + HIST_FLOW_SRV_INFO = { + name = "hist_flow_srv_info", + export = true + }, + + HIST_FLOW_INFO = { + name = "hist_flow_info", + export = true + }, + } -- ############################################## @@ -859,7 +848,8 @@ end -- ############################################## -- @brief Convert an alert coming from the DB (value) to a record returned by the REST API -function flow_alert_store:format_record(value, no_html) +-- if verbose == true returns historical flow data info +function flow_alert_store:format_record(value, no_html, verbose) local record = self:format_json_record_common(value, alert_entities.flow.entity_id, no_html) local alert_info = alert_utils.getAlertInfo(value) local alert_name = alert_consts.alertTypeLabel(tonumber(value["alert_id"]), true --[[ no_html --]] , @@ -877,7 +867,7 @@ function flow_alert_store:format_record(value, no_html) end -- Add link to active flow - local flow_related_info = addExtraFlowInfo(alert_json, value, no_html --[[ Send these info in json format ]]) + local flow_related_info = addExtraFlowInfo(alert_json, value, no_html --[[ Send these info in json format ]] ) local alert_risk if tonumber(value.alert_id) then @@ -904,6 +894,47 @@ function flow_alert_store:format_record(value, no_html) end end + -- add additional flow related info + if (verbose == "true") then + + -- get alert details page info + local flow = db_search_manager.get_flow(value["rowid"], value["tstamp_epoch"], "") + + -- format cli flow info + local flow_cli_info = { + isVictim = tonumber(flow["IS_CLI_VICTIM"] or ""), + isAttacker = tonumber(flow["IS_CLI_ATTACKER"] or "") + } + + -- format srv flow info + local flow_srv_ip = { + isVictim = tonumber(flow["IS_SRV_VICTIM"] or ""), + isAttacker = tonumber(flow["IS_SRV_ATTACKER"] or "") + } + + -- detailed flow info -> /lua/pro/db_flow_details.lua page + local detailed_flow_info = { + alert_name = alert_name, + -- dst2src_bytes = tonumber(flow["DST2SRC_BYTES"] or 0), + -- src2dst_bytes = tonumber(flow["SRC2DST_BYTES"] or 0), + status = tonumber(flow["STATUS"] or ""), + alert_status = tonumber(flow["ALERT_STATUS"] or ""), + packets = tonumber(flow["PACKETS"] or ""), + major_connection_state = tonumber(flow["MAJOR_CONNECTION_STATE"] or ""), + dst2src_tcp_flags = tonumber(flow["DST2SRC_TCP_FLAGS"] or ""), + src2dst_tcp_flags = tonumber(flow["SRC2DST_TCP_FLAGS"] or ""), + latency_to_srv_us = tonumber(flow["SERVER_NW_LATENCY_US"] or ""), + latency_to_cli_us = tonumber(flow["CLIENT_NW_LATENCY_US"] or ""), + -- protocol = tonumber(flow["PROTOCOL"] or ""), + flow_risk = tonumber(flow["FLOW_RISK"] or ""), + } + + -- add extracted data to JSON response + record[RNAME.HIST_FLOW_CLI_INFO.name] = flow_cli_info + record[RNAME.HIST_FLOW_SRV_INFO.name] = flow_cli_info + record[RNAME.HIST_FLOW_INFO.name] = detailed_flow_info + end + local other_alerts_by_score = {} -- Table used to keep messages ordered by score local additional_alerts = {} @@ -1554,6 +1585,4 @@ function flow_alert_store:get_alert_details(value) return details end --- ############################################## - -return flow_alert_store +return flow_alert_store \ No newline at end of file diff --git a/scripts/lua/rest/v2/get/flow/alert/list.lua b/scripts/lua/rest/v2/get/flow/alert/list.lua index d0d7e5a2dba2..65def2603747 100644 --- a/scripts/lua/rest/v2/get/flow/alert/list.lua +++ b/scripts/lua/rest/v2/get/flow/alert/list.lua @@ -23,6 +23,7 @@ local ifid = _GET["ifid"] local format = _GET["format"] local epoch_begin = _GET["epoch_begin"] local epoch_end = _GET["epoch_end"] +local verbose = _GET["verbose"] local no_html = false local download = false @@ -69,7 +70,7 @@ if not download then for _, _value in ipairs(alerts or {}) do -- tprint(_value) - res[#res + 1] = flow_alert_store:format_record(_value, no_html) + res[#res + 1] = flow_alert_store:format_record(_value, no_html, verbose) end if format == "txt" then From 5d382efbf680b8f73f50623096c0b27810a32ced Mon Sep 17 00:00:00 2001 From: DGabri Date: Mon, 1 Jul 2024 10:28:08 +0200 Subject: [PATCH 2/6] Formatted insert_stmt as it was --- .../modules/alert_store/flow_alert_store.lua | 77 +++++++++++++------ 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/scripts/lua/modules/alert_store/flow_alert_store.lua b/scripts/lua/modules/alert_store/flow_alert_store.lua index 1bbcb6938bbe..caf8a62b2c7a 100644 --- a/scripts/lua/modules/alert_store/flow_alert_store.lua +++ b/scripts/lua/modules/alert_store/flow_alert_store.lua @@ -263,28 +263,61 @@ function flow_alert_store:insert(alert) "VALUES (%s%u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', '%s', " .. "'%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', %u, %u, %s'%s', %u, %u, %u, %u, '%s', %u, %u, '%s', '%s'); " - local insert_stmt = string.format(fmt, self:get_write_table_name(), extra_columns, extra_values, alert.alert_id, - ternary(alert.acknowledged, alert_consts.alert_status.acknowledged.alert_status_id, 0), alert.alert_category, - self:_convert_ifid(interface.getId()), alert.first_seen, alert.tstamp, -- 10 - map_score_to_severity(alert.score), alert.ip_version, alert.cli_ip, alert.srv_ip, alert.cli_port, - alert.srv_port, alert.vlan_id, ternary(alert.is_cli_attacker, 1, 0), ternary(alert.is_cli_victim, 1, 0), - ternary(alert.is_srv_attacker, 1, 0), -- 20 - ternary(alert.is_srv_victim, 1, 0), alert.proto, alert.l7_proto, alert.l7_master_proto, alert.l7_cat, - self:_escape(alert.cli_name), self:_escape(alert.srv_name), alert.cli_country_name, alert.srv_country_name, - ternary(alert.cli_blacklisted, 1, 0), -- 30 - ternary(alert.srv_blacklisted, 1, 0), alert.cli_location or 0, alert.srv_location or 0, alert.cli2srv_bytes, - alert.srv2cli_bytes, alert.cli2srv_packets, alert.srv2cli_packets, alert.first_seen, alert.community_id, - alert.score, alert.flow_risk_bitmap or 0, -- 40 - hex_prefix, alert.alerts_map, tonumber(alert.cli_host_pool_id or pools.DEFAULT_POOL_ID), - tonumber(alert.srv_host_pool_id or pools.DEFAULT_POOL_ID), - tonumber(alert.cli_network or network_consts.UNKNOWN_NETWORK), - tonumber(alert.srv_network or network_consts.UNKNOWN_NETWORK), alert.probe_ip, alert.input_snmp, - alert.output_snmp, self:_escape(alert.json), -- 50 - self:_escape(alert.info or '')) - - -- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt) - - return interface.alert_store_query(insert_stmt) + local insert_stmt = string.format(fmt, + self:get_write_table_name(), + extra_columns, + extra_values, + alert.alert_id, + ternary(alert.acknowledged, alert_consts.alert_status.acknowledged.alert_status_id, 0), + alert.alert_category, + self:_convert_ifid(interface.getId()), + alert.first_seen, + alert.tstamp, -- 10 + map_score_to_severity(alert.score), + alert.ip_version, + alert.cli_ip, + alert.srv_ip, + alert.cli_port, + alert.srv_port, + alert.vlan_id, + ternary(alert.is_cli_attacker, 1, 0), + ternary(alert.is_cli_victim, 1, 0), + ternary(alert.is_srv_attacker, 1, 0), -- 20 + ternary(alert.is_srv_victim, 1, 0), + alert.proto, + alert.l7_proto, + alert.l7_master_proto, + alert.l7_cat, + self:_escape(alert.cli_name), + self:_escape(alert.srv_name), + alert.cli_country_name, + alert.srv_country_name, + ternary(alert.cli_blacklisted, 1, 0), -- 30 + ternary(alert.srv_blacklisted, 1, 0), + alert.cli_location or 0, + alert.srv_location or 0, + alert.cli2srv_bytes, + alert.srv2cli_bytes, + alert.cli2srv_packets, alert.srv2cli_packets, + alert.first_seen, + alert.community_id, + alert.score, + alert.flow_risk_bitmap or 0, -- 40 + hex_prefix, + alert.alerts_map, + tonumber(alert.cli_host_pool_id or pools.DEFAULT_POOL_ID), + tonumber(alert.srv_host_pool_id or pools.DEFAULT_POOL_ID), + tonumber(alert.cli_network or network_consts.UNKNOWN_NETWORK), + tonumber(alert.srv_network or network_consts.UNKNOWN_NETWORK), + alert.probe_ip, + alert.input_snmp, + alert.output_snmp, + self:_escape(alert.json), -- 50 + self:_escape(alert.info or '')) + + -- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt) + + return interface.alert_store_query(insert_stmt) end -- ############################################## From 110796ebadb6739596eb8b5e2926460c060e7422 Mon Sep 17 00:00:00 2001 From: DGabri Date: Mon, 1 Jul 2024 10:31:07 +0200 Subject: [PATCH 3/6] Appplied PR review suggestion --- scripts/lua/modules/alert_store/flow_alert_store.lua | 9 ++++++--- scripts/lua/rest/v2/get/flow/alert/list.lua | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/lua/modules/alert_store/flow_alert_store.lua b/scripts/lua/modules/alert_store/flow_alert_store.lua index caf8a62b2c7a..ff93753d1082 100644 --- a/scripts/lua/modules/alert_store/flow_alert_store.lua +++ b/scripts/lua/modules/alert_store/flow_alert_store.lua @@ -3,8 +3,11 @@ -- local dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/alert_store/?.lua;" .. package.path -package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path -package.path = dirs.installdir .. "/scripts/lua/pro/modules/flow_db/?.lua;" .. package.path + +if ntop.isPro() then + package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path + package.path = dirs.installdir .. "/scripts/lua/pro/modules/flow_db/?.lua;" .. package.path +end -- Import the classes library. local classes = require "classes" @@ -928,7 +931,7 @@ function flow_alert_store:format_record(value, no_html, verbose) end -- add additional flow related info - if (verbose == "true") then + if (verbose) then -- get alert details page info local flow = db_search_manager.get_flow(value["rowid"], value["tstamp_epoch"], "") diff --git a/scripts/lua/rest/v2/get/flow/alert/list.lua b/scripts/lua/rest/v2/get/flow/alert/list.lua index 65def2603747..0ba52508aa93 100644 --- a/scripts/lua/rest/v2/get/flow/alert/list.lua +++ b/scripts/lua/rest/v2/get/flow/alert/list.lua @@ -70,7 +70,7 @@ if not download then for _, _value in ipairs(alerts or {}) do -- tprint(_value) - res[#res + 1] = flow_alert_store:format_record(_value, no_html, verbose) + res[#res + 1] = flow_alert_store:format_record(_value, no_html, toboolean(verbose)) end if format == "txt" then From 9c9af3550cd56a219a47eabfe1b28fe08cd6c396 Mon Sep 17 00:00:00 2001 From: DGabri Date: Mon, 1 Jul 2024 10:37:12 +0200 Subject: [PATCH 4/6] Fixed enterprise version check --- .../modules/alert_store/flow_alert_store.lua | 94 ++++++------------- 1 file changed, 30 insertions(+), 64 deletions(-) diff --git a/scripts/lua/modules/alert_store/flow_alert_store.lua b/scripts/lua/modules/alert_store/flow_alert_store.lua index ff93753d1082..504611a11695 100644 --- a/scripts/lua/modules/alert_store/flow_alert_store.lua +++ b/scripts/lua/modules/alert_store/flow_alert_store.lua @@ -3,15 +3,14 @@ -- local dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/alert_store/?.lua;" .. package.path +package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path -if ntop.isPro() then - package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path +if ntop.isEnterprise() then package.path = dirs.installdir .. "/scripts/lua/pro/modules/flow_db/?.lua;" .. package.path end -- Import the classes library. local classes = require "classes" -require "http_lint" require "lua_utils" local alert_store = require "alert_store" @@ -266,61 +265,28 @@ function flow_alert_store:insert(alert) "VALUES (%s%u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', '%s', " .. "'%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', %u, %u, %s'%s', %u, %u, %u, %u, '%s', %u, %u, '%s', '%s'); " - local insert_stmt = string.format(fmt, - self:get_write_table_name(), - extra_columns, - extra_values, - alert.alert_id, - ternary(alert.acknowledged, alert_consts.alert_status.acknowledged.alert_status_id, 0), - alert.alert_category, - self:_convert_ifid(interface.getId()), - alert.first_seen, - alert.tstamp, -- 10 - map_score_to_severity(alert.score), - alert.ip_version, - alert.cli_ip, - alert.srv_ip, - alert.cli_port, - alert.srv_port, - alert.vlan_id, - ternary(alert.is_cli_attacker, 1, 0), - ternary(alert.is_cli_victim, 1, 0), - ternary(alert.is_srv_attacker, 1, 0), -- 20 - ternary(alert.is_srv_victim, 1, 0), - alert.proto, - alert.l7_proto, - alert.l7_master_proto, - alert.l7_cat, - self:_escape(alert.cli_name), - self:_escape(alert.srv_name), - alert.cli_country_name, - alert.srv_country_name, - ternary(alert.cli_blacklisted, 1, 0), -- 30 - ternary(alert.srv_blacklisted, 1, 0), - alert.cli_location or 0, - alert.srv_location or 0, - alert.cli2srv_bytes, - alert.srv2cli_bytes, - alert.cli2srv_packets, alert.srv2cli_packets, - alert.first_seen, - alert.community_id, - alert.score, - alert.flow_risk_bitmap or 0, -- 40 - hex_prefix, - alert.alerts_map, - tonumber(alert.cli_host_pool_id or pools.DEFAULT_POOL_ID), - tonumber(alert.srv_host_pool_id or pools.DEFAULT_POOL_ID), - tonumber(alert.cli_network or network_consts.UNKNOWN_NETWORK), - tonumber(alert.srv_network or network_consts.UNKNOWN_NETWORK), - alert.probe_ip, - alert.input_snmp, - alert.output_snmp, - self:_escape(alert.json), -- 50 - self:_escape(alert.info or '')) - - -- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt) - - return interface.alert_store_query(insert_stmt) + local insert_stmt = string.format(fmt, self:get_write_table_name(), extra_columns, extra_values, alert.alert_id, + ternary(alert.acknowledged, alert_consts.alert_status.acknowledged.alert_status_id, 0), alert.alert_category, + self:_convert_ifid(interface.getId()), alert.first_seen, alert.tstamp, -- 10 + map_score_to_severity(alert.score), alert.ip_version, alert.cli_ip, alert.srv_ip, alert.cli_port, + alert.srv_port, alert.vlan_id, ternary(alert.is_cli_attacker, 1, 0), ternary(alert.is_cli_victim, 1, 0), + ternary(alert.is_srv_attacker, 1, 0), -- 20 + ternary(alert.is_srv_victim, 1, 0), alert.proto, alert.l7_proto, alert.l7_master_proto, alert.l7_cat, + self:_escape(alert.cli_name), self:_escape(alert.srv_name), alert.cli_country_name, alert.srv_country_name, + ternary(alert.cli_blacklisted, 1, 0), -- 30 + ternary(alert.srv_blacklisted, 1, 0), alert.cli_location or 0, alert.srv_location or 0, alert.cli2srv_bytes, + alert.srv2cli_bytes, alert.cli2srv_packets, alert.srv2cli_packets, alert.first_seen, alert.community_id, + alert.score, alert.flow_risk_bitmap or 0, -- 40 + hex_prefix, alert.alerts_map, tonumber(alert.cli_host_pool_id or pools.DEFAULT_POOL_ID), + tonumber(alert.srv_host_pool_id or pools.DEFAULT_POOL_ID), + tonumber(alert.cli_network or network_consts.UNKNOWN_NETWORK), + tonumber(alert.srv_network or network_consts.UNKNOWN_NETWORK), alert.probe_ip, alert.input_snmp, + alert.output_snmp, self:_escape(alert.json), -- 50 + self:_escape(alert.info or '')) + + -- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt) + + return interface.alert_store_query(insert_stmt) end -- ############################################## @@ -839,7 +805,7 @@ local RNAME = { HIST_FLOW_INFO = { name = "hist_flow_info", export = true - }, + } } @@ -931,11 +897,11 @@ function flow_alert_store:format_record(value, no_html, verbose) end -- add additional flow related info - if (verbose) then - + if (verbose and ntop.isEnterprise()) then + -- get alert details page info local flow = db_search_manager.get_flow(value["rowid"], value["tstamp_epoch"], "") - + -- format cli flow info local flow_cli_info = { isVictim = tonumber(flow["IS_CLI_VICTIM"] or ""), @@ -962,7 +928,7 @@ function flow_alert_store:format_record(value, no_html, verbose) latency_to_srv_us = tonumber(flow["SERVER_NW_LATENCY_US"] or ""), latency_to_cli_us = tonumber(flow["CLIENT_NW_LATENCY_US"] or ""), -- protocol = tonumber(flow["PROTOCOL"] or ""), - flow_risk = tonumber(flow["FLOW_RISK"] or ""), + flow_risk = tonumber(flow["FLOW_RISK"] or "") } -- add extracted data to JSON response @@ -1621,4 +1587,4 @@ function flow_alert_store:get_alert_details(value) return details end -return flow_alert_store \ No newline at end of file +return flow_alert_store From 328cbd943bbe2ce698ad399b57036693af62ff42 Mon Sep 17 00:00:00 2001 From: DGabri Date: Mon, 1 Jul 2024 10:40:47 +0200 Subject: [PATCH 5/6] Fixed enterprise version check import --- scripts/lua/modules/alert_store/flow_alert_store.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lua/modules/alert_store/flow_alert_store.lua b/scripts/lua/modules/alert_store/flow_alert_store.lua index 504611a11695..dbe32a097618 100644 --- a/scripts/lua/modules/alert_store/flow_alert_store.lua +++ b/scripts/lua/modules/alert_store/flow_alert_store.lua @@ -7,6 +7,7 @@ package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path if ntop.isEnterprise() then package.path = dirs.installdir .. "/scripts/lua/pro/modules/flow_db/?.lua;" .. package.path + local db_search_manager = require "db_search_manager" end -- Import the classes library. @@ -27,7 +28,6 @@ local flow_alert_keys = require "flow_alert_keys" local href_icon = "" local rest_utils = require "rest_utils" local historical_flow_details_formatter = require "historical_flow_details_formatter" -local db_search_manager = require "db_search_manager" -- ############################################## From d98299520d524ecde2ac43c70c3965353b53d9e1 Mon Sep 17 00:00:00 2001 From: DGabri Date: Mon, 1 Jul 2024 10:43:11 +0200 Subject: [PATCH 6/6] Formatted insert_stmt as it was --- .../modules/alert_store/flow_alert_store.lua | 87 ++++++++++++++----- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/scripts/lua/modules/alert_store/flow_alert_store.lua b/scripts/lua/modules/alert_store/flow_alert_store.lua index dbe32a097618..19b0b3d052a2 100644 --- a/scripts/lua/modules/alert_store/flow_alert_store.lua +++ b/scripts/lua/modules/alert_store/flow_alert_store.lua @@ -265,28 +265,71 @@ function flow_alert_store:insert(alert) "VALUES (%s%u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', '%s', " .. "'%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', %u, %u, %s'%s', %u, %u, %u, %u, '%s', %u, %u, '%s', '%s'); " - local insert_stmt = string.format(fmt, self:get_write_table_name(), extra_columns, extra_values, alert.alert_id, - ternary(alert.acknowledged, alert_consts.alert_status.acknowledged.alert_status_id, 0), alert.alert_category, - self:_convert_ifid(interface.getId()), alert.first_seen, alert.tstamp, -- 10 - map_score_to_severity(alert.score), alert.ip_version, alert.cli_ip, alert.srv_ip, alert.cli_port, - alert.srv_port, alert.vlan_id, ternary(alert.is_cli_attacker, 1, 0), ternary(alert.is_cli_victim, 1, 0), - ternary(alert.is_srv_attacker, 1, 0), -- 20 - ternary(alert.is_srv_victim, 1, 0), alert.proto, alert.l7_proto, alert.l7_master_proto, alert.l7_cat, - self:_escape(alert.cli_name), self:_escape(alert.srv_name), alert.cli_country_name, alert.srv_country_name, - ternary(alert.cli_blacklisted, 1, 0), -- 30 - ternary(alert.srv_blacklisted, 1, 0), alert.cli_location or 0, alert.srv_location or 0, alert.cli2srv_bytes, - alert.srv2cli_bytes, alert.cli2srv_packets, alert.srv2cli_packets, alert.first_seen, alert.community_id, - alert.score, alert.flow_risk_bitmap or 0, -- 40 - hex_prefix, alert.alerts_map, tonumber(alert.cli_host_pool_id or pools.DEFAULT_POOL_ID), - tonumber(alert.srv_host_pool_id or pools.DEFAULT_POOL_ID), - tonumber(alert.cli_network or network_consts.UNKNOWN_NETWORK), - tonumber(alert.srv_network or network_consts.UNKNOWN_NETWORK), alert.probe_ip, alert.input_snmp, - alert.output_snmp, self:_escape(alert.json), -- 50 - self:_escape(alert.info or '')) - - -- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt) - - return interface.alert_store_query(insert_stmt) + local fmt = "INSERT INTO %s " .. + "(%salert_id, alert_status, alert_category, interface_id, tstamp, tstamp_end, severity, ip_version, cli_ip, srv_ip, cli_port, srv_port, vlan_id, " .. + "is_cli_attacker, is_cli_victim, is_srv_attacker, is_srv_victim, proto, l7_proto, l7_master_proto, l7_cat, " .. + "cli_name, srv_name, cli_country, srv_country, cli_blacklisted, srv_blacklisted, cli_location, srv_location, " .. + "cli2srv_bytes, srv2cli_bytes, cli2srv_pkts, srv2cli_pkts, first_seen, community_id, score, " .. + "flow_risk_bitmap, alerts_map, cli_host_pool_id, srv_host_pool_id, cli_network, srv_network, probe_ip, input_snmp, output_snmp, " .. + "json, info) " .. + "VALUES (%s%u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', '%s', '%s', " .. + "'%s', %u, %u, %u, %u, %u, %u, %u, %u, %u, '%s', %u, %u, %s'%s', %u, %u, %u, %u, '%s', %u, %u, '%s', '%s'); " + + local insert_stmt = string.format(fmt, + self:get_write_table_name(), + extra_columns, + extra_values, + alert.alert_id, + ternary(alert.acknowledged, alert_consts.alert_status.acknowledged.alert_status_id, 0), + alert.alert_category, + self:_convert_ifid(interface.getId()), + alert.first_seen, + alert.tstamp, -- 10 + map_score_to_severity(alert.score), + alert.ip_version, + alert.cli_ip, + alert.srv_ip, + alert.cli_port, + alert.srv_port, + alert.vlan_id, + ternary(alert.is_cli_attacker, 1, 0), + ternary(alert.is_cli_victim, 1, 0), + ternary(alert.is_srv_attacker, 1, 0), -- 20 + ternary(alert.is_srv_victim, 1, 0), + alert.proto, + alert.l7_proto, + alert.l7_master_proto, + alert.l7_cat, + self:_escape(alert.cli_name), + self:_escape(alert.srv_name), + alert.cli_country_name, + alert.srv_country_name, + ternary(alert.cli_blacklisted, 1, 0), -- 30 + ternary(alert.srv_blacklisted, 1, 0), + alert.cli_location or 0, + alert.srv_location or 0, + alert.cli2srv_bytes, + alert.srv2cli_bytes, + alert.cli2srv_packets, alert.srv2cli_packets, + alert.first_seen, + alert.community_id, + alert.score, + alert.flow_risk_bitmap or 0, -- 40 + hex_prefix, + alert.alerts_map, + tonumber(alert.cli_host_pool_id or pools.DEFAULT_POOL_ID), + tonumber(alert.srv_host_pool_id or pools.DEFAULT_POOL_ID), + tonumber(alert.cli_network or network_consts.UNKNOWN_NETWORK), + tonumber(alert.srv_network or network_consts.UNKNOWN_NETWORK), + alert.probe_ip, + alert.input_snmp, + alert.output_snmp, + self:_escape(alert.json), -- 50 + self:_escape(alert.info or '')) + + -- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt) + + return interface.alert_store_query(insert_stmt) end -- ##############################################