Merge branch 'dev' of https://github.com/ntop/ntopng into dev

doc/src/api/lua_c/ntop/ntop_misc.lua

@@ -39,7 +39,6 @@ function ntop.addLocalNetwork(string network_cidr)
 function ntop.getNetworkNameById(int network_id)
 
 
-
 --! @brief Get information about the currest host.
 --! @return table (ip, instance_name).
 function ntop.getHostInformation()
@@ -48,12 +47,10 @@ function ntop.getHostInformation()
 --! @return table (cpu_load, cpu_idle, mem_total, mem_free, mem_buffers, mem_cached, mem_shmem. mem_used).
 function ntop.systemHostStat()
 
-
-
 --! @brief Send a message to syslog.
---! @param is_error if true, level will be LOG_ERR, otherwise LOG_INFO.
 --! @param message the message to send.
-function ntop.syslog(bool is_error, string message)
+--! @param syslog_severity an integer representing the standard syslog severity as per RFC 5424. LOG_INFO is used when severity is not specified.
+function syslog(string message, int syslog_severity);
 
 --! @brief Set ntopng logging level.
 --! @param level one of "trace", "debug", "info", "normal", "warning", "error".

doc/src/web_gui/alerts.rst

@@ -10,3 +10,53 @@ information such as Date, Severity, Type and Description.
   :alt: Alerts Page
 
   The Alerts Page
+
+Alert Endopints
+---------------
+
+Generated alerts can also be sent to third-party endpoints. Currently supported endpoints are:
+
+- Email
+- Slack
+- Syslog
+- Nagios
+
+Endpoints can be enabled and configured from the ntopng preferences page.
+
+
+Syslog
+~~~~~~
+
+Alerts are sent to syslog using standard syslog severities as per RFC
+5424 and have a fixed format:
+
+.. code:: bash
+
+   [timestamp][severity][type][entity][entity value][action] ... and a plain text message...
+
+Fields have the following meaning:
+
+- :code:`[timestamp]` is the time at which ntopng detected the alert. This time
+  is not necessarily equal to the time the alert has reached syslog.
+- :code:`[severity]` is the severity of the alert. Severities are also
+  used when dispatching messages to syslog. Severities are "Warning", "Error" of "Info".
+- :code:`[type]` is a string that indicates the type of alert.
+- :code:`[entity]` is a class that categorizes the originator of the
+  alert. It can be an "host", an "interface" and so on.
+- :code:`[entity value]` is an identifier that uniquely identifies the
+  originator along with the :code:`[entity]`. For example, entity
+  value for an "host" is its IP address, for an "interface" is its
+  name, for a "device" is its MAC address, and so on.
+- :code:`[action]` indicates whether this is an engaged alert, an
+  alert that has been released or if it just an alert that has to be stored.
+
+The full list of alert severities, types, and entities can be found at
+https://github.com/ntop/ntopng/blob/dev/scripts/lua/modules/alert_consts.lua
+
+Examples of alerts sent to syslog are
+
+.. code:: bash
+
+   devel ntopng: [<timestamp>][Info][Device Connection][Device][58:40:4E:CE:28:29] The device Apple_CE:28:29 has connected to the network.
+   devel ntopng: [<timestamp>][Error][Threshold Cross][Interface][iface_0][Engaged] Minute traffic crossed by interface eno1 [1.08 MB > 2 Bytes]
+   devel ntopng: [<timestamp>][Warning][Remote to Remote Flow][Flow] Remote client and remote server [Flow: 192.168.1.100:138 192.168.1.255:138] [L4 Protocol: UDP]

httpdocs/js/graph_utils.js

@@ -470,6 +470,8 @@ function attachStackedChartCallback(chart, schema_name, chart_id, zoom_reset_id,
         has_full_data = !schema_name.startsWith("top:");
       }
 
+      var past_serie = null;
+
       if(data.additional_series) {
         for(var key in data.additional_series) {
           if(key == "total") {
@@ -481,6 +483,7 @@ function attachStackedChartCallback(chart, schema_name, chart_id, zoom_reset_id,
           var ratio_over_total = d3.max(serie_data) / d3.max(visual_total);
           var values = arrayToNvSerie(serie_data, data.start, data.step);
           var is_disabled = isLegendDisabled(key, false);
+          past_serie = serie_data; // TODO: more reliable way to determine past serie
 
           /* Hide comparison serie at first load if it's too high */
           if(first_time_loaded && (ratio_over_total > max_over_total_ratio))
@@ -507,12 +510,24 @@ function attachStackedChartCallback(chart, schema_name, chart_id, zoom_reset_id,
           trend: [graph_i18n.trend, "#62ADF6", smooth, num_smoothed_points],
           ema: ["EMA", "#F96BFF", exponentialMovingAverageArray, {periods: num_smoothed_points}],
           sma: ["SMA", "#A900FF", simpleMovingAverageArray, {periods: num_smoothed_points}],
-          rsi: ["RSI", "#00FF5D", relativeStrengthIndexArray, {periods: num_smoothed_points}],
+          rsi: ["RSI cur vs past", "#00FF5D", relativeStrengthIndexArray, {periods: num_smoothed_points}],
         }
 
         function add_smoothed_serie(fn_to_use) {
           var options = smooth_functions[fn_to_use];
-          var smoothed = options[2](total_serie, options[3]);
+          var smoothed;
+
+          if(fn_to_use == "rsi") {
+            if(!past_serie)
+              return;
+
+            var delta_serie = [];
+            for(var i=0; i<total_serie.length; i++) {
+              delta_serie[i] = total_serie[i] - past_serie[i];
+            }
+            smoothed = options[2](delta_serie, options[3]);
+          } else
+            smoothed = options[2](total_serie, options[3]);
 
           var max_val = d3.max(smoothed);
           if(max_val > 0) {

scripts/locales/en.lua

@@ -2308,6 +2308,7 @@ local lang = {
     ["snmp"] = "SNMP",
     ["status_alerts"] = "Status Alerts",
     ["successfully_connected_influxdb"] = "Successfully initialized database \"%{db}\" on InfluxDB(%{version})",
+    ["syslog_notification"] = "Syslog Notification",
     ["timeseries"] = "Timeseries",
     ["timeseries_database"] = "Timeseries Database",
     ["timeseries_resolution_resolution_description"] = "The interval between data points in high resolution timeseries, in particular the local hosts and interface L7 applications.<br>NOTE: High resolution can have a strong impact on memory and disk usage for large networks.",
@@ -2321,8 +2322,8 @@ local lang = {
     ["toggle_alert_nagios_title"] = "Toggle Nagios Notifications",
     ["toggle_alert_probing_description"] = "Toggle alerts generated when probing attempts are detected.",
     ["toggle_alert_probing_title"] = "Probing Alerts",
-    ["toggle_alert_syslog_description"] = "Toggle alerts logging on system syslog.",
-    ["toggle_alert_syslog_title"] = "Alerts On Syslog",
+    ["toggle_alert_syslog_description"] = "Toggle alerts logging to system syslog.",
+    ["toggle_alert_syslog_title"] = "Toggle Syslog Notification",
     ["toggle_alerts_notifications_description"] = "Toggle the reporting of alert to third-party endpoints.",
     ["toggle_alerts_notifications_title"] = "Alerts to Third-Party Endpoints",
     ["toggle_asn_rrds_description"] = "Toggle the creation of bytes and application timeseries for autonomous systems.",

scripts/lua/admin/prefs.lua

@@ -248,7 +248,7 @@ function printAlerts()
   end
 
  local elementToSwitch = { "max_num_alerts_per_entity", "max_num_flow_alerts", "row_toggle_alert_probing",
-			   "row_toggle_malware_probing", "row_toggle_dns_alerts", "row_toggle_alert_syslog",
+			   "row_toggle_malware_probing", "row_toggle_dns_alerts",
 			   "row_toggle_flow_alerts_iface", "row_alerts_retention_header", "row_alerts_security_header",
 			   "row_toggle_ssl_alerts", "row_toggle_dns_alerts", "row_toggle_remote_to_remote_alerts",
 			   "row_toggle_ip_reassignment_alerts", "row_toggle_dropped_flows_alerts", "row_alerts_informative_header",
@@ -272,13 +272,6 @@ function printAlerts()
      showElements = false
   end
 
-  prefsToggleButton({
-    field = "toggle_alert_syslog",
-    pref = "alerts_syslog",
-    default = "0",
-    hidden = not showElements,
-  })
-
   --[[
   prefsToggleButton({
     field = "toggle_flow_alerts_iface",
@@ -540,6 +533,18 @@ function printExternalAlertsReport()
 
     print('<tr id="slack_test" style="' .. ternary(showSlackNotificationPrefs, "", "display:none;").. '"><td><button class="btn btn-default disable-on-dirty" type="button" onclick="sendTestSlack();" style="width:230px; float:left;">'..i18n("prefs.send_test_slack")..'</button></td></tr>')
 
+    if ntop.syslog then
+      print('<tr><th colspan="2" class="info">'..i18n("prefs.syslog_notification")..'</th></tr>')
+
+      prefsToggleButton({
+        field = "toggle_alert_syslog",
+        pref = getAlertNotificationModuleEnableKey("syslog", true),
+        default = "0",
+	disabled = showElements == false,
+      })
+
+    end
+
     if(ntop.isPro() and hasNagiosSupport()) then
       print('<tr><th colspan="2" class="info">'..i18n("prefs.nagios_integration")..'</th></tr>')
 
@@ -574,6 +579,7 @@ function printExternalAlertsReport()
       prefsInputFieldPrefs(subpage_active.entries["nagios_service_name"].title, subpage_active.entries["nagios_service_name"].description, "ntopng.prefs.", "nagios_service_name", prefs.nagios_service_name, nil, showElements)
     end
   end
+
 
   print('<tr><th colspan=2 style="text-align:right;"><button type="submit" class="btn btn-primary" style="width:115px" disabled="disabled">'..i18n("save")..'</button></th></tr>')
   print('</table>')

scripts/lua/if_stats.lua

@@ -24,6 +24,8 @@ require "alert_utils"
 require "db_utils"
 local ts_utils = require "ts_utils"
 
+processAlertNotifications(os.time(), 1, false)
+
 local have_nedge = ntop.isnEdge()
 
 if ntop.isPro() then

scripts/lua/modules/alert_consts.lua

@@ -9,12 +9,12 @@ local format_utils  = require "format_utils"
 
 -- Alerts (see ntop_typedefs.h)
 -- each table entry is an array as:
--- {"alert html string", "alert C enum value", "plain string"}
+-- {"alert html string", "alert C enum value", "plain string", "syslog severity"}
 alert_consts.alert_severity_keys = {
-   { "<span class='label label-info'>" .. i18n("alerts_dashboard.none") .. "</span>",      -1, "none"    },
-   { "<span class='label label-info'>" .. i18n("alerts_dashboard.info") .. "</span>",       0, "info"    },
-   { "<span class='label label-warning'>" .. i18n("alerts_dashboard.warning") .. "</span>", 1, "warning" },
-   { "<span class='label label-danger'>" .. i18n("alerts_dashboard.error") .. "</span>",    2, "error"   }
+   { "<span class='label label-info'>" .. i18n("alerts_dashboard.none") .. "</span>",      -1, "none",    6, --[[ LOG_INFO --]]    },
+   { "<span class='label label-info'>" .. i18n("alerts_dashboard.info") .. "</span>",       0, "info",    6, --[[ LOG_INFO --]]    },
+   { "<span class='label label-warning'>" .. i18n("alerts_dashboard.warning") .. "</span>", 1, "warning", 4, --[[ LOG_WARNING --]] },
+   { "<span class='label label-danger'>" .. i18n("alerts_dashboard.error") .. "</span>",    2, "error",   3, --[[ LOG_ERR --]]     }
 }
 
 alert_consts.alert_type_keys = {

scripts/lua/modules/alert_endpoints/syslog.lua

@@ -0,0 +1,52 @@
+--
+-- (C) 2018 - ntop.org
+--
+
+require "lua_utils"
+local json = require "dkjson"
+
+local syslog = {}
+
+syslog.DEFAULT_SEVERITY = "info"
+syslog.EXPORT_FREQUENCY = 1 -- 1 second, i.e., as soon as possible
+
+function syslog.dequeueAlerts(queue)
+   local notifications = ntop.lrangeCache(queue, 0, -1)
+
+   if not notifications then
+      return {success = true}
+   end
+
+   -- Separate by severity and channel
+   local alerts_by_types = {}
+
+   for _, json_message in ipairs(notifications) do
+      local notif = alertNotificationToObject(json_message)
+
+      alerts_by_types[notif.entity_type] = alerts_by_types[notif.entity_type] or {}
+      alerts_by_types[notif.entity_type][notif.severity] = alerts_by_types[notif.entity_type][notif.severity] or {}
+      table.insert(alerts_by_types[notif.entity_type][notif.severity], notif)
+   end
+
+   for entity_type, by_severity in pairs(alerts_by_types) do
+      for severity, notifications in pairs(by_severity) do
+	 -- Most recent notifications first
+	 for _, notif in pairsByValues(notifications, notification_timestamp_rev) do
+	    local msg = formatAlertNotification(notif, {nohtml = true,
+							show_severity = true,
+							show_entity = true})
+
+
+	    local syslog_severity = alertLevelToSyslogLevel(notif.severity)
+	    ntop.syslog(msg, syslog_severity)
+	 end
+      end
+   end
+
+   -- Remove all the messages from queue on success
+   ntop.delCache(queue)
+
+   return {success = true}
+end
+
+return syslog

scripts/lua/modules/alert_utils.lua

@@ -2659,9 +2659,9 @@ function alertNotificationActionToLabel(action)
    local label = ""
 
    if action == "engage" then
-      label = "Alert Engaged: "
+      label = "[Engaged]"
    elseif action == "release" then
-      label = "Alert Released: "
+      label = "[Released]"
    end
 
    return label
@@ -2684,11 +2684,24 @@ local ALERT_NOTIFICATION_MODULES = {
    "custom", "nagios", "slack"
 }
 
+if ntop.syslog then
+   table.insert(ALERT_NOTIFICATION_MODULES, 1, "syslog")
+end
+
 if ntop.sendMail then -- only if email support is available
    table.insert(ALERT_NOTIFICATION_MODULES, 1, "email")
 end
 
 function getAlertNotificationModuleEnableKey(module_name, short)
+   if module_name == "syslog" and ntop.getPref("ntopng.prefs.alerts_syslog") ~= "" then
+      -- For backward compatibility
+      if short then
+	 return "alerts_syslog"
+      else
+	 return "ntopng.prefs.alerts_syslog"
+      end
+   end
+
    local short_k = "alerts." .. module_name .. "_notifications_enabled"
 
    if short then
@@ -2784,15 +2797,32 @@ function formatAlertNotification(notif, options)
    }
    options = table.merge(defaults, options)
 
-   local msg_prefix = alertNotificationActionToLabel(notif.action)
-   local msg = "[" .. formatEpoch(notif.tstamp or 0) .. "]" ..
-      ternary(defaults.show_severity == true, "", "[" .. alertSeverityLabel(alertSeverity(notif.severity), options.nohtml) .. "]") ..
-      "[" .. alertTypeLabel(alertType(notif.type), options.nohtml) .."]: "
+   local msg = "[" .. formatEpoch(notif.tstamp or 0) .. "]"
+   msg = msg .. ternary(options.show_severity == false, "", "[" .. alertSeverityLabel(alertSeverity(notif.severity), options.nohtml) .. "]") ..
+      "[" .. alertTypeLabel(alertType(notif.type), options.nohtml) .."]"
+
+   -- entity can be hidden for example when one is OK with just the message
+   if options.show_entity then
+      msg = msg.."["..alertEntityLabel(alertEntity(notif.entity_type)).."]"
+
+      if notif.entity_type ~= "flow" then
+	 local ev = notif.entity_value
+	 if notif.entity_type == "host" then
+	    -- suppresses @0 when the vlan is zero
+	    ev = hostinfo2hostkey(hostkey2hostinfo(notif.entity_value))
+	 end
+
+	 msg = msg.."["..ev.."]"
+      end
+   end
+
+   -- add the label, that is, engaged or released
+   msg = msg .. alertNotificationActionToLabel(notif.action).. " "
 
    if options.nohtml then
-      msg = msg .. noHtml(msg_prefix .. notif.message)
+      msg = msg .. noHtml(notif.message)
    else
-      msg = msg .. msg_prefix .. notif.message
+      msg = msg .. notif.message
    end
 
    return msg
@@ -2827,6 +2857,7 @@ function processAlertNotifications(now, periodic_frequency, force_export)
    -- Process export notifications
    for _, m in ipairs(modules) do
       if force_export or ((now % m.export_frequency) < periodic_frequency) then
+
          local rv = m.module.dequeueAlerts(m.export_queue)
 
          if not rv.success then

scripts/lua/modules/lua_utils.lua

@@ -553,11 +553,26 @@ end
 
 function noHtml(s)
    if s == nil then return nil end
+
+   local gsub, char = string.gsub, string.char
+   local entityMap  = {lt = "<", gt = ">" , amp = "&", quot ='"', apos = "'"}
+   local entitySwap = function(orig, n, s)
+      return (n == '' and entityMap[s])
+	 or (n == "#" and tonumber(s)) and string.char(s)
+	 or (n == "#x" and tonumber(s,16)) and string.char(tonumber(s,16))
+	 or orig
+   end
+
+   local function unescape(str)
+      return (gsub( str, '(&(#?x?)([%d%a]+);)', entitySwap ))
+   end
+
    local cleaned = s:gsub("<[aA].->(.-)</[aA]>","%1")
       :gsub("%s*<[iI].->(.-)</[iI]>","%1")
       :gsub("<.->(.-)</.->","%1") -- note: this does not handle nested tags
       :gsub("^%s*(.-)%s*$", "%1")
-   return cleaned
+
+   return unescape(cleaned)
 end
 
 function alertSeverityLabel(v, nohtml)
@@ -625,6 +640,15 @@ function alertLevel(v)
    return(_handleArray(leveltable, v))
 end
 
+function alertLevelToSyslogLevel(v)
+   local leveltable = {}
+
+   for i, t in ipairs(alert_consts.alert_severity_keys) do
+      leveltable[#leveltable + 1] = {t[4], t[3]}
+   end
+   return(_handleArray(leveltable, v))
+end
+
 function alertTypeRaw(alert_idx)
    if(alert_idx == nil) then return nil end