Properly handle L4 protocols charts

scripts/locales/en.lua

@@ -149,6 +149,7 @@ local lang = {
   ["system"] = "System",
   ["talkers"] = "Talkers",
   ["tcp_flags"] = "TCP Flags",
+  ["tcp_stats"] = "TCP Statistics",
   ["throughput"] = "Throughput",
   ["time"] = "Time",
   ["timeframe"] = "Timeframe",

scripts/lua/host_details.lua

@@ -1957,13 +1957,15 @@ local tags = {
    host = host_key,
    protocol = _GET["protocol"],
    category = _GET["category"],
+   l4proto = _GET["l4proto"],
 }
 
 local url = ntop.getHttpPrefix()..'/lua/host_details.lua?ifid='..ifId..'&'..host_url..'&page=historical'
 
 drawGraphs(ifId, schema, tags, _GET["zoom"], url, selected_epoch, {
    top_protocols = "top:host:ndpi",
    top_categories = "top:host:ndpi_categories",
+   l4_protocols = "host:l4protos",
    show_historical = true,
    timeseries = {
       {schema="host:traffic",                label=i18n("traffic")},

scripts/lua/if_stats.lua

@@ -954,12 +954,12 @@ elseif(page == "historical") then
          {schema="iface:nfq_pct",               label=i18n("graphs.num_nfq_pct"), nedge_only=1},
 
          {schema="iface:zmq_recv_flows",        label=i18n("graphs.zmq_received_flows"), nedge_exclude=1},
-         {separator=1, nedge_exclude=1},
+         {separator=1, nedge_exclude=1, label=i18n("tcp_stats")},
          {schema="iface:tcp_lost",              label=i18n("graphs.tcp_packets_lost"), nedge_exclude=1},
          {schema="iface:tcp_out_of_order",      label=i18n("graphs.tcp_packets_ooo"), nedge_exclude=1},
          --{schema="tcp_retr_ooo_lost",   label=i18n("graphs.tcp_retr_ooo_lost"), nedge_exclude=1},
          {schema="iface:tcp_retransmissions",   label=i18n("graphs.tcp_packets_retr"), nedge_exclude=1},
-         {separator=1},
+         {separator=1, label=i18n("tcp_flags")},
          {schema="iface:tcp_syn",               label=i18n("graphs.tcp_syn_packets"), nedge_exclude=1},
          {schema="iface:tcp_synack",            label=i18n("graphs.tcp_synack_packets"), nedge_exclude=1},
          {schema="iface:tcp_finack",            label=i18n("graphs.tcp_finack_packets"), nedge_exclude=1},

scripts/lua/modules/graph_utils.lua

@@ -284,9 +284,13 @@ function graphMenuDivider()
    graph_menu_entries[#graph_menu_entries + 1] = {html='<li class="divider"></li>'}
 end
 
+function graphMenuHeader(label)
+   graph_menu_entries[#graph_menu_entries + 1] = {html='<li class="dropdown-header">'.. label ..'</li>'}
+end
+
 function graphMenuGetActive(schema, params)
    -- These tags are used to determine the active timeseries entry
-   local match_tags = {ts_schema=1, ts_query=1, protocol=1, category=1, snmp_port_idx=1}
+   local match_tags = {ts_schema=1, ts_query=1, protocol=1, category=1, snmp_port_idx=1, l4proto=1}
 
    for _, entry in pairs(graph_menu_entries) do
       if entry.schema == schema and entry.params then
@@ -334,6 +338,7 @@ end
 function printSeries(options, tags, start_time, base_url, params)
    local series = options.timeseries
    local needs_separator = false
+   local separator_label = nil
 
    for _, serie in ipairs(series) do
       if (have_nedge and serie.nedge_exclude) or (not have_nedge and serie.nedge_only) then
@@ -342,6 +347,7 @@ function printSeries(options, tags, start_time, base_url, params)
 
       if serie.separator then
          needs_separator = true
+         separator_label = serie.label
      else
          local k = serie.schema
          local v = serie.label
@@ -375,6 +381,11 @@ function printSeries(options, tags, start_time, base_url, params)
             if needs_separator then
                -- Only add the separator if there are actually some entries in the group
                graphMenuDivider()
+
+               if separator_label then
+                  graphMenuHeader(separator_label)
+               end
+
                needs_separator = false
             end
 
@@ -395,6 +406,7 @@ function printSeries(options, tags, start_time, base_url, params)
 
       if not table.empty(series) then
          graphMenuDivider()
+         graphMenuHeader(i18n("l7_protocols"))
 
          local by_protocol = {}
 
@@ -409,6 +421,46 @@ function printSeries(options, tags, start_time, base_url, params)
       end
    end
 
+   -- L4 protocols
+   if options.l4_protocols then
+      local schema = options.l4_protocols
+      local l4_tags = table.clone(tags)
+      l4_tags.l4proto = nil
+
+      local series = ts_utils.listSeries(schema, l4_tags, start_time)
+
+      if not table.empty(series) then
+         graphMenuDivider()
+         graphMenuHeader(i18n("l4_protocols"))
+
+         local by_protocol = {}
+
+         for _, serie in pairs(series) do
+            local sortkey = serie.l4proto
+
+            if sortkey == "other_ip" then
+               -- place at the end
+               sortkey = "z" .. sortkey
+            end
+
+            by_protocol[sortkey] = serie.l4proto
+         end
+
+         for _, protocol in pairsByKeys(by_protocol, asc) do
+            local proto_id = protocol
+            local label
+
+            if proto_id == "other_ip" then
+               label = i18n("other")
+            else
+               label = string.upper(protocol)
+            end
+
+            populateGraphMenuEntry(label, base_url, table.merge(params, {ts_schema=schema, l4proto=proto_id}))
+         end
+      end
+   end
+
    -- nDPI application categories
    if options.top_categories then
       local schema = split(options.top_categories, "top:")[2]
@@ -418,6 +470,7 @@ function printSeries(options, tags, start_time, base_url, params)
 
       if not table.empty(series) then
          graphMenuDivider()
+         graphMenuHeader(i18n("categories"))
 
          local by_category = {}
 

scripts/lua/modules/http_lint.lua

@@ -585,10 +585,8 @@ local function validateApplication(app)
 end
 
 local function validateProtocolId(p)
-   local l4 = {"icmp", "tcp", "udp"}
-
    return validateChoice(ndpi_protos, p) or
-      validateChoice(l4, p) or
+      validateChoiceByKeys(L4_PROTO_KEYS, p) or
       validateChoiceByKeys(ndpi_protos, p)
 end
 

scripts/lua/modules/lua_utils.lua

@@ -144,6 +144,8 @@ l4_keys = {
   { "Other IP", "other_ip", -1 }
 }
 
+L4_PROTO_KEYS = {tcp=6, udp=17, icmp=1, other_ip=-1}
+
 function __FILE__() return debug.getinfo(2,'S').source end
 function __LINE__() return debug.getinfo(2, 'l').currentline end
 

scripts/lua/modules/timeseries/drivers/rrd.lua

@@ -425,7 +425,7 @@ end
 -- *Limitation*
 -- tags_filter is expected to contain all the tags of the schema except the last
 -- one. For such tag, a list of available values will be returned.
-local function _listSeries(schema, tags_filter, wildcard_tags, start_time, with_l4)
+local function _listSeries(schema, tags_filter, wildcard_tags, start_time)
   if #wildcard_tags > 1 then
     traceError(TRACE_ERROR, TRACE_CONSOLE, "RRD driver does not support listSeries on multiple tags")
     return nil
@@ -449,9 +449,6 @@ local function _listSeries(schema, tags_filter, wildcard_tags, start_time, with_
     return nil
   end
 
-  -- TODO remove after migration
-  local l4_keys = {tcp=1, udp=1, icmp=1}
-
   local base, rrd = schema_get_path(schema, table.merge(tags_filter, {[wildcard_tag] = ""}))
   local files = ntop.readdir(base)
   local res = {}
@@ -466,9 +463,18 @@ local function _listSeries(schema, tags_filter, wildcard_tags, start_time, with_
       if last_update ~= nil and last_update >= start_time then
         -- TODO remove after migration
         local value = v[1]
+        local toadd = false
 
-        if ((wildcard_tag ~= "protocol") or (with_l4 and l4_keys[value] ~= nil) or ((l4_keys[value] == nil) and (interface.getnDPIProtoId(value) ~= -1))) and
+        if wildcard_tag == "l4proto" then
+          if L4_PROTO_KEYS[value] ~= nil then
+            toadd = true
+          end
+        elseif ((wildcard_tag ~= "protocol") or ((L4_PROTO_KEYS[value] == nil) and (interface.getnDPIProtoId(value) ~= -1))) and
             ((wildcard_tag ~= "category") or (interface.getnDPICategoryId(value) ~= -1)) then
+          toadd = true
+        end
+
+        if toadd then
           res[#res + 1] = table.merge(tags_filter, {[wildcard_tag] = value})
         end
       end
@@ -489,7 +495,7 @@ end
 -- ##############################################
 
 function driver:listSeries(schema, tags_filter, wildcard_tags, start_time)
-  return _listSeries(schema, tags_filter, wildcard_tags, start_time, true --[[ with l4 protos ]])
+  return _listSeries(schema, tags_filter, wildcard_tags, start_time)
 end
 
 -- ##############################################
@@ -507,7 +513,7 @@ function driver:topk(schema, tags, tstart, tend, options, top_tags)
     return nil
   end
 
-  local series = _listSeries(schema, tags, top_tags, tstart, false --[[ no l4 protos ]])
+  local series = _listSeries(schema, tags, top_tags, tstart)
   if not series then
     return nil
   end