Merge pull request #14 from ntop/dev

sync

docker/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Pass --build-arg TZ=<YOUR_TZ> when running docker build to override this.
+ARG TZ=America/Los_Angeles
+
+RUN apt-get update && apt-get -y install wget lsb-release gnupg tzdata
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+RUN wget -O apt-ntop-stable.deb http://apt-stable.ntop.org/18.04/all/apt-ntop-stable.deb && \
+	dpkg -i apt-ntop-stable.deb && rm -f apt-ntop-stable.deb
+RUN apt-get update && apt-get -y install ntopng
+RUN echo '#!/usr/bin/env bash\n/etc/init.d/redis-server start && ntopng "$@"' > /tmp/run.sh
+RUN chmod +x /tmp/run.sh
+
+ENTRYPOINT ["/tmp/run.sh"]

httpdocs/js/graph_utils.js

@@ -15,7 +15,10 @@ function getSerieLabel(schema, serie) {
   var new_label = data_2_label[data_label];
 
   if((schema == "top:local_senders") || (schema == "top:local_receivers")) {
-    return serie.tags.host
+    if(serie.ext_label)
+      return serie.ext_label;
+    else
+      return serie.tags.host
   } else if(schema.startsWith("top:")) { // topk graphs
     if(serie.tags.protocol)
       return serie.tags.protocol;

scripts/locales/en.lua

@@ -28,6 +28,7 @@ local lang = {
   ["change_number_of_rows"] = "Change the number of rows per page",
   ["chart"] = "Chart",
   ["client"] = "Client",
+  ["domain"] = "Domain",
   ["clone"] = "Clone",
   ["close"] = "Close",
   ["countries"] = "Countries",
@@ -66,6 +67,7 @@ local lang = {
   ["from"] = "from",
   ["gateway"] = "Gateway",
   ["host"] = "Host %{host}",
+  ["host_address"] = "Host Address",
   ["hosts"] = "Hosts",
   ["hour"] = "Hour",
   ["http"] = "HTTP",
@@ -158,9 +160,9 @@ local lang = {
   ["today"] = "Today",
   ["total"] = "Total",
   ["total_incoming_anomalous_flows"] = "Total Misbehaving Flows As Server",
-  ["total_incoming_unreachable_flows"] = "Total Unreachable Flows As Server",
+  ["total_incoming_unreachable_flows"] = "Total Port Unreachable Flows As Server",
   ["total_outgoing_anomalous_flows"] = "Total Misbehaving Flows As Client",
-  ["total_outgoing_unreachable_flows"] = "Total Unreachable Flows As Client",
+  ["total_outgoing_unreachable_flows"] = "Total Port Unreachable Flows As Client",
   ["traffic"] = "Traffic",
   ["traffic_policy"] = "Traffic Policy",
   ["undo"] = "Undo",
@@ -214,6 +216,7 @@ local lang = {
   },
   ["alert_messages"] = {
     ["a_new_device_has_connected"] = "New device <a href=\"%{url}\">%{device}</a> has connected to the network.",
+    ["broadcast_domain_too_large"] = "ARP traffic from <a href=\"%{src_mac_url}\">%{src_mac}</a>/<a href=\"%{spa_url}\">%{spa}</a> to <a href=\"%{dst_mac_url}\">%{dst_mac}</a>/<a href=\"%{tpa_url}\">%{tpa}</a> detected. It is unlikely to see ARP traffic between those IPs as they are seemingly belonging to different broadcast domains. Check for hosts and networks configurations.",
     ["device_has_connected"] = "The device <a href=\"%{url}\">%{device}</a> has connected to the network.",
     ["device_has_disconnected"] = "The device <a href=\"%{url}\">%{device}</a> has disconnected from the network.",
     ["host_entity"] = "host %{entity_value}",
@@ -224,7 +227,6 @@ local lang = {
     ["influxdb_write_error"] = "There was an error while sending timeseries data to \"%{influxdb}\": %{err}",
     ["interface_entity"] = "interface %{entity_value}",
     ["mac_ip_association_change"] = "IP %{ip} changed association from <a href=\"%{old_mac_url}\">%{old_mac}</a> to <a href=\"%{new_mac_url}\">%{new_mac}</a>",
-    ["broadcast_domain_too_large"] = "ARP traffic from <a href=\"%{src_mac_url}\">%{src_mac}</a>/<a href=\"%{spa_url}\">%{spa}</a> to <a href=\"%{dst_mac_url}\">%{dst_mac}</a>/<a href=\"%{tpa_url}\">%{tpa}</a> detected. It is unlikely to see ARP traffic between those IPs as they are seemingly belonging to different broadcast domains. Check for hosts and networks configurations.",
     ["network_entity"] = "network %{entity_value}",
     ["nfq_flushed"] = "Interface <a href=\"%{url}\">%{name}</a> packets queue flushed. Queue %{pct}%% full with %{tot} packets and %{dropped} drops.",
     ["ntopng_anomalous_termination"] = "Started after anomalous termination (<a href=\"%{url}\">bug report</a>)",
@@ -246,6 +248,7 @@ local lang = {
     ["all_hosts"] = "All Hosts",
     ["blacklisted_flow"] = "Blacklisted Flow",
     ["blocked_flow"] = "Blocked Flow",
+    ["broadcast_domain_too_large"] = "Broadcast domain",
     ["custom_period"] = "Custom Period",
     ["device_connection"] = "Device Connection",
     ["device_disconnection"] = "Device Disconnection",
@@ -260,7 +263,6 @@ local lang = {
     ["host_pool_connection"] = "Host Pool Connection",
     ["host_pool_disconnection"] = "Host Pool Disconnection",
     ["icmp_anomaly"] = "ICMP Anomaly",
-    ["broadcast_domain_too_large"] = "Broadcast domain",
     ["ids_alert"] = "IDS Alert",
     ["inactivity"] = "Inactivity",
     ["influxdb_export_failure"] = "InfluxDB Export Failure",
@@ -525,7 +527,7 @@ local lang = {
     ["flow_snmp_localization"] = "Flow SNMP Localization",
     ["flows_dropped_by_bridge"] = "Flows Blocked due to Traffic Policies",
     ["flows_non_packet_iface"] = "Recently Active Flows / Total / Total Anomalous",
-    ["flows_packet_iface"] = "Flows: Active / Total / Low Goodput / Anomalous / Unreach",
+    ["flows_packet_iface"] = "Flows: Active / Total / Low Goodput / Anomalous / Port Unreach",
     ["flows_packet_pcap_dump_iface"] = "Flows / Total Active / Total Anomalous",
     ["further_host_names_information"] = "Additional Host Names",
     ["goodput"] = "Goodput",
@@ -654,6 +656,7 @@ local lang = {
     ["average_cpu_load"] = "Average CPU Load",
     ["bittorrent_hash"] = "BitTorrent hash",
     ["blacklisted_flow"] = "Client, server or domain is blacklisted",
+    ["blacklisted_flow_detailed"] = "Blacklisted %{who}",
     ["c_ip_addresses"] = "C IP Addresses",
     ["call_canceled"] = "Call Canceled",
     ["call_completed"] = "Call Completed",
@@ -755,6 +758,8 @@ local lang = {
     ["ssh_signature"] = "SSH Signature",
     ["ssl_certificate"] = "SSL Certificate",
     ["ssl_certificate_mismatch"] = "SSL Certificate Mismatch",
+    ["ssl_client_certificate"] = "Client Certificate",
+    ["ssl_server_certificate"] = "Server Certificate",
     ["suspicious_client_device_protocol"] = "Client application \"%{proto}\" is not allowed by the configured <a href=\"%{url}\">%{devtype} application policy</a>",
     ["suspicious_server_device_protocol"] = "Server application \"%{proto}\" is not allowed by the configured <a href=\"%{url}\">%{devtype} application policy</a>",
     ["suspicious_tcp_probing"] = "Suspicious TCP Probing",
@@ -1296,8 +1301,6 @@ local lang = {
     ["active_devices"] = "Active Devices",
     ["active_flows"] = "Active Flows",
     ["active_host_contacts"] = "Active Host Contacts",
-    ["dns_qry_sent_rsp_rcvd"] = "DNS Sent Queries / Rcvd Repls",
-    ["dns_qry_rcvd_rsp_sent"] = "DNS Rcvd Queries / Sent Repls",
     ["active_hosts"] = "Active Hosts",
     ["active_http_servers"] = "Active HTTP Servers",
     ["actual_memory"] = "Actual Memory",
@@ -1309,6 +1312,8 @@ local lang = {
     ["binary_operators_note"] = "For more information about the EMA, SMA and RSI indicators check out the <a href=\"%{url}\">online documentation</a>.",
     ["blocked_flows"] = "Blocked Flows",
     ["custom"] = "Custom",
+    ["dns_qry_rcvd_rsp_sent"] = "DNS Rcvd Queries / Sent Repls",
+    ["dns_qry_sent_rsp_rcvd"] = "DNS Sent Queries / Rcvd Repls",
     ["download_chart_data"] = "Download the current chart data",
     ["export_flows_as_txt"] = "Export flows as TXT file",
     ["flows_status_overview"] = "%{protocol} Flows Status Overview",
@@ -1358,7 +1363,7 @@ local lang = {
     ["top_senders"] = "Top Senders",
     ["top_servers"] = "Top %{protocol} Servers",
     ["total_anomalous_flows"] = "Total Misbehaving Flows",
-    ["total_unreachable_flows"] = "Total Unreachable Flows",
+    ["total_unreachable_flows"] = "Total Port Unreachable Flows",
     ["traffic_txrx"] = "Traffic TX/RX",
     ["trend"] = "Trend",
     ["zmq_flow_coll_drops"] = "ZMQ Flow Collection Drops",

scripts/locales/it.lua

@@ -28,6 +28,7 @@ local lang = {
   ["change_number_of_rows"] = "Cambia il numero di righe per pagina",
   ["chart"] = "Grafico",
   ["client"] = "Client",
+  ["domain"] = "Dominio",
   ["clone"] = "Clona",
   ["close"] = "Chiudi",
   ["countries"] = "Paesi",
@@ -620,6 +621,7 @@ local lang = {
     ["average_cpu_load"] = "Carico medio CPU",
     ["bittorrent_hash"] = "Hash BitTorrent",
     ["blacklisted_flow"] = "Client o server blacklistato (o entrambi)",
+    ["blacklisted_flow_detailed"] = "%{who} in blacklist",
     ["c_ip_addresses"] = "C IP Addresses",
     ["call_canceled"] = "Chiamata Cancellata",
     ["call_completed"] = "Chiamata Completata",

scripts/lua/admin/edit_category_lists.lua

@@ -6,6 +6,7 @@ local dirs = ntop.getDirs()
 package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path
 
 require "lua_utils"
+require "graph_utils"
 local template = require "template_utils"
 local categories_utils = require "categories_utils"
 local lists_utils = require "lists_utils"
@@ -20,6 +21,13 @@ page_utils.print_header()
 
 dofile(dirs.installdir .. "/scripts/lua/inc/menu.lua")
 
+local base_url = ntop.getHttpPrefix() .. "/lua/admin/edit_category_lists.lua"
+local page_params = {
+  category = _GET["category"],
+}
+
+local lists = lists_utils.getCategoryLists()
+
 if _POST["action"] == "edit" then
   local enabled = not isEmptyString(_POST["list_enabled"])
   local list_name = _POST["list_name"]
@@ -91,7 +99,7 @@ print[[
               <div class="row form-group">
                 <div class="col col-md-6">
                   <label class="form-label">]] print(i18n("category")) print[[</label>
-                  <select name="category" class="form-control" readonly>]]
+                  <select name="category" class="form-control" readonly disabled="disabled">]]
 
                   for cat_name, cat_id in pairsByKeys(interface.getnDPICategories()) do
                     print(string.format([[<option value="cat_%s">%s</option>]], cat_id, cat_name))
@@ -126,11 +134,27 @@ print[[
 <div id="table-edit-lists-form"></div>
 
 <script>
+  var url_update = "]] print(getPageUrl(ntop.getHttpPrefix()..[[/lua/admin/get_category_lists.lua]], page_params)) print[[";
+
   $("#table-edit-lists-form").datatable({
-    url: "]] print (ntop.getHttpPrefix()) print [[/lua/admin/get_category_lists.lua",
+    url: url_update,
     class: "table table-striped table-bordered",
     title:"",
-    buttons: [],
+    buttons: []]
+
+local categories = {}
+
+for _, list in pairs(lists) do
+  local catid = tostring(list.category)
+  categories[catid] = categories[catid] or 0
+  categories[catid] = categories[catid] + 1
+end
+
+  printCategoryDropdownButton(false, page_params.category, base_url, page_params, function (catid, catname)
+    return(categories[catid] or 0)
+  end)
+
+print[[],
     columns: [
       {
         title: "]] print(i18n("name")) print[[",

scripts/lua/admin/get_category_lists.lua

@@ -11,6 +11,7 @@ local json = require("dkjson")
 local lists_utils = require("lists_utils")
 local format_utils = require("format_utils")
 
+local category_filter = _GET["category"]
 local lists = lists_utils.getCategoryLists()
 local now = os.time()
 
@@ -89,9 +90,15 @@ local totalRows = 0
 local sort_to_key = {}
 
 for list_name, list in pairs(lists) do
+  local catname = interface.getnDPICategoryName(list.category)
+
+  if((not isEmptyString(category_filter)) and (category_filter ~= catname)) then
+    goto continue
+  end
+
   totalRows = totalRows + 1
 
-  list.category_name = interface.getnDPICategoryName(list.category)
+  list.category_name = catname
   list.name = list_name
   list.status_label = getListStatusLabel(list)
 
@@ -107,6 +114,8 @@ for list_name, list in pairs(lists) do
     -- default
     sort_to_key[list_name] = list_name
   end
+
+  ::continue::
 end
 
 -- ################################################

scripts/lua/host_details.lua

@@ -193,6 +193,9 @@ print [[
 ]]
 if((debug_hosts) and (host["ip"] ~= nil)) then traceError(TRACE_DEBUG,TRACE_CONSOLE, i18n("host_details.trace_debug_host_ip",{hostip=host["ip"],vlan=host["vlan"]}).."\n") end
 url = ntop.getHttpPrefix().."/lua/host_details.lua?ifid="..ifId.."&"..hostinfo2url(host_info)
+if _GET["tskey"] ~= nil then
+   url = url .. "&tskey=" .. _GET["tskey"]
+end
 
 print("<li><a href=\"#\">"..i18n("host_details.host")..": "..host_info["host"])
 if host["broadcast_domain_host"] then
@@ -203,6 +206,15 @@ if(host.dhcpHost) then
    print(" <i class='fa fa-flash fa-lg' aria-hidden='true' title='DHCP Host'></i>")
 end
 
+--[[
+local tskey = _GET["tskey"] or host["tskey"]
+
+if tskey ~= hostkey_compact then
+   -- Print the tskey
+   print(string.format(" [LBD: %s]", visualTsKey(tskey)))
+end
+]]
+
 print("</A> </li>")
 
 if not only_historical then
@@ -1389,6 +1401,7 @@ local page_params = {
    traffic_type = _GET["traffic_type"],
    version = _GET["version"],
    host = hostinfo2hostkey(host_info),
+   tskey = _GET["tskey"],
 }
 
 print(getPageUrl(ntop.getHttpPrefix().."/lua/get_flows_data.lua", page_params))

scripts/lua/modules/graph_utils.lua

@@ -260,7 +260,7 @@ end
 
 local graph_menu_entries = {}
 
-function populateGraphMenuEntry(label, base_url, params, tab_id, needs_separator, separator_label, pending)
+function populateGraphMenuEntry(label, base_url, params, tab_id, needs_separator, separator_label, pending, disabled)
    local url = getPageUrl(base_url, params)
 
    local entry_params = table.clone(params)
@@ -277,6 +277,7 @@ function populateGraphMenuEntry(label, base_url, params, tab_id, needs_separator
       needs_separator = needs_separator,
       separator_label = separator_label,
       pending = pending, -- true for batched operations
+      disabled = disabled,
    }
 
    graph_menu_entries[#graph_menu_entries + 1] = entry

scripts/lua/modules/http_lint.lua

@@ -1136,6 +1136,7 @@ local known_parameters = {
    ["toggle_remote_to_remote_alerts"]              = validateBool,
    ["toggle_dropped_flows_alerts"]                 = validateBool,
    ["toggle_malware_probing"]                      = validateBool,
+   ["toggle_ids_alerts"]                           = validateBool,
    ["toggle_device_protocols_alerts"]              = validateBool,
    ["toggle_elephant_flows_alerts"]                = validateBool,
    ["toggle_ip_reassignment_alerts"]               = validateBool,

scripts/lua/modules/lists_utils.lua

@@ -13,11 +13,22 @@ local json = require("dkjson")
 
 local CUSTOM_CATEGORY_MINING = 99
 local CUSTOM_CATEGORY_MALWARE = 100
+local CUSTOM_CATEGORY_ADVERTISEMENT = 101
 
 local DEFAULT_UPDATE_INTERVAL = 86400
 local MAX_LIST_ERRORS = 3
 
+local is_nedge = ntop.isnEdge()
+
 -- supported formats: ip, domain, hosts
+--
+-- Examples:
+--    [ip] 1.2.3.4
+--    [ip] 1.2.3.0/24
+--    [domain] amalwaredomain.com
+--    [hosts] 127.0.0.1   amalwaredomain.com
+--    [hosts] 127.0.0.1   1.2.3.4
+--
 local BUILTIN_LISTS = {
   ["Emerging Threats"] = {
     url = "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
@@ -73,7 +84,25 @@ local BUILTIN_LISTS = {
     format = "hosts",
     enabled = true,
     update_interval = DEFAULT_UPDATE_INTERVAL,
-  }
+  }, ["Disconnect.me Simple Ad List"] = {
+    url = "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
+    category = CUSTOM_CATEGORY_ADVERTISEMENT,
+    format = "domain",
+    enabled = is_nedge,
+    update_interval = DEFAULT_UPDATE_INTERVAL,
+  }, ["hpHosts Ad and Tracking"] = {
+    url = "https://hosts-file.net/ad_servers.txt",
+    category = CUSTOM_CATEGORY_ADVERTISEMENT,
+    format = "hosts",
+    enabled = is_nedge,
+    update_interval = DEFAULT_UPDATE_INTERVAL,
+  }, ["AdAway default blocklist"] = {
+    url = "https://adaway.org/hosts.txt",
+    category = CUSTOM_CATEGORY_ADVERTISEMENT,
+    format = "hosts",
+    enabled = is_nedge,
+    update_interval = DEFAULT_UPDATE_INTERVAL,
+  },
 }
 
 -- ##############################################
@@ -376,7 +405,7 @@ local function loadFromListFile(list_name, list, user_custom_categories)
         if words and (#words == 2) then
           host = words[2]
 
-          if((host == "localhost") or (host == "127.0.0.1")) then
+          if((host == "localhost") or (host == "127.0.0.1") or (host == "::1")) then
             host = nil
           end
         else