much better performance with big files; add parse stats chart

python.d/isc_dhcpd.chart.py

@@ -3,8 +3,8 @@
 # Author: l2isbad
 
 from base import SimpleService
-from re import compile, search
-from time import mktime, strptime, gmtime
+from re import compile
+from time import mktime, strptime, gmtime, time
 try:
     from ipaddress import IPv4Address as ipaddress
     from ipaddress import ip_network
@@ -23,14 +23,14 @@ def __init__(self, configuration=None, name=None):
         self.pools = self.configuration.get('pools')
 
         # Will work only with 'default' db-time-format (weekday year/month/day hour:minute:second)
-        # TODO: update the regex to parse correctly 'local' db-time-format
+        # TODO: update algorithm to parse correctly 'local' db-time-format
         # (epoch <seconds-since-epoch>; # <day-name> <month-name> <day-number> <hours>:<minutes>:<seconds> <year>)
         # Also only ipv4 supported
-        self.regex = compile(r'(\d+(?:\.\d+){3}).*?((?<=ends )[0-9].*?(?=;))')
+        self.regex = compile(r'\d+(?:\.\d+){3}')
 
     def check(self):
         if not self._get_raw_data():
-            self.error('Make sure leases_path is correct and dhcpd.leases is readable by netdata')
+            self.error('Make sure leases_path is correct and leases log file is readable by netdata')
             return False
         elif not have_ipaddress:
             self.error('No ipaddress module. Please install (py2-ipaddress in case of python2)')
@@ -46,8 +46,9 @@ def check(self):
                 return False
 
             # Creating dynamic charts
-            self.order = ['utilization']
-            self.definitions = {'utilization': {'options': [None, 'Pools utilization', 'used %', 'Utulization', 'isc_dhcpd.util', 'line'], 'lines': []} }
+            self.order = ['parse_time', 'utilization']
+            self.definitions = {'utilization': {'options': [None, 'Pools utilization', 'used %', 'Utulization', 'isc_dhcpd.util', 'line'], 'lines': []},
+                               'parse_time': {'options': [None, 'Parse time', 'ms', 'Parse statistics', 'isc_dhcpd.parse', 'line'], 'lines': [['ptime', 'time', 'absolute']]}}
             for pool in self.pools:
                 self.definitions['utilization']['lines'].append([''.join(['ut_', pool]), pool, 'absolute'])
                 self.order.append(''.join(['leases_', pool]))
@@ -60,35 +61,55 @@ def check(self):
 
     def _get_raw_data(self):
         """
-        Open log file
-        :return: str
+        Parses log file
+        :return: tuple(
+                       [ipaddress, lease end time, ...],
+                       length of list,
+                       time to parse leases file
+                      )
         """
         try:
-            with open(self.leases_path, 'rt') as leases:
-                result = leases.read()
+            with open(self.leases_path, 'rt') as dhcp_leases:
+                raw_result = []
+
+                time_start = time()
+                for line in dhcp_leases:
+                    if line[0:3] == 'lea':
+                        raw_result.append(self.regex.search(line).group())
+                    elif line[2:6] == 'ends':
+                        raw_result.append(line[7:28])
+                    else:
+                        continue
+                time_end = time()
+                file_parse_time = round((time_end - time_start) * 1000)
+
         except Exception:
             return None
+
         else:
+            raw_result_length = len(raw_result)
+            result = (raw_result, raw_result_length, file_parse_time)
             return result
 
     def _get_data(self):
         """
-        Parse dhcpd.leases file.
+        :return: dict
         """
         raw_leases = self._get_raw_data()
-        all_leases = dict(self.regex.findall(' '.join(raw_leases.split())))
-
-        if  not all_leases:
-            self.error('Cant parse leases file correctly')
+
+        if not raw_leases:
             return None
 
+        # Result: {ipaddress: end lease time, ...}
+        all_leases = dict(zip([raw_leases[0][_] for _ in range(0, raw_leases[1], 2)], [raw_leases[0][_] for _ in range(1, raw_leases[1], 2)]))
+
         # Result: [active binding, active binding....]. (Expire time (ends date;) - current time > 0)
         active_leases = [k for k, v in all_leases.items() if is_bind_active(all_leases[k])]
 
         # Result: {pool: number of active bindings in pool, ...}
         pools_count = {pool: len([lease for lease in active_leases if is_address_in(lease, pool)]) for pool in self.pools}
 
-        # Result: {pool: number of host ip addresses in pool, }
+        # Result: {pool: number of host ip addresses in pool, ...}
         pools_max = {pool: (2 ** (32 - int(pool.split('/')[1])) - 2) for pool in self.pools}
 
         # Result: {pool: % utilization, ....} (percent)
@@ -98,9 +119,11 @@ def _get_data(self):
         final_count = {''.join(['le_', k]): v for k, v in pools_count.items()}
         final_util = {''.join(['ut_', k]): v for k, v in pools_util.items()}
 
-        final_count.update(final_util)
-
-        return final_count
+        to_netdata = {'ptime': int(raw_leases[2])}
+        to_netdata.update(final_util)
+        to_netdata.update(final_count)
+
+        return to_netdata
 
 def is_bind_active(binding):
     return mktime(strptime(binding, '%w %Y/%m/%d %H:%M:%S')) - mktime(gmtime()) > 0