Added the ability to purge old data

fib_optimizer/README.md

@@ -78,6 +78,7 @@ This application needs the following variables inside the SIR agent:
 * max_lpm_prefixes - How many LPM prefixes you want. I recommend 16000 for the Arista 7280.
 * path - Where do you want to store the prefix lists. Recommended '/tmp'.
 * age - How many hours you want to go back to compute the TopN prefixes. For example, if you set 168 you will compute the TopN prefixes for the last 7 days.
+* purge_older_than - BGP data and flows that are older than 'purge_older_than' (in hours) will be purged. Recommended; age*2. If set to 0, no data is purged ever.
 
 You can set the variables from the python shell doing the following:
 
@@ -91,6 +92,7 @@ You can set the variables from the python shell doing the following:
         'max_lpm_prefixes': 16000,
         'path': '/tmp/',
         'age': 168,
+        'purge_older_than': 336,
     }
     sir = pySIR(base_url, verify_ssl=False)
     sir.delete_variables_by_category_and_name(name='fib_optimizer', category='apps')

fib_optimizer/fib_optimizer.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-# TODO metrics!!!
-# TODO when we have metrics stop the run if there is no new data since the last run
 
 from pySIR.pySIR import pySIR
 
@@ -73,6 +71,8 @@ def _complete_pl(pl, bgp_pl, num):
     return lem_pl, lpm_pl
 
 '''
+
+
 def get_variables():
     logger.debug('Getting variables from SIR')
     v = sir.get_variables_by_category_and_name('apps', 'fib_optimizer').result[0]
@@ -148,8 +148,10 @@ def _build_pl(name, prefixes):
 def install_prefix_lists():
     logger.debug('Installing the prefix-lists in the system')
 
-    cli_lpm = shlex.split('printf "conf t\n ip prefix-list fib_optimizer_lpm_v4 file:{}/fib_optimizer_lpm_v4"'.format(conf['path']))
-    cli_lem = shlex.split('printf "conf t\n ip prefix-list fib_optimizer_lem_v4 file:{}/fib_optimizer_lem_v4"'.format(conf['path']))
+    cli_lpm = shlex.split('printf "conf t\n ip prefix-list fib_optimizer_lpm_v4 file:{}/fib_optimizer_lpm_v4"'.format(
+                                                                                                        conf['path']))
+    cli_lem = shlex.split('printf "conf t\n ip prefix-list fib_optimizer_lem_v4 file:{}/fib_optimizer_lem_v4"'.format(
+                                                                                                        conf['path']))
     cli = shlex.split('sudo ip netns exec default FastCli -p 15 -A')
 
     p_lpm = subprocess.Popen(cli_lpm, stdout=subprocess.PIPE)
@@ -174,7 +176,8 @@ def _merge_pl(pl, pl_file, max_p):
                     original_pl[prefix.rstrip()] = int(seq)
 
             if len(original_pl)*0.75 > len(pl):
-                msg = 'New prefix list ({}) is more than 25\% smaller than the new one ({})'.format(len(original_pl), len(pl))
+                msg = 'New prefix list ({}) is more than 25\% smaller than the new one ({})'.format(
+                                                                                            len(original_pl), len(pl))
                 logger.error(msg)
                 raise Exception(msg)
 
@@ -205,6 +208,16 @@ def _merge_pl(pl, pl_file, max_p):
     return lem, lpm
 
 
+def purge_old_data():
+    logger.debug('Purging old data')
+    date = datetime.datetime.now() - datetime.timedelta(hours=conf['purge_older_than'])
+    date_text = date.strftime('%Y-%m-%dT%H:%M:%S')
+    logger.debug('Deleting BGP data older than: {}'.format(date_text))
+    sir.purge_bgp(older_than=date_text)
+    logger.debug('Deleting flow data older than: {}'.format(date_text))
+    sir.purge_flows(older_than=date_text)
+
+
 if __name__ == "__main__":
     if len(sys.argv) < 2:
         print 'You have to specify the base URL. For example: {} http://127.0.0.1:5000'.format(sys.argv[0])
@@ -232,4 +245,5 @@ def _merge_pl(pl, pl_file, max_p):
     # We build the files with the prefix lists
     build_prefix_lists()
     install_prefix_lists()
+    purge_old_data()
     logger.info('End fib_optimizer')