From ed893b86a3e81d974ba9fc4c0304a5481bbe76fb Mon Sep 17 00:00:00 2001 From: David Barroso Date: Wed, 19 Aug 2015 16:22:35 +0200 Subject: [PATCH] Added the ability to purge old data --- fib_optimizer/README.md | 2 ++ fib_optimizer/fib_optimizer.py | 24 +++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/fib_optimizer/README.md b/fib_optimizer/README.md index ddde4ca..ddd6b69 100644 --- a/fib_optimizer/README.md +++ b/fib_optimizer/README.md @@ -78,6 +78,7 @@ This application needs the following variables inside the SIR agent: * max_lpm_prefixes - How many LPM prefixes you want. I recommend 16000 for the Arista 7280. * path - Where do you want to store the prefix lists. Recommended '/tmp'. * age - How many hours you want to go back to compute the TopN prefixes. For example, if you set 168 you will compute the TopN prefixes for the last 7 days. +* purge_older_than - BGP data and flows that are older than 'purge_older_than' (in hours) will be purged. Recommended; age*2. If set to 0, no data is purged ever. You can set the variables from the python shell doing the following: @@ -91,6 +92,7 @@ You can set the variables from the python shell doing the following: 'max_lpm_prefixes': 16000, 'path': '/tmp/', 'age': 168, + 'purge_older_than': 336, } sir = pySIR(base_url, verify_ssl=False) sir.delete_variables_by_category_and_name(name='fib_optimizer', category='apps') diff --git a/fib_optimizer/fib_optimizer.py b/fib_optimizer/fib_optimizer.py index 71b70f6..d98082b 100755 --- a/fib_optimizer/fib_optimizer.py +++ b/fib_optimizer/fib_optimizer.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -# TODO metrics!!! -# TODO when we have metrics stop the run if there is no new data since the last run from pySIR.pySIR import pySIR @@ -73,6 +71,8 @@ def _complete_pl(pl, bgp_pl, num): return lem_pl, lpm_pl ''' + + def get_variables(): logger.debug('Getting variables from SIR') v = sir.get_variables_by_category_and_name('apps', 'fib_optimizer').result[0] @@ -148,8 +148,10 @@ def _build_pl(name, prefixes): def install_prefix_lists(): logger.debug('Installing the prefix-lists in the system') - cli_lpm = shlex.split('printf "conf t\n ip prefix-list fib_optimizer_lpm_v4 file:{}/fib_optimizer_lpm_v4"'.format(conf['path'])) - cli_lem = shlex.split('printf "conf t\n ip prefix-list fib_optimizer_lem_v4 file:{}/fib_optimizer_lem_v4"'.format(conf['path'])) + cli_lpm = shlex.split('printf "conf t\n ip prefix-list fib_optimizer_lpm_v4 file:{}/fib_optimizer_lpm_v4"'.format( + conf['path'])) + cli_lem = shlex.split('printf "conf t\n ip prefix-list fib_optimizer_lem_v4 file:{}/fib_optimizer_lem_v4"'.format( + conf['path'])) cli = shlex.split('sudo ip netns exec default FastCli -p 15 -A') p_lpm = subprocess.Popen(cli_lpm, stdout=subprocess.PIPE) @@ -174,7 +176,8 @@ def _merge_pl(pl, pl_file, max_p): original_pl[prefix.rstrip()] = int(seq) if len(original_pl)*0.75 > len(pl): - msg = 'New prefix list ({}) is more than 25\% smaller than the new one ({})'.format(len(original_pl), len(pl)) + msg = 'New prefix list ({}) is more than 25\% smaller than the new one ({})'.format( + len(original_pl), len(pl)) logger.error(msg) raise Exception(msg) @@ -205,6 +208,16 @@ def _merge_pl(pl, pl_file, max_p): return lem, lpm +def purge_old_data(): + logger.debug('Purging old data') + date = datetime.datetime.now() - datetime.timedelta(hours=conf['purge_older_than']) + date_text = date.strftime('%Y-%m-%dT%H:%M:%S') + logger.debug('Deleting BGP data older than: {}'.format(date_text)) + sir.purge_bgp(older_than=date_text) + logger.debug('Deleting flow data older than: {}'.format(date_text)) + sir.purge_flows(older_than=date_text) + + if __name__ == "__main__": if len(sys.argv) < 2: print 'You have to specify the base URL. For example: {} http://127.0.0.1:5000'.format(sys.argv[0]) @@ -232,4 +245,5 @@ def _merge_pl(pl, pl_file, max_p): # We build the files with the prefix lists build_prefix_lists() install_prefix_lists() + purge_old_data() logger.info('End fib_optimizer')