From f6a7a34fee22360aab93c26de0ef05d64319f8ed Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 31 Jan 2018 14:43:32 +0000 Subject: [PATCH 1/3] Add config file parsing --- bwscanner/configutil.py | 28 ++++++++++++++++++++++++++++ bwscanner/scanner.py | 34 ++++++++++++++++++++++++++-------- data/config.ini | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 bwscanner/configutil.py create mode 100644 data/config.ini diff --git a/bwscanner/configutil.py b/bwscanner/configutil.py new file mode 100644 index 0000000..a78d99b --- /dev/null +++ b/bwscanner/configutil.py @@ -0,0 +1,28 @@ +import os.path +from shutil import copyfile +from ConfigParser import SafeConfigParser + +from bwscanner.logger import log + + +def read_config(cfg_path): + log.debug('reading config %s' % cfg_path) + if not config_exists(cfg_path): + copy_config(cfg_path) + parser = SafeConfigParser() + parser.read([cfg_path]) + # FIXME: handle section names + section = 'default' + return dict(parser.items(section)) + + +def config_exists(cfg_path): + return os.path.isfile(cfg_path) + + +def copy_config(cfg_path, cfg_default_path=None): + if cfg_default_path is None: + cfg_default_path = os.path.join(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))), 'data', 'config.ini') + log.debug("cfg_default_path %s" % cfg_default_path) + copyfile(cfg_default_path, cfg_path) diff --git a/bwscanner/scanner.py b/bwscanner/scanner.py index a7ba050..39d16dd 100755 --- a/bwscanner/scanner.py +++ b/bwscanner/scanner.py @@ -6,12 +6,21 @@ from twisted.internet import reactor from bwscanner.attacher import connect_to_tor +from bwscanner.configutil import read_config from bwscanner.logger import setup_logging, log from bwscanner.measurement import BwScan from bwscanner.aggregate import write_aggregate_data BWSCAN_VERSION = '0.0.1' +APP_NAME = 'bwscanner' +DATA_DIR = os.environ.get("BWSCANNER_DATADIR", click.get_app_dir(APP_NAME)) +CONFIG_FILE = 'config.ini' +LOG_FILE = 'bwscanner.log' + +CTX = dict( + default_map=read_config(os.path.join(DATA_DIR, CONFIG_FILE)) +) class ScanInstance(object): @@ -30,18 +39,27 @@ def __repr__(self): pass_scan = click.make_pass_decorator(ScanInstance) -@click.group() +# FIXME: change all options to take defaults from CTX, ie config file? +@click.group(context_settings=CTX) @click.option('--data-dir', type=click.Path(), - default=os.environ.get("BWSCANNER_DATADIR", click.get_app_dir('bwscanner')), + default=os.environ.get("BWSCANNER_DATADIR", + CTX.get('data_dir', + click.get_app_dir(APP_NAME))), help='Directory where bwscan should stores its measurements and ' 'other data.') -@click.option('-l', '--loglevel', help='The logging level the scanner will use (default: info)', - default='info', type=click.Choice(['debug', 'info', 'warn', 'error', 'critical'])) -@click.option('-f', '--logfile', type=click.Path(), help='The file the log will be written to', - default=os.environ.get("BWSCANNER_LOGFILE", 'bwscanner.log')) -@click.option('--launch-tor/--no-launch-tor', default=False, +@click.option('-l', '--loglevel', + help='The logging level the scanner will use (default: info)', + default=CTX.get('loglevel', 'info'), + type=click.Choice(['debug', 'info', 'warn', 'error', 'critical'])) +@click.option('-f', '--logfile', type=click.Path(), + help='The file the log will be written to', + default=os.environ.get("BWSCANNER_LOGFILE", + CTX.get('logfile', LOG_FILE))) +@click.option('--launch-tor/--no-launch-tor', + default=CTX.get('launch_tor', False), help='Launch Tor or try to connect to an existing Tor instance.') -@click.option('--circuit-build-timeout', default=20, +@click.option('--circuit-build-timeout', + default=CTX.get('circuit_build_timeout', 20), help='Option passed when launching Tor.') @click.version_option(BWSCAN_VERSION) @click.pass_context diff --git a/data/config.ini b/data/config.ini new file mode 100644 index 0000000..86b5d73 --- /dev/null +++ b/data/config.ini @@ -0,0 +1,33 @@ +[default] +data_dir = $HOME/.config/bwscanner +measurement_dir = $HOME/.config/bwscanner/measurements +tor_dir = $HOME/.config/bwscanner/tordata +loglevel = debug +logfile = bwscanner.log +baseurl = https://siv.sunet.se/bwauth/ +bw_files = { + 64 * 1024: ("64M", "6258de4f4d602be75a3458117b29d2c580c4bcb7ba5b9d2c4135c7603109f554"), + 32 * 1024: ("32M", "5a5d66d7865f09498d776f20c9e9791b055a4fff357185f84fb4ecfca7da93f0"), + 16 * 1024: ("16M", "6258de4f4d602be75a3458117b29d2c580c4bcb7ba5b9d2c4135c7603109f554"), + 8 * 1024: ("8M", "738c5604295b9377f7636ce0c2c116f093bb50372f589a6c2332a3bb6bba096a"), + 4 * 1024: ("4M", "4daaa42377d3c87577797d44a8fa569038e7a9d6a5d417a09d8ba41a69456164"), + 2 * 1024: ("2M", "3e39b0bb92912cf1ad6c01fb7c9d592e814a691c61de1f649416f6bba2d15082"), + + # TODO: check whether files smaller than 2M should be provided + # TODO: are k size files key correct? + # 1024: ("1M", "daf6da82bc4a20567dcd5eb7e583f3137800c31eb31f5fed79f27a4278903780"), + # 512: ("512k", "20e1e9b44c3cb445a59138df8a03767356637ec751beee1f9233ca881121adc6"), + # 256: ("256k", "f3655613066fd0db916b0b00bde1a3905516584ea2c4ee4cac3a8ffb08f2f31c"), + # 128: ("128k", "072b052df2fba25a9578b69d49986024747ad9e43472db345a03ca6e22027ba6"), + # 64: ("64k", "73bee20c527362b18d4adb7e638a6513504954367379e7c61f7f45bdc71c5ddb"), + # 32: ("32k", "2ec95ff2c8beca72996161e2bd7831008baf2e012d12b6c84d51e9264fc50fdc"), + # 16: ("16k", "924bddcc93f8f76effd495c47b0d39451e34d8204029fe2b7f85905522255e7b"), +} + +launch_tor = True +circuit_build_timeout = 20 + +partitions = 1 +current_partition = 1 +timeout = 120 +request_limit = 10 From 03978c907d9e6a9fef5ba62eb748f18b24ac2716 Mon Sep 17 00:00:00 2001 From: juga0 Date: Thu, 22 Mar 2018 11:12:16 +0000 Subject: [PATCH 2/3] Use pkg_resources to find the config file path, instead of using __file__ as it would fail when packed in a zip. Include the config file in the package adding it to MANIFEST.in --- MANIFEST.in | 2 +- bwscanner/configutil.py | 12 ++++++++---- bwscanner/data/config.ini | 25 +++++++++++++++++++++++++ data/config.ini | 33 --------------------------------- setup.py | 2 +- 5 files changed, 35 insertions(+), 39 deletions(-) create mode 100644 bwscanner/data/config.ini delete mode 100644 data/config.ini diff --git a/MANIFEST.in b/MANIFEST.in index 8b7aeb1..633075c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include README LICENSE requirements.txt +include README.rst requirements.txt bwscanner/data/config.ini diff --git a/bwscanner/configutil.py b/bwscanner/configutil.py index a78d99b..aef3500 100644 --- a/bwscanner/configutil.py +++ b/bwscanner/configutil.py @@ -1,6 +1,6 @@ import os.path -from shutil import copyfile from ConfigParser import SafeConfigParser +from pkg_resources import resource_string from bwscanner.logger import log @@ -21,8 +21,12 @@ def config_exists(cfg_path): def copy_config(cfg_path, cfg_default_path=None): + # FIXME: obtain the path instead of the content if cfg_default_path is None: - cfg_default_path = os.path.join(os.path.dirname(os.path.dirname( - os.path.abspath(__file__))), 'data', 'config.ini') + content = resource_string(__name__, 'data/config.ini') + else: + with open(cfg_default_path) as fp: + content = cfg_default_path.read() log.debug("cfg_default_path %s" % cfg_default_path) - copyfile(cfg_default_path, cfg_path) + with open(cfg_path, 'w') as fp: + fp.write(content) diff --git a/bwscanner/data/config.ini b/bwscanner/data/config.ini new file mode 100644 index 0000000..f7f01da --- /dev/null +++ b/bwscanner/data/config.ini @@ -0,0 +1,25 @@ +[default] + +data_dir = ~/.config/bwscanner +measurement_dir = ~/.config/bwscanner/measurements +tor_dir = ~/.config/bwscanner/tordata +loglevel = info +logfile = bwscanner.log +baseurl = https://siv.sunet.se/bwauth/ +launch_tor = True +circuit_build_timeout = 20 +partitions = 1 +current_partition = 1 +timeout = 120 +request_limit = 10 +int_keys = partitions current_partition timeout request_limit +bool_keys = launch_tor + +[bw_files] + +64M = 6258de4f4d602be75a3458117b29d2c580c4bcb7ba5b9d2c4135c7603109f554 +32M = 5a5d66d7865f09498d776f20c9e9791b055a4fff357185f84fb4ecfca7da93f0 +16M = 6258de4f4d602be75a3458117b29d2c580c4bcb7ba5b9d2c4135c7603109f554 +8M = 738c5604295b9377f7636ce0c2c116f093bb50372f589a6c2332a3bb6bba096a +4M = 4daaa42377d3c87577797d44a8fa569038e7a9d6a5d417a09d8ba41a69456164 +2M = 3e39b0bb92912cf1ad6c01fb7c9d592e814a691c61de1f649416f6bba2d15082 diff --git a/data/config.ini b/data/config.ini deleted file mode 100644 index 86b5d73..0000000 --- a/data/config.ini +++ /dev/null @@ -1,33 +0,0 @@ -[default] -data_dir = $HOME/.config/bwscanner -measurement_dir = $HOME/.config/bwscanner/measurements -tor_dir = $HOME/.config/bwscanner/tordata -loglevel = debug -logfile = bwscanner.log -baseurl = https://siv.sunet.se/bwauth/ -bw_files = { - 64 * 1024: ("64M", "6258de4f4d602be75a3458117b29d2c580c4bcb7ba5b9d2c4135c7603109f554"), - 32 * 1024: ("32M", "5a5d66d7865f09498d776f20c9e9791b055a4fff357185f84fb4ecfca7da93f0"), - 16 * 1024: ("16M", "6258de4f4d602be75a3458117b29d2c580c4bcb7ba5b9d2c4135c7603109f554"), - 8 * 1024: ("8M", "738c5604295b9377f7636ce0c2c116f093bb50372f589a6c2332a3bb6bba096a"), - 4 * 1024: ("4M", "4daaa42377d3c87577797d44a8fa569038e7a9d6a5d417a09d8ba41a69456164"), - 2 * 1024: ("2M", "3e39b0bb92912cf1ad6c01fb7c9d592e814a691c61de1f649416f6bba2d15082"), - - # TODO: check whether files smaller than 2M should be provided - # TODO: are k size files key correct? - # 1024: ("1M", "daf6da82bc4a20567dcd5eb7e583f3137800c31eb31f5fed79f27a4278903780"), - # 512: ("512k", "20e1e9b44c3cb445a59138df8a03767356637ec751beee1f9233ca881121adc6"), - # 256: ("256k", "f3655613066fd0db916b0b00bde1a3905516584ea2c4ee4cac3a8ffb08f2f31c"), - # 128: ("128k", "072b052df2fba25a9578b69d49986024747ad9e43472db345a03ca6e22027ba6"), - # 64: ("64k", "73bee20c527362b18d4adb7e638a6513504954367379e7c61f7f45bdc71c5ddb"), - # 32: ("32k", "2ec95ff2c8beca72996161e2bd7831008baf2e012d12b6c84d51e9264fc50fdc"), - # 16: ("16k", "924bddcc93f8f76effd495c47b0d39451e34d8204029fe2b7f85905522255e7b"), -} - -launch_tor = True -circuit_build_timeout = 20 - -partitions = 1 -current_partition = 1 -timeout = 120 -request_limit = 10 diff --git a/setup.py b/setup.py index 96257c4..cf89041 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ }, python_requires=">=2.7", # data_files = [('path', ['filename'])] - data_files=[], + include_package_data=True, entry_points={ "console_scripts": [ 'bwscan = bwscanner.scanner:cli', From 2457b8e1e057e8387762329429488a2800bfc1d6 Mon Sep 17 00:00:00 2001 From: juga0 Date: Fri, 23 Mar 2018 09:22:37 +0000 Subject: [PATCH 3/3] Replace click context settings by default_map * Create start function to call cli with parsed config * Parse bw_files section * Convert parsed config values to their type --- bwscanner/configutil.py | 22 ++++++++++++++---- bwscanner/scanner.py | 50 ++++++++++++++++++++++------------------- setup.py | 2 +- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/bwscanner/configutil.py b/bwscanner/configutil.py index aef3500..fc23245 100644 --- a/bwscanner/configutil.py +++ b/bwscanner/configutil.py @@ -6,14 +6,28 @@ def read_config(cfg_path): - log.debug('reading config %s' % cfg_path) + log.debug('Reading config %s' % cfg_path) if not config_exists(cfg_path): copy_config(cfg_path) parser = SafeConfigParser() parser.read([cfg_path]) - # FIXME: handle section names - section = 'default' - return dict(parser.items(section)) + cfg_dict = dict(parser.items('default')) + int_keys = cfg_dict['int_keys'].split(' ') + bool_keys = cfg_dict['bool_keys'].split(' ') + for k in int_keys: + cfg_dict[k] = int(cfg_dict[k]) + for i in bool_keys: + cfg_dict[k] = bool(cfg_dict[k]) + bw_files = dict(parser.items('bw_files')) + cfg_bw_files = {} + for k, v in bw_files.items(): + print(k, v) + if 'm' in k: + number = k.rstrip('m') + size = 1024 * int(number) + cfg_bw_files[size] = (k.upper(), v) + cfg_dict['bw_files'] = cfg_bw_files + return cfg_dict def config_exists(cfg_path): diff --git a/bwscanner/scanner.py b/bwscanner/scanner.py index 39d16dd..32f1d18 100755 --- a/bwscanner/scanner.py +++ b/bwscanner/scanner.py @@ -18,19 +18,18 @@ CONFIG_FILE = 'config.ini' LOG_FILE = 'bwscanner.log' -CTX = dict( - default_map=read_config(os.path.join(DATA_DIR, CONFIG_FILE)) -) - class ScanInstance(object): """ Store the configuration and state for the CLI tool. """ - def __init__(self, data_dir): + def __init__(self, data_dir, measurement_dir=None): self.data_dir = data_dir - self.measurement_dir = os.path.join(data_dir, 'measurements') - self.tor_dir = os.path.join(data_dir, 'tor_data') + if measurement_dir is None: + self.measurement_dir = os.path.join(data_dir, 'measurements') + else: + self.measurement_dir = measurement_dir + self.tor_state = None def __repr__(self): return '' % self.data_dir @@ -39,27 +38,22 @@ def __repr__(self): pass_scan = click.make_pass_decorator(ScanInstance) -# FIXME: change all options to take defaults from CTX, ie config file? -@click.group(context_settings=CTX) +# FIXME: check these errors +# pylint: disable=unexpected-keyword-arg +# pylint: disable=no-value-for-parameter +@click.group() @click.option('--data-dir', type=click.Path(), - default=os.environ.get("BWSCANNER_DATADIR", - CTX.get('data_dir', - click.get_app_dir(APP_NAME))), help='Directory where bwscan should stores its measurements and ' 'other data.') @click.option('-l', '--loglevel', help='The logging level the scanner will use (default: info)', - default=CTX.get('loglevel', 'info'), - type=click.Choice(['debug', 'info', 'warn', 'error', 'critical'])) + type=click.Choice( + ['debug', 'info', 'warn', 'error', 'critical'])) @click.option('-f', '--logfile', type=click.Path(), - help='The file the log will be written to', - default=os.environ.get("BWSCANNER_LOGFILE", - CTX.get('logfile', LOG_FILE))) + help='The file the log will be written to') @click.option('--launch-tor/--no-launch-tor', - default=CTX.get('launch_tor', False), help='Launch Tor or try to connect to an existing Tor instance.') @click.option('--circuit-build-timeout', - default=CTX.get('circuit_build_timeout', 20), help='Option passed when launching Tor.') @click.version_option(BWSCAN_VERSION) @click.pass_context @@ -69,9 +63,12 @@ def cli(ctx, data_dir, loglevel, logfile, launch_tor, circuit_build_timeout): bandwidth measurements can then be aggregate to create the bandwidth values used by the Tor bandwidth authorities when creating the Tor consensus. """ + for k, v in ctx.default_map.items(): + if ctx.params.get(k) is None: + ctx.params[k] = v + # Create the data directory if it doesn't exist - data_dir = os.path.abspath(data_dir) - ctx.obj = ScanInstance(data_dir) + ctx.obj = ScanInstance(ctx.params.get('data_dir')) if not os.path.isdir(ctx.obj.measurement_dir): os.makedirs(ctx.obj.measurement_dir) @@ -81,7 +78,8 @@ def cli(ctx, data_dir, loglevel, logfile, launch_tor, circuit_build_timeout): ctx.obj.tor_dir) # Set up the logger to only output log lines of level `loglevel` and above. - setup_logging(log_level=loglevel, log_name=logfile) + setup_logging(log_level=ctx.params.get('loglevel'), + log_name=ctx.params.get('logfile')) @cli.command(short_help="Measure the Tor relays.") @@ -103,7 +101,8 @@ def scan(scan, partitions, current_partition, timeout, request_limit): # XXX: check that each run is producing the same input set! scan_time = str(int(time.time())) - scan_data_dir = os.path.join(scan.measurement_dir, '{}.running'.format(scan_time)) + scan_data_dir = os.path.join(scan.measurement_dir, + '{}.running'.format(scan_time)) if not os.path.isdir(scan_data_dir): os.makedirs(scan_data_dir) @@ -178,3 +177,8 @@ def aggregate(scan, scan_name, previous): scan.tor_state.addErrback(lambda failure: log.failure("Unexpected error")) scan.tor_state.addCallback(lambda _: reactor.stop()) reactor.run() + + +def start(): + config = read_config(os.path.join(DATA_DIR, CONFIG_FILE)) + return cli(default_map=config) diff --git a/setup.py b/setup.py index cf89041..df235e4 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,6 @@ include_package_data=True, entry_points={ "console_scripts": [ - 'bwscan = bwscanner.scanner:cli', + 'bwscan = bwscanner.scanner:start', ]}, )