From bb62a35cef9a85fc8e2ac94f7d26a48f91041e16 Mon Sep 17 00:00:00 2001 From: Tobias Schramm Date: Tue, 20 Feb 2018 00:51:58 +0100 Subject: [PATCH 1/2] Rewrite code snippet handling Each former code snippet has been transformed into a full-fledged python module. Responses generated by modules are cached in a single ttl-based cache reducing required CPU-time for subsequent queries drastically. --- announce.py | 13 +- announce.sh | 4 +- gather.py | 68 -------- neighbours.d/batadv | 14 -- neighbours.d/node_id | 1 - nodeinfo.d/hardware/nproc | 1 - nodeinfo.d/hostname | 1 - nodeinfo.d/network/addresses | 17 -- nodeinfo.d/network/mac | 1 - nodeinfo.d/network/mesh | 13 -- nodeinfo.d/node_id | 1 - nodeinfo.d/software/batman-adv/version | 1 - nodeinfo.d/software/fastd/enabled | 1 - nodeinfo.d/software/fastd/version | 1 - nodeinfo.d/software/firmware/base | 1 - nodeinfo.d/software/firmware/release | 1 - nodeinfo.d/vpn | 1 - providers/__init__.py | 157 ++++++++++++++++++ providers/neighbours/batadv.py | 22 +++ providers/neighbours/node_id.py | 1 + providers/nodeinfo/hardware/nproc.py | 6 + providers/nodeinfo/hostname.py | 6 + providers/nodeinfo/network/addresses.py | 26 +++ providers/nodeinfo/network/mac.py | 8 + providers/nodeinfo/network/mesh.py | 21 +++ providers/nodeinfo/node_id.py | 9 + .../nodeinfo/software/batman-adv/version.py | 5 + providers/nodeinfo/software/fastd/enabled.py | 5 + providers/nodeinfo/software/fastd/version.py | 6 + providers/nodeinfo/software/firmware/base.py | 6 + .../nodeinfo/software/firmware/release.py | 6 + providers/nodeinfo/vpn.py | 5 + providers/statistics/idletime.py | 5 + providers/statistics/loadavg.py | 5 + providers/statistics/memory.py | 9 + providers/statistics/node_id.py | 1 + providers/statistics/processes.py | 5 + providers/statistics/traffic.py | 24 +++ providers/statistics/uptime.py | 5 + providers/util.py | 7 + respondd.py | 31 ++-- respondd.service | 2 +- statistics.d/idletime | 1 - statistics.d/loadavg | 1 - statistics.d/memory | 5 - statistics.d/node_id | 1 - statistics.d/processes | 1 - statistics.d/traffic | 16 -- statistics.d/uptime | 1 - util.py | 28 ++++ 50 files changed, 399 insertions(+), 178 deletions(-) delete mode 100644 gather.py delete mode 100644 neighbours.d/batadv delete mode 120000 neighbours.d/node_id delete mode 100644 nodeinfo.d/hardware/nproc delete mode 100644 nodeinfo.d/hostname delete mode 100644 nodeinfo.d/network/addresses delete mode 100644 nodeinfo.d/network/mac delete mode 100644 nodeinfo.d/network/mesh delete mode 100644 nodeinfo.d/node_id delete mode 100644 nodeinfo.d/software/batman-adv/version delete mode 100644 nodeinfo.d/software/fastd/enabled delete mode 100644 nodeinfo.d/software/fastd/version delete mode 100644 nodeinfo.d/software/firmware/base delete mode 100644 nodeinfo.d/software/firmware/release delete mode 100644 nodeinfo.d/vpn create mode 100644 providers/__init__.py create mode 100644 providers/neighbours/batadv.py create mode 120000 providers/neighbours/node_id.py create mode 100644 providers/nodeinfo/hardware/nproc.py create mode 100644 providers/nodeinfo/hostname.py create mode 100644 providers/nodeinfo/network/addresses.py create mode 100644 providers/nodeinfo/network/mac.py create mode 100644 providers/nodeinfo/network/mesh.py create mode 100644 providers/nodeinfo/node_id.py create mode 100644 providers/nodeinfo/software/batman-adv/version.py create mode 100644 providers/nodeinfo/software/fastd/enabled.py create mode 100644 providers/nodeinfo/software/fastd/version.py create mode 100644 providers/nodeinfo/software/firmware/base.py create mode 100644 providers/nodeinfo/software/firmware/release.py create mode 100644 providers/nodeinfo/vpn.py create mode 100644 providers/statistics/idletime.py create mode 100644 providers/statistics/loadavg.py create mode 100644 providers/statistics/memory.py create mode 120000 providers/statistics/node_id.py create mode 100644 providers/statistics/processes.py create mode 100644 providers/statistics/traffic.py create mode 100644 providers/statistics/uptime.py create mode 100644 providers/util.py delete mode 100644 statistics.d/idletime delete mode 100644 statistics.d/loadavg delete mode 100644 statistics.d/memory delete mode 120000 statistics.d/node_id delete mode 100644 statistics.d/processes delete mode 100644 statistics.d/traffic delete mode 100644 statistics.d/uptime create mode 100644 util.py diff --git a/announce.py b/announce.py index 7811f71..49da8fd 100755 --- a/announce.py +++ b/announce.py @@ -3,7 +3,7 @@ import json import argparse -from gather import gather_data +from providers import Provider parser = argparse.ArgumentParser() @@ -15,7 +15,10 @@ args = parser.parse_args() -print(json.dumps(gather_data( - args.directory, - {'batadv_dev': args.batman} -))) +provider = Provider.from_directory('providers', + args.directory) + +print(json.dumps(provider.call( + {'batadv_dev': args.batman} + ) +)) diff --git a/announce.sh b/announce.sh index 6732a26..df7f159 100755 --- a/announce.sh +++ b/announce.sh @@ -31,5 +31,5 @@ done export GZIP="--best" -"${DIR}"/announce.py -d "${DIR}"/nodeinfo.d/ ${BATADV} | gzip | alfred $INTERFACE $SOCKET -s 158 -"${DIR}"/announce.py -d "${DIR}"/statistics.d/ ${BATADV} | gzip | alfred $INTERFACE $SOCKET -s 159 +"${DIR}"/announce.py -d nodeinfo ${BATADV} | gzip | alfred $INTERFACE $SOCKET -s 158 +"${DIR}"/announce.py -d statistics ${BATADV} | gzip | alfred $INTERFACE $SOCKET -s 159 diff --git a/gather.py b/gather.py deleted file mode 100644 index 60cd648..0000000 --- a/gather.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import subprocess -import traceback -from importlib import import_module - - -def _set_value(node, path, value): - ''' Sets a value inside a complex data dictionary. - The path Array must have at least one element. - ''' - key = path[0] - if len(path) == 1: - node[key] = value - elif key in node: - _set_value(node[key], path[1:], value) - else: - node[path[0]] = {} - _set_value(node[key], path[1:], value) - - -def _eval_snippet(f, env): - # definitions of functions used in the snippets go here - def call(cmdnargs): - output = subprocess.check_output(cmdnargs) - lines = output.splitlines() - lines = [line.decode("utf-8") for line in lines] - return lines - - real_env = dict(env) - real_env.update({ - 'call': call, - 'import_module': import_module - }) - - # compile file - with open(f) as fh: - code = compile(fh.read(), os.path.abspath(f), 'eval') - - # ensure that files are opened as UTF-8 - import locale - encoding_backup = locale.getpreferredencoding - locale.getpreferredencoding = lambda _=None: 'UTF-8' - try: - ret = eval(code, real_env) - finally: - locale.getpreferredencoding = encoding_backup - return ret - - -def gather_data(directory, env={}): - data = {} - - if not os.path.isdir(directory): - return - - for dirname, dirnames, filenames in os.walk(directory): - for filename in filenames: - if filename[0] != '.': - path = os.path.join(dirname, filename) - relPath = os.path.relpath(path, directory) - try: - value = _eval_snippet(path, env) - _set_value(data, relPath.rsplit(os.sep), value) - except: - # print but don't abort - traceback.print_exc() - - return data diff --git a/neighbours.d/batadv b/neighbours.d/batadv deleted file mode 100644 index a3ff251..0000000 --- a/neighbours.d/batadv +++ /dev/null @@ -1,14 +0,0 @@ -(lambda neighbours: { - open("{}/address".format(path)).read().strip(): {"neighbours": { - neigh[0]: {"lastseen": neigh[1], "tq": neigh[2]} - for neigh in neighbours - if neigh[3] == path[len('/sys/class/net/{}/lower_'.format(batadv_dev)):] - }} - for path in - import_module('glob').glob('/sys/class/net/{}/lower_*'.format(batadv_dev)) -})([ - (line[0], float(line[1].strip('s')), int(line[2].strip(')')), line[4].strip('[]:')) - for line in map(lambda l: l.replace('(', '').replace('[', '').split(), - open('/sys/kernel/debug/batman_adv/{}/originators'.format(batadv_dev))) - if line[0] == line[3] -]) diff --git a/neighbours.d/node_id b/neighbours.d/node_id deleted file mode 120000 index 7c99767..0000000 --- a/neighbours.d/node_id +++ /dev/null @@ -1 +0,0 @@ -../nodeinfo.d/node_id \ No newline at end of file diff --git a/nodeinfo.d/hardware/nproc b/nodeinfo.d/hardware/nproc deleted file mode 100644 index 4959abc..0000000 --- a/nodeinfo.d/hardware/nproc +++ /dev/null @@ -1 +0,0 @@ -import_module('multiprocessing').cpu_count() diff --git a/nodeinfo.d/hostname b/nodeinfo.d/hostname deleted file mode 100644 index 2730f20..0000000 --- a/nodeinfo.d/hostname +++ /dev/null @@ -1 +0,0 @@ -import_module('socket').gethostname() diff --git a/nodeinfo.d/network/addresses b/nodeinfo.d/network/addresses deleted file mode 100644 index 5cf617a..0000000 --- a/nodeinfo.d/network/addresses +++ /dev/null @@ -1,17 +0,0 @@ -# filter addresses from /proc/net/if_inet6 belonging to given ifaces -(lambda ifaces: - [ - str(import_module("ipaddress").IPv6Address(int(line[0], 16))) - for line in map(str.split, open('/proc/net/if_inet6')) - if line[5] in ifaces - and not int(line[4], 16) & (0x40 | 0x20) - # address is neither tentative nor deprecated - ] -)( - # generate the list of ifaces - [ batadv_dev ] + - list(map( - lambda s: s[len('/sys/class/net/{}/upper_'.format(batadv_dev)):], - import_module('glob').glob('/sys/class/net/{}/upper_*'.format(batadv_dev)) - )) -) diff --git a/nodeinfo.d/network/mac b/nodeinfo.d/network/mac deleted file mode 100644 index ee21ac9..0000000 --- a/nodeinfo.d/network/mac +++ /dev/null @@ -1 +0,0 @@ -open('/sys/class/net/' + batadv_dev + '/address').read().strip() diff --git a/nodeinfo.d/network/mesh b/nodeinfo.d/network/mesh deleted file mode 100644 index 13687ce..0000000 --- a/nodeinfo.d/network/mesh +++ /dev/null @@ -1,13 +0,0 @@ -{ - batadv_dev: { - "interfaces": { - "tunnel": [ - open('/sys/class/net/{}/address'.format(iface)).read().strip() - for iface in map( - lambda line: line.split(':')[0], - call(['batctl', '-m', batadv_dev, 'if']) - ) - ] - } - } -} diff --git a/nodeinfo.d/node_id b/nodeinfo.d/node_id deleted file mode 100644 index efdd705..0000000 --- a/nodeinfo.d/node_id +++ /dev/null @@ -1 +0,0 @@ -open('/sys/class/net/' + batadv_dev + '/address').read().strip().replace(':', '') diff --git a/nodeinfo.d/software/batman-adv/version b/nodeinfo.d/software/batman-adv/version deleted file mode 100644 index 9a77d55..0000000 --- a/nodeinfo.d/software/batman-adv/version +++ /dev/null @@ -1 +0,0 @@ -open('/sys/module/batman_adv/version').read().strip() diff --git a/nodeinfo.d/software/fastd/enabled b/nodeinfo.d/software/fastd/enabled deleted file mode 100644 index 0ca9514..0000000 --- a/nodeinfo.d/software/fastd/enabled +++ /dev/null @@ -1 +0,0 @@ -True diff --git a/nodeinfo.d/software/fastd/version b/nodeinfo.d/software/fastd/version deleted file mode 100644 index 0acc2c7..0000000 --- a/nodeinfo.d/software/fastd/version +++ /dev/null @@ -1 +0,0 @@ -call(['fastd','-v'])[0].split(' ')[1] diff --git a/nodeinfo.d/software/firmware/base b/nodeinfo.d/software/firmware/base deleted file mode 100644 index 54db50b..0000000 --- a/nodeinfo.d/software/firmware/base +++ /dev/null @@ -1 +0,0 @@ -call(['lsb_release','-is'])[0] diff --git a/nodeinfo.d/software/firmware/release b/nodeinfo.d/software/firmware/release deleted file mode 100644 index f40250e..0000000 --- a/nodeinfo.d/software/firmware/release +++ /dev/null @@ -1 +0,0 @@ -call(['lsb_release','-rs'])[0] diff --git a/nodeinfo.d/vpn b/nodeinfo.d/vpn deleted file mode 100644 index 0ca9514..0000000 --- a/nodeinfo.d/vpn +++ /dev/null @@ -1 +0,0 @@ -True diff --git a/providers/__init__.py b/providers/__init__.py new file mode 100644 index 0000000..5b1e5f2 --- /dev/null +++ b/providers/__init__.py @@ -0,0 +1,157 @@ +import os +import time +import traceback +from util import source_dirs, find_modules +from importlib import import_module + +class DataSource(): + ''' Base data source, inherited by sources in provider directories + ''' + def cache_ttl(self): + ''' Default cache ttl, set to 0 to disable cache + ''' + return 30 + + def required_args(self): + ''' Returns a list of required argument names + ''' + return [] + + def call(self, *args): + ''' Override with actual implementation, args contains arguments listed + in required_args in order + ''' + raise NotImplementedError() + + + +def _set_value(node, path, value): + ''' Sets a value inside a complex data dictionary. + The path Array must have at least one element. + ''' + key = path[0] + if len(path) == 1: + node[key] = value + elif key in node: + _set_value(node[key], path[1:], value) + else: + node[path[0]] = {} + _set_value(node[key], path[1:], value) + +class SourceCache(): + ''' Simple singleton-based cache + ''' + class CacheEntry(): + def __init__(self, key, value, ttl): + self.key = key + self.value = value + self.expiry = time.monotonic() + ttl + + def isvalid(self): + return time.time() < self.expiry + + instance = None + + @classmethod + def getinstance(cls): + if not cls.instance: + cls.instance = cls() + + return cls.instance + + def __init__(self): + self.cache = {} + + def put(self, key, value, ttl): + self.cache[key] = self.CacheEntry(key, value, ttl) + + def has(self, key): + return (key in self.cache) and self.cache[key].isvalid() + + def get(self, key): + if not self.has(key): + return None + + return self.cache[key].value + +class InvalidSourceModuleExeption(Exception): + def __init__(self, path): + super().__init__(self, 'Invalid data source,' + + ' module at "{}" does not spcify required member "Source"') + +class Source(): + @classmethod + def from_file(cls, modulepath, jsonpath): + # construct a relative path + relpath = '.'.join([''] + modulepath) + # import modules relative to this package + module = import_module(relpath, __package__) + if not module.Source: + raise InvalidSourceModuleExeption(relpath) + + return Source(jsonpath, module.Source()) + + def __init__(self, path, source): + self.path = path + self.source = source + + def call(self, env): + cache = SourceCache.getinstance() + cache_result = cache.get(self) + if cache_result: + return cache_result + + args = [] + for argname in self.source.required_args(): + args.append(env[argname]) + + result = self.source.call(*args) + cache.put(self, result, self.source.cache_ttl()) + + return result + +class Provider(): + @classmethod + def from_directory(cls, basepath, dirname): + provider = Provider(dirname) + + sourcepaths = find_modules(os.path.join(basepath, dirname)) + for path in sourcepaths: + for fname in path[1]: + src = Source.from_file([dirname] + path[0] + [fname], path[0] + [fname]) + provider.add_source(src) + + return provider + + + def __init__(self, name): + self.name = name + self.sources = [] + + def add_source(self, source): + self.sources.append(source) + + def call(self, env): + ret = {} + + for source in self.sources: + try: + _set_value(ret, source.path, source.call(env)) + except: + traceback.print_exc() + + return ret + + +def get_providers(directory): + providers = {} + + if not os.path.isdir(directory): + raise FileNotFoundError("The path '{}' is not a valid directory".format(directory)) + + for dir in source_dirs(directory): + provider = Provider.from_directory(directory, dir) + + providers[provider.name] = provider + + return providers diff --git a/providers/neighbours/batadv.py b/providers/neighbours/batadv.py new file mode 100644 index 0000000..5fd4aaa --- /dev/null +++ b/providers/neighbours/batadv.py @@ -0,0 +1,22 @@ +import providers +from glob import glob + +class Source(providers.DataSource): + def required_args(self): + return ['batadv_dev'] + + def call(self, batadv_dev): + return (lambda neighbours: { + open("{}/address".format(path)).read().strip(): {"neighbours": { + neigh[0]: {"lastseen": neigh[1], "tq": neigh[2]} + for neigh in neighbours + if neigh[3] == path[len('/sys/class/net/{}/lower_'.format(batadv_dev)):] + }} + for path in + glob('/sys/class/net/{}/lower_*'.format(batadv_dev)) + })([ + (line[0], float(line[1].strip('s')), int(line[2].strip(')')), line[4].strip('[]:')) + for line in map(lambda l: l.replace('(', '').replace('[', '').split(), + open('/sys/kernel/debug/batman_adv/{}/originators'.format(batadv_dev))) + if line[0] == line[3] + ]) diff --git a/providers/neighbours/node_id.py b/providers/neighbours/node_id.py new file mode 120000 index 0000000..4906303 --- /dev/null +++ b/providers/neighbours/node_id.py @@ -0,0 +1 @@ +../nodeinfo/node_id.py \ No newline at end of file diff --git a/providers/nodeinfo/hardware/nproc.py b/providers/nodeinfo/hardware/nproc.py new file mode 100644 index 0000000..94e55bd --- /dev/null +++ b/providers/nodeinfo/hardware/nproc.py @@ -0,0 +1,6 @@ +import providers +import multiprocessing + +class Source(providers.DataSource): + def call(self): + return multiprocessing.cpu_count() diff --git a/providers/nodeinfo/hostname.py b/providers/nodeinfo/hostname.py new file mode 100644 index 0000000..fbcf7f4 --- /dev/null +++ b/providers/nodeinfo/hostname.py @@ -0,0 +1,6 @@ +import providers +import socket + +class Source(providers.DataSource): + def call(self): + return socket.gethostname() diff --git a/providers/nodeinfo/network/addresses.py b/providers/nodeinfo/network/addresses.py new file mode 100644 index 0000000..437225f --- /dev/null +++ b/providers/nodeinfo/network/addresses.py @@ -0,0 +1,26 @@ +import providers +from ipaddress import IPv6Address +from glob import glob + +class Source(providers.DataSource): + def required_args(self): + return ['batadv_dev'] + + def call(self, batadv_dev): + # filter addresses from /proc/net/if_inet6 belonging to given ifaces + return (lambda ifaces: + [ + str(IPv6Address(int(line[0], 16))) + for line in map(str.split, open('/proc/net/if_inet6')) + if line[5] in ifaces + and not int(line[4], 16) & (0x40 | 0x20) + # address is neither tentative nor deprecated + ] + )( + # generate the list of ifaces + [ batadv_dev ] + + list(map( + lambda s: s[len('/sys/class/net/{}/upper_'.format(batadv_dev)):], + glob('/sys/class/net/{}/upper_*'.format(batadv_dev)) + )) + ) diff --git a/providers/nodeinfo/network/mac.py b/providers/nodeinfo/network/mac.py new file mode 100644 index 0000000..94beaf5 --- /dev/null +++ b/providers/nodeinfo/network/mac.py @@ -0,0 +1,8 @@ +import providers + +class Source(providers.DataSource): + def required_args(self): + return ['batadv_dev'] + + def call(self, batadv_dev): + return open('/sys/class/net/' + batadv_dev + '/address').read().strip() diff --git a/providers/nodeinfo/network/mesh.py b/providers/nodeinfo/network/mesh.py new file mode 100644 index 0000000..05870e8 --- /dev/null +++ b/providers/nodeinfo/network/mesh.py @@ -0,0 +1,21 @@ +import providers +from providers.util import call + +class Source(providers.DataSource): + def required_args(self): + return ['batadv_dev'] + + def call(self, batadv_dev): + return { + batadv_dev: { + "interfaces": { + "tunnel": [ + open('/sys/class/net/{}/address'.format(iface)).read().strip() + for iface in map( + lambda line: line.split(':')[0], + call(['batctl', '-m', batadv_dev, 'if']) + ) + ] + } + } + } diff --git a/providers/nodeinfo/node_id.py b/providers/nodeinfo/node_id.py new file mode 100644 index 0000000..0169c8b --- /dev/null +++ b/providers/nodeinfo/node_id.py @@ -0,0 +1,9 @@ +import providers +import socket + +class Source(providers.DataSource): + def required_args(self): + return ['batadv_dev'] + + def call(self, batadv_dev): + return open('/sys/class/net/' + batadv_dev + '/address').read().strip().replace(':', '') diff --git a/providers/nodeinfo/software/batman-adv/version.py b/providers/nodeinfo/software/batman-adv/version.py new file mode 100644 index 0000000..6ac1f9d --- /dev/null +++ b/providers/nodeinfo/software/batman-adv/version.py @@ -0,0 +1,5 @@ +import providers + +class Source(providers.DataSource): + def call(self): + return open('/sys/module/batman_adv/version').read().strip() diff --git a/providers/nodeinfo/software/fastd/enabled.py b/providers/nodeinfo/software/fastd/enabled.py new file mode 100644 index 0000000..dc20dc5 --- /dev/null +++ b/providers/nodeinfo/software/fastd/enabled.py @@ -0,0 +1,5 @@ +import providers + +class Source(providers.DataSource): + def call(self): + return True diff --git a/providers/nodeinfo/software/fastd/version.py b/providers/nodeinfo/software/fastd/version.py new file mode 100644 index 0000000..84f8e57 --- /dev/null +++ b/providers/nodeinfo/software/fastd/version.py @@ -0,0 +1,6 @@ +import providers +from providers.util import call + +class Source(providers.DataSource): + def call(self): + return call(['fastd','-v'])[0].split(' ')[1] diff --git a/providers/nodeinfo/software/firmware/base.py b/providers/nodeinfo/software/firmware/base.py new file mode 100644 index 0000000..663707b --- /dev/null +++ b/providers/nodeinfo/software/firmware/base.py @@ -0,0 +1,6 @@ +import providers +from providers.util import call + +class Source(providers.DataSource): + def call(self): + return call(['lsb_release','-is'])[0] diff --git a/providers/nodeinfo/software/firmware/release.py b/providers/nodeinfo/software/firmware/release.py new file mode 100644 index 0000000..f674797 --- /dev/null +++ b/providers/nodeinfo/software/firmware/release.py @@ -0,0 +1,6 @@ +import providers +from providers.util import call + +class Source(providers.DataSource): + def call(self): + return call(['lsb_release','-rs'])[0] diff --git a/providers/nodeinfo/vpn.py b/providers/nodeinfo/vpn.py new file mode 100644 index 0000000..dc20dc5 --- /dev/null +++ b/providers/nodeinfo/vpn.py @@ -0,0 +1,5 @@ +import providers + +class Source(providers.DataSource): + def call(self): + return True diff --git a/providers/statistics/idletime.py b/providers/statistics/idletime.py new file mode 100644 index 0000000..d067f54 --- /dev/null +++ b/providers/statistics/idletime.py @@ -0,0 +1,5 @@ +import providers + +class Source(providers.DataSource): + def call(self): + return float(open('/proc/uptime').read().split(' ')[1]) diff --git a/providers/statistics/loadavg.py b/providers/statistics/loadavg.py new file mode 100644 index 0000000..e3b9987 --- /dev/null +++ b/providers/statistics/loadavg.py @@ -0,0 +1,5 @@ +import providers + +class Source(providers.DataSource): + def call(self): + return float(open('/proc/loadavg').read().split(' ')[0]) diff --git a/providers/statistics/memory.py b/providers/statistics/memory.py new file mode 100644 index 0000000..bc677f0 --- /dev/null +++ b/providers/statistics/memory.py @@ -0,0 +1,9 @@ +import providers + +class Source(providers.DataSource): + def call(self): + return dict( + (key.replace('Mem', '').lower(), int(value.split(' ')[0])) + for key, value in map(lambda s: map(str.strip, s.split(': ', 1)), open('/proc/meminfo').readlines()) + if key in ('MemTotal', 'MemFree', 'Buffers', 'Cached') + ) diff --git a/providers/statistics/node_id.py b/providers/statistics/node_id.py new file mode 120000 index 0000000..4906303 --- /dev/null +++ b/providers/statistics/node_id.py @@ -0,0 +1 @@ +../nodeinfo/node_id.py \ No newline at end of file diff --git a/providers/statistics/processes.py b/providers/statistics/processes.py new file mode 100644 index 0000000..880fa27 --- /dev/null +++ b/providers/statistics/processes.py @@ -0,0 +1,5 @@ +import providers + +class Source(providers.DataSource): + def call(self): + return dict(zip(('running', 'total'), map(int, open('/proc/loadavg').read().split(' ')[3].split('/')))) diff --git a/providers/statistics/traffic.py b/providers/statistics/traffic.py new file mode 100644 index 0000000..72a554f --- /dev/null +++ b/providers/statistics/traffic.py @@ -0,0 +1,24 @@ +import providers +from providers.util import call + +class Source(providers.DataSource): + def required_args(self): + return ['batadv_dev'] + + def call(self, batadv_dev): + return (lambda fields: + dict( + (key, dict( + (type_, int(value_)) + for key_, type_, value_ in fields + if key_ == key)) + for key in ['rx', 'tx', 'forward', 'mgmt_rx', 'mgmt_tx'] + ) + )(list( + ( + key.replace('_bytes', '').replace('_dropped', ''), + 'bytes' if key.endswith('_bytes') else 'dropped' if key.endswith('_dropped') else 'packets', + value + ) + for key, value in map(lambda s: list(map(str.strip, s.split(': ', 1))), call(['ethtool', '-S', batadv_dev])[1:]) + )) diff --git a/providers/statistics/uptime.py b/providers/statistics/uptime.py new file mode 100644 index 0000000..019dbea --- /dev/null +++ b/providers/statistics/uptime.py @@ -0,0 +1,5 @@ +import providers + +class Source(providers.DataSource): + def call(self): + return float(open('/proc/uptime').read().split(' ')[0]) diff --git a/providers/util.py b/providers/util.py new file mode 100644 index 0000000..2d49206 --- /dev/null +++ b/providers/util.py @@ -0,0 +1,7 @@ +import subprocess + +def call(cmdline): + output = subprocess.check_output(cmdline) + lines = output.splitlines() + lines = [line.decode("utf-8") for line in lines] + return lines diff --git a/respondd.py b/respondd.py index 03de9f6..0d9d422 100755 --- a/respondd.py +++ b/respondd.py @@ -8,25 +8,17 @@ import os from zlib import compress -from gather import gather_data +from providers import get_providers -def get_handler(directory, env): +def get_handler(providers, env): class ResponddUDPHandler(socketserver.BaseRequestHandler): - @staticmethod - def _get_provider_dir(provider): - return os.path.join(directory, "{}.d".format(provider)) - - def multi_request(self, providers): + def multi_request(self, providernames): ret = {} - for provider in providers: - if '/' in provider: - continue + for name in providernames: try: - ret[provider] = gather_data( - self._get_provider_dir(provider), - env - ) + provider = providers[name] + ret[provider.name] = provider.call(env) except: pass return compress(str.encode(json.dumps(ret)))[2:-4] @@ -38,11 +30,8 @@ def handle(self): if data.startswith("GET "): response = self.multi_request(data.split(" ")[1:]) - elif '/' not in data: - answer = gather_data( - self._get_provider_dir(data), - env - ) + else: + answer = providers[data].call(env) if answer: response = str.encode(json.dumps(answer)) @@ -65,7 +54,7 @@ def handle(self): action='append', metavar='', help='interface on which the group is joined') parser.add_argument('-d', dest='directory', - default='.', metavar='', + default='./providers', metavar='', help='data provider directory (default: $PWD)') parser.add_argument('-b', dest='batadv_iface', default='bat0', metavar='', @@ -75,7 +64,7 @@ def handle(self): socketserver.ThreadingUDPServer.address_family = socket.AF_INET6 server = socketserver.ThreadingUDPServer( ("", args.port), - get_handler(args.directory, {'batadv_dev': args.batadv_iface}) + get_handler(get_providers(args.directory), {'batadv_dev': args.batadv_iface}) ) if args.mcast_ifaces: diff --git a/respondd.service b/respondd.service index 47d93bb..6555878 100644 --- a/respondd.service +++ b/respondd.service @@ -2,7 +2,7 @@ Description=Respondd [Service] -ExecStart=/opt/mesh-announce/respondd.py -d /opt/mesh-announce -i -i -b +ExecStart=/opt/mesh-announce/respondd.py -d /opt/mesh-announce/providers -i -i -b Restart=always Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/statistics.d/idletime b/statistics.d/idletime deleted file mode 100644 index c608729..0000000 --- a/statistics.d/idletime +++ /dev/null @@ -1 +0,0 @@ -float(open('/proc/uptime').read().split(' ')[1]) diff --git a/statistics.d/loadavg b/statistics.d/loadavg deleted file mode 100644 index 8fa16a4..0000000 --- a/statistics.d/loadavg +++ /dev/null @@ -1 +0,0 @@ -float(open('/proc/loadavg').read().split(' ')[0]) diff --git a/statistics.d/memory b/statistics.d/memory deleted file mode 100644 index 1ca2476..0000000 --- a/statistics.d/memory +++ /dev/null @@ -1,5 +0,0 @@ -dict( - (key.replace('Mem', '').lower(), int(value.split(' ')[0])) - for key, value in map(lambda s: map(str.strip, s.split(': ', 1)), open('/proc/meminfo').readlines()) - if key in ('MemTotal', 'MemFree', 'Buffers', 'Cached') -) diff --git a/statistics.d/node_id b/statistics.d/node_id deleted file mode 120000 index 7c99767..0000000 --- a/statistics.d/node_id +++ /dev/null @@ -1 +0,0 @@ -../nodeinfo.d/node_id \ No newline at end of file diff --git a/statistics.d/processes b/statistics.d/processes deleted file mode 100644 index 6fddb14..0000000 --- a/statistics.d/processes +++ /dev/null @@ -1 +0,0 @@ -dict(zip(('running', 'total'), map(int, open('/proc/loadavg').read().split(' ')[3].split('/')))) diff --git a/statistics.d/traffic b/statistics.d/traffic deleted file mode 100644 index 17a7789..0000000 --- a/statistics.d/traffic +++ /dev/null @@ -1,16 +0,0 @@ -(lambda fields: - dict( - (key, dict( - (type_, int(value_)) - for key_, type_, value_ in fields - if key_ == key)) - for key in ['rx', 'tx', 'forward', 'mgmt_rx', 'mgmt_tx'] - ) -)(list( - ( - key.replace('_bytes', '').replace('_dropped', ''), - 'bytes' if key.endswith('_bytes') else 'dropped' if key.endswith('_dropped') else 'packets', - value - ) - for key, value in map(lambda s: list(map(str.strip, s.split(': ', 1))), call(['ethtool', '-S', batadv_dev])[1:]) -)) diff --git a/statistics.d/uptime b/statistics.d/uptime deleted file mode 100644 index 55aec8c..0000000 --- a/statistics.d/uptime +++ /dev/null @@ -1 +0,0 @@ -float(open('/proc/uptime').read().split(' ')[0]) diff --git a/util.py b/util.py new file mode 100644 index 0000000..27c7201 --- /dev/null +++ b/util.py @@ -0,0 +1,28 @@ +import os + +def _file_name_filter(fname): + return not (fname.startswith('__') or fname.startswith('.')) + +def source_dirs(dir): + dirs = next(os.walk(dir))[1] + return filter(_file_name_filter, dirs) + +def modules(dir): + return [os.path.splitext(f)[0] for f in next(os.walk(dir))[2] + if f.endswith('.py') and _file_name_filter(f)] + +def find_modules(base, *path): + path_files = [] + dir_abs = base + dir_abs = os.path.join(base, *path) + dirs = source_dirs(dir_abs) + for dir in dirs: + # Ugly hack because python < 3.5 does not support arguments after asterisk expression + path_files += find_modules(base, *(list(path) + [dir])) + + files = list(modules(dir_abs)) + if len(files) > 0: + lpath = list(map(os.path.basename, path)) + path_files.append((lpath, files)) + return path_files + From 39d2aa54ce6d797a353d686d6ae70d9b194551de Mon Sep 17 00:00:00 2001 From: Tobias Schramm Date: Tue, 24 Jul 2018 02:19:51 +0200 Subject: [PATCH 2/2] Add module announcing mesh legacy IP address of gateway Required for ddhcpd gateway legacy IP address selection --- providers/gateway/address.py | 8 ++++++++ providers/gateway/node_id.py | 1 + respondd.py | 5 ++++- respondd.service | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 providers/gateway/address.py create mode 120000 providers/gateway/node_id.py diff --git a/providers/gateway/address.py b/providers/gateway/address.py new file mode 100644 index 0000000..c16ea23 --- /dev/null +++ b/providers/gateway/address.py @@ -0,0 +1,8 @@ +import providers + +class Source(providers.DataSource): + def required_args(self): + return ['mesh_ipv4'] + + def call(self, mesh_ipv4): + return { "ipv4": mesh_ipv4 } diff --git a/providers/gateway/node_id.py b/providers/gateway/node_id.py new file mode 120000 index 0000000..4906303 --- /dev/null +++ b/providers/gateway/node_id.py @@ -0,0 +1 @@ +../nodeinfo/node_id.py \ No newline at end of file diff --git a/respondd.py b/respondd.py index 0d9d422..452bbcf 100755 --- a/respondd.py +++ b/respondd.py @@ -59,12 +59,15 @@ def handle(self): parser.add_argument('-b', dest='batadv_iface', default='bat0', metavar='', help='batman-adv interface (default: bat0)') + parser.add_argument('-m', dest='mesh_ipv4', + metavar='', + help='mesh ipv4 address') args = parser.parse_args() socketserver.ThreadingUDPServer.address_family = socket.AF_INET6 server = socketserver.ThreadingUDPServer( ("", args.port), - get_handler(get_providers(args.directory), {'batadv_dev': args.batadv_iface}) + get_handler(get_providers(args.directory), {'batadv_dev': args.batadv_iface, 'mesh_ipv4': args.mesh_ipv4}) ) if args.mcast_ifaces: diff --git a/respondd.service b/respondd.service index 6555878..57adf38 100644 --- a/respondd.service +++ b/respondd.service @@ -2,7 +2,7 @@ Description=Respondd [Service] -ExecStart=/opt/mesh-announce/respondd.py -d /opt/mesh-announce/providers -i -i -b +ExecStart=/opt/mesh-announce/respondd.py -d /opt/mesh-announce/providers -i -i -b -m Restart=always Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin