From 1fb3b1ec56d57d697587222fe70fd62133b2b6b4 Mon Sep 17 00:00:00 2001 From: anathema Date: Fri, 5 Aug 2016 10:08:51 +0100 Subject: [PATCH] version 1.1 --- CHANGELOG | 11 ++++++ engines/es.py | 93 +++++++++++++++++++++++++++++++++++++++++++++------ onionoo-ng.py | 36 +++++++++++--------- settings.py | 6 ++-- 4 files changed, 118 insertions(+), 28 deletions(-) create mode 100644 CHANGELOG diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..4ff9d6c --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,11 @@ +ONIONOO-NG CHANGELOG + + +VERSION 1.1 (05/08/2016) +- added 'relays_published' and 'bridges_published' values +- improved support for 'search' parameter +- fixed a bug about 'offset' parameter (see issue #1) +- minor style fixes + +VERSION 1.0 (03/08/2016) +- first release: almost totally backward compatible with current Onionoo's protocol \ No newline at end of file diff --git a/engines/es.py b/engines/es.py index 188122a..0bfbbdb 100644 --- a/engines/es.py +++ b/engines/es.py @@ -4,8 +4,9 @@ from tornado import gen, escape from tornado.web import HTTPError from datetime import date, timedelta, datetime -import urllib from IPy import IP +import urllib +import base64 AsyncHTTPClient.configure(None, max_clients=settings.ASYNC_ES_MAX_CLIENT) @@ -27,7 +28,7 @@ def check_connection(self): response = yield self.es.get_by_path("/") if response.code != 200 or not response.body: - raise Exception + raise EnvironmentError @gen.coroutine def search(self, mapping, query, extras): @@ -98,6 +99,7 @@ def search(self, mapping, query, extras): result = yield self.es.search(index=self.index, type=self.type_mapping, source=body_query, size=extra_parameters['limit']) + if result.code != 200 or not result.body: raise HTTPError(status_code=result.code) @@ -111,6 +113,35 @@ def search(self, mapping, query, extras): raise gen.Return(return_data) + @gen.coroutine + def get_last_node(self, mapping): + + body_query = { + "sort": { + "last_seen": { + "unmapped_type": "string", "order": "desc" + } + }, + "query": { + "match_all": {} + }, "from": 0, "_source": ["*"], "size": 1 + } + + result = yield self.es.search(index=self.index, type=mapping, source=body_query) + + if result.code != 200 or not result.body: + raise HTTPError(status_code=result.code) + + result = escape.json_decode(result.body) + + if 'hits' in result and 'hits' in result['hits']: + last_seen = result['hits']['hits'][0]['_source']['last_seen'] + else: + # generic error related to ES - shouldn't happen... + raise Exception(result.error) + + raise gen.Return(last_seen) + def __parse_query(self, q): if 'as' in q: q['as_number'] = q.pop('as').split(',')[0] @@ -154,8 +185,14 @@ def __parse_query(self, q): if len(tokenz) == 1: if self.___is_ip(tokenz[0]): new_se.append("(or_address:{0} OR dir_address:{0})".format(tokenz[0])) + elif self.__is_fingerprint(tokenz[0]): + fingerprint = self.__transform_fingerprint(tokenz[0]) + if fingerprint[0] == "$": + new_se.append("(fingerprint:{0}* OR hashed_fingerprint:{0}*)".format(fingerprint[1:])) + else: + new_se.append("(fingerprint:*{0}* OR hashed_fingerprint:*{0}*)".format(fingerprint)) else: - new_se.append("(nickname:*{0}* OR fingerprint:*{0}* OR hashed_fingerprint:*{0}*)".format(tokenz[0])) + new_se.append("nickname:*{0}*".format(tokenz[0])) else: if tokenz[0] in forbidden_keys: raise HTTPError(status_code=400) @@ -166,17 +203,24 @@ def __parse_query(self, q): return q - def __parse_extra(self, extra): + @staticmethod + def __parse_extra(extra): return_data = {'limit': settings.ES_RESULT_SIZE, 'offset': 0, 'fields': ["*"], 'sort': {'field': None, 'order': 'asc', 'type': 'string'}} - if 'limit' in extra and int(extra['limit']) > 0: - return_data['limit'] = extra['limit'] - elif 'limit' in extra and int(extra['limit']) <= 0: - return_data['limit'] = 0 + if 'limit' in extra: + # 'limit' cannot go beyond the settings.ES_RESULT_SIZE value + if 0 < int(extra['limit']) < settings.ES_RESULT_SIZE: + return_data['limit'] = int(extra['limit']) + elif int(extra['limit']) <= 0: + return_data['limit'] = 0 if 'offset' in extra and int(extra['offset']) > 0: - return_data['offset'] = extra['offset'] + return_data['offset'] = int(extra['offset']) + + # offset + limit must be <= settings.ES_RESULT_SIZE + if (return_data['offset'] + return_data['limit']) > settings.ES_RESULT_SIZE: + return_data['limit'] -= return_data['offset'] if 'fields' in extra: return_data['fields'] = extra['fields'].split(',') @@ -191,7 +235,8 @@ def __parse_extra(self, extra): return return_data - def __extract_range(self, val, field): + @staticmethod + def __extract_range(val, field): drange = val.split('-') # malformed range if len(drange) > 2: @@ -240,5 +285,31 @@ def ___is_ip(ip): try: IP(ip) return True - except Exception: + except ValueError: return False + + @staticmethod + def __is_fingerprint(s): + try: + # if it's base64 then it's an encoded fingerprint + base64.b64decode(s + "==") + + # or it can be an hex string + int(s, 16) + + return True + except (TypeError, ValueError): + return False + + @staticmethod + def __transform_fingerprint(fingerprint): + + # if it's already hex we don't need to transform it + try: + int(fingerprint, 16) + return fingerprint + except ValueError: + pass + + # we assume that 'fingerprint' is a valid base64 string + return base64.b64decode(fingerprint + "==") diff --git a/onionoo-ng.py b/onionoo-ng.py index d9a19fa..7d2b812 100644 --- a/onionoo-ng.py +++ b/onionoo-ng.py @@ -3,23 +3,25 @@ import settings as ts from engines.es import ES from tornado import gen -#from guppy import hpy -#from timer import Timer -#from pympler import tracker + + +# from guppy import hpy +# from timer import Timer +# from pympler import tracker # with Timer() as t: # blablabal # print "=> elapsed time: %s s" % t.secs -#t = tracker.SummaryTracker() +# t = tracker.SummaryTracker() -#h = hpy() -#print h.heap() +# h = hpy() +# print h.heap() class BaseHandler(RequestHandler): - #def write_error(self, status_code, **kwargs): + # def write_error(self, status_code, **kwargs): # self.set_header('Content-Type', 'text/json') - # self.finish({'code': ts.STATUS_ERROR, 'message': self._reason, 'version': ts.VERSION}) + # self.finish({'code': ts.STATUS_ERROR, 'message': self._reason, 'version': ts.PROTOCOL_VERSION}) pass @@ -28,9 +30,9 @@ class APIError(HTTPError): class APIBaseHandler(BaseHandler): - #def write_error(self, status_code, **kwargs): + # def write_error(self, status_code, **kwargs): # self.set_header('Content-Type', 'text/json') - # self.finish({'code': ts.STATUS_ERROR, 'message': self._reason, 'version': ts.VERSION}) + # self.finish({'code': ts.STATUS_ERROR, 'message': self._reason, 'version': ts.PROTOCOL_VERSION}) pass @@ -46,7 +48,7 @@ def get(self): query_extra = dict() es_index = None msg = { - 'version': ts.VERSION, + 'version': ts.PROTOCOL_VERSION, 'relays': [], 'bridges': [], 'relays_published': None, @@ -58,7 +60,8 @@ def get(self): query_split = self.request.query.lower().split('&') for p in query_split: p_split = p.split('=') - if len(p_split) == 1 or (p_split[0] not in ts.OOO_QUERYPARAMS and p_split[0] not in ts.OOO_QUERYPARAMS_EXTRAS): + if len(p_split) == 1 or ( + p_split[0] not in ts.OOO_QUERYPARAMS and p_split[0] not in ts.OOO_QUERYPARAMS_EXTRAS): # raise APIError(reason="invalid query parameters") raise HTTPError(status_code=400) @@ -73,11 +76,14 @@ def get(self): if es_index != "relay" and es_index != "bridge": raise HTTPError(status_code=400) # little trick to be consistent with the old protocol - msg[es_index+"s"] = yield self.application.es_instance.search(es_index, query_params, query_extra) + msg[es_index + "s"] = yield self.application.es_instance.search(es_index, query_params, query_extra) else: msg['relays'] = yield self.application.es_instance.search("relay", query_params, query_extra) msg['bridges'] = yield self.application.es_instance.search("bridge", query_params, query_extra) + msg['relays_published'] = yield self.application.es_instance.get_last_node("relay") + msg['bridges_published'] = yield self.application.es_instance.get_last_node("bridge") + self.write(msg) self.finish() @@ -104,7 +110,7 @@ def __init__(self): try: IOLoop.current().run_sync(self.es_instance.check_connection) - except Exception: + except EnvironmentError: print "ERROR: ElasticSearch not running...quitting" IOLoop.current().stop() exit() @@ -114,7 +120,7 @@ def __init__(self): try: app = App() app.listen(ts.LISTENING_PORT) - print "Onionoo-ng listening on {0}".format(ts.LISTENING_PORT) + print "Onionoo-ng ({0}) listening on {1}".format(ts.VERSION, ts.LISTENING_PORT) IOLoop.current().start() except KeyboardInterrupt: IOLoop.current().stop() diff --git a/settings.py b/settings.py index 005bb83..fd27376 100644 --- a/settings.py +++ b/settings.py @@ -7,7 +7,9 @@ ES_RESULT_SIZE = 10000 -VERSION = 3.1 +PROTOCOL_VERSION = 3.1 + +VERSION = 1.1 OOO_QUERYPARAMS = [ 'type', @@ -33,4 +35,4 @@ DB_NAME = "onionoo-ng" COLL_RELAYS = "relay" -COLL_BRIDGES = "bridge" \ No newline at end of file +COLL_BRIDGES = "bridge"