version 1.1

CHANGELOG

@@ -0,0 +1,11 @@
+ONIONOO-NG CHANGELOG
+
+
+VERSION 1.1 (05/08/2016)
+- added 'relays_published' and 'bridges_published' values
+- improved support for 'search' parameter
+- fixed a bug about 'offset' parameter (see issue #1)
+- minor style fixes
+
+VERSION 1.0 (03/08/2016)
+- first release: almost totally backward compatible with current Onionoo's protocol

engines/es.py

@@ -4,8 +4,9 @@
 from tornado import gen, escape
 from tornado.web import HTTPError
 from datetime import date, timedelta, datetime
-import urllib
 from IPy import IP
+import urllib
+import base64
 
 AsyncHTTPClient.configure(None, max_clients=settings.ASYNC_ES_MAX_CLIENT)
 
@@ -27,7 +28,7 @@ def check_connection(self):
         response = yield self.es.get_by_path("/")
 
         if response.code != 200 or not response.body:
-            raise Exception
+            raise EnvironmentError
 
     @gen.coroutine
     def search(self, mapping, query, extras):
@@ -98,6 +99,7 @@ def search(self, mapping, query, extras):
 
         result = yield self.es.search(index=self.index, type=self.type_mapping, source=body_query,
                                       size=extra_parameters['limit'])
+
         if result.code != 200 or not result.body:
             raise HTTPError(status_code=result.code)
 
@@ -111,6 +113,35 @@ def search(self, mapping, query, extras):
 
         raise gen.Return(return_data)
 
+    @gen.coroutine
+    def get_last_node(self, mapping):
+
+        body_query = {
+            "sort": {
+                "last_seen": {
+                    "unmapped_type": "string", "order": "desc"
+                }
+            },
+            "query": {
+                "match_all": {}
+            }, "from": 0, "_source": ["*"], "size": 1
+        }
+
+        result = yield self.es.search(index=self.index, type=mapping, source=body_query)
+
+        if result.code != 200 or not result.body:
+            raise HTTPError(status_code=result.code)
+
+        result = escape.json_decode(result.body)
+
+        if 'hits' in result and 'hits' in result['hits']:
+            last_seen = result['hits']['hits'][0]['_source']['last_seen']
+        else:
+            # generic error related to ES - shouldn't happen...
+            raise Exception(result.error)
+
+        raise gen.Return(last_seen)
+
     def __parse_query(self, q):
         if 'as' in q:
             q['as_number'] = q.pop('as').split(',')[0]
@@ -154,8 +185,14 @@ def __parse_query(self, q):
                 if len(tokenz) == 1:
                     if self.___is_ip(tokenz[0]):
                         new_se.append("(or_address:{0} OR dir_address:{0})".format(tokenz[0]))
+                    elif self.__is_fingerprint(tokenz[0]):
+                        fingerprint = self.__transform_fingerprint(tokenz[0])
+                        if fingerprint[0] == "$":
+                            new_se.append("(fingerprint:{0}* OR hashed_fingerprint:{0}*)".format(fingerprint[1:]))
+                        else:
+                            new_se.append("(fingerprint:*{0}* OR hashed_fingerprint:*{0}*)".format(fingerprint))
                     else:
-                        new_se.append("(nickname:*{0}* OR fingerprint:*{0}* OR hashed_fingerprint:*{0}*)".format(tokenz[0]))
+                        new_se.append("nickname:*{0}*".format(tokenz[0]))
                 else:
                     if tokenz[0] in forbidden_keys:
                         raise HTTPError(status_code=400)
@@ -166,17 +203,24 @@ def __parse_query(self, q):
 
         return q
 
-    def __parse_extra(self, extra):
+    @staticmethod
+    def __parse_extra(extra):
         return_data = {'limit': settings.ES_RESULT_SIZE, 'offset': 0, 'fields': ["*"],
                        'sort': {'field': None, 'order': 'asc', 'type': 'string'}}
 
-        if 'limit' in extra and int(extra['limit']) > 0:
-            return_data['limit'] = extra['limit']
-        elif 'limit' in extra and int(extra['limit']) <= 0:
-            return_data['limit'] = 0
+        if 'limit' in extra:
+            # 'limit' cannot go beyond the settings.ES_RESULT_SIZE value
+            if 0 < int(extra['limit']) < settings.ES_RESULT_SIZE:
+                return_data['limit'] = int(extra['limit'])
+            elif int(extra['limit']) <= 0:
+                return_data['limit'] = 0
 
         if 'offset' in extra and int(extra['offset']) > 0:
-            return_data['offset'] = extra['offset']
+            return_data['offset'] = int(extra['offset'])
+
+        # offset + limit must be <= settings.ES_RESULT_SIZE
+        if (return_data['offset'] + return_data['limit']) > settings.ES_RESULT_SIZE:
+            return_data['limit'] -= return_data['offset']
 
         if 'fields' in extra:
             return_data['fields'] = extra['fields'].split(',')
@@ -191,7 +235,8 @@ def __parse_extra(self, extra):
 
         return return_data
 
-    def __extract_range(self, val, field):
+    @staticmethod
+    def __extract_range(val, field):
         drange = val.split('-')
         # malformed range
         if len(drange) > 2:
@@ -240,5 +285,31 @@ def ___is_ip(ip):
         try:
             IP(ip)
             return True
-        except Exception:
+        except ValueError:
             return False
+
+    @staticmethod
+    def __is_fingerprint(s):
+        try:
+            # if it's base64 then it's an encoded fingerprint
+            base64.b64decode(s + "==")
+
+            # or it can be an hex string
+            int(s, 16)
+
+            return True
+        except (TypeError, ValueError):
+            return False
+
+    @staticmethod
+    def __transform_fingerprint(fingerprint):
+
+        # if it's already hex we don't need to transform it
+        try:
+            int(fingerprint, 16)
+            return fingerprint
+        except ValueError:
+            pass
+
+        # we assume that 'fingerprint' is a valid base64 string
+        return base64.b64decode(fingerprint + "==")

onionoo-ng.py

@@ -3,23 +3,25 @@
 import settings as ts
 from engines.es import ES
 from tornado import gen
-#from guppy import hpy
-#from timer import Timer
-#from pympler import tracker
+
+
+# from guppy import hpy
+# from timer import Timer
+# from pympler import tracker
 
 # with Timer() as t:
 #   blablabal
 # print "=> elapsed time: %s s" % t.secs
 
-#t = tracker.SummaryTracker()
+# t = tracker.SummaryTracker()
 
-#h = hpy()
-#print h.heap()
+# h = hpy()
+# print h.heap()
 
 class BaseHandler(RequestHandler):
-    #def write_error(self, status_code, **kwargs):
+    # def write_error(self, status_code, **kwargs):
     #    self.set_header('Content-Type', 'text/json')
-    #    self.finish({'code': ts.STATUS_ERROR, 'message': self._reason, 'version': ts.VERSION})
+    #    self.finish({'code': ts.STATUS_ERROR, 'message': self._reason, 'version': ts.PROTOCOL_VERSION})
     pass
 
 
@@ -28,9 +30,9 @@ class APIError(HTTPError):
 
 
 class APIBaseHandler(BaseHandler):
-    #def write_error(self, status_code, **kwargs):
+    # def write_error(self, status_code, **kwargs):
     #    self.set_header('Content-Type', 'text/json')
-    #    self.finish({'code': ts.STATUS_ERROR, 'message': self._reason, 'version': ts.VERSION})
+    #    self.finish({'code': ts.STATUS_ERROR, 'message': self._reason, 'version': ts.PROTOCOL_VERSION})
     pass
 
 
@@ -46,7 +48,7 @@ def get(self):
         query_extra = dict()
         es_index = None
         msg = {
-            'version': ts.VERSION,
+            'version': ts.PROTOCOL_VERSION,
             'relays': [],
             'bridges': [],
             'relays_published': None,
@@ -58,7 +60,8 @@ def get(self):
             query_split = self.request.query.lower().split('&')
             for p in query_split:
                 p_split = p.split('=')
-                if len(p_split) == 1 or (p_split[0] not in ts.OOO_QUERYPARAMS and p_split[0] not in ts.OOO_QUERYPARAMS_EXTRAS):
+                if len(p_split) == 1 or (
+                        p_split[0] not in ts.OOO_QUERYPARAMS and p_split[0] not in ts.OOO_QUERYPARAMS_EXTRAS):
                     # raise APIError(reason="invalid query parameters")
                     raise HTTPError(status_code=400)
 
@@ -73,11 +76,14 @@ def get(self):
             if es_index != "relay" and es_index != "bridge":
                 raise HTTPError(status_code=400)
             # little trick to be consistent with the old protocol
-            msg[es_index+"s"] = yield self.application.es_instance.search(es_index, query_params, query_extra)
+            msg[es_index + "s"] = yield self.application.es_instance.search(es_index, query_params, query_extra)
         else:
             msg['relays'] = yield self.application.es_instance.search("relay", query_params, query_extra)
             msg['bridges'] = yield self.application.es_instance.search("bridge", query_params, query_extra)
 
+        msg['relays_published'] = yield self.application.es_instance.get_last_node("relay")
+        msg['bridges_published'] = yield self.application.es_instance.get_last_node("bridge")
+
         self.write(msg)
         self.finish()
 
@@ -104,7 +110,7 @@ def __init__(self):
 
         try:
             IOLoop.current().run_sync(self.es_instance.check_connection)
-        except Exception:
+        except EnvironmentError:
             print "ERROR: ElasticSearch not running...quitting"
             IOLoop.current().stop()
             exit()
@@ -114,7 +120,7 @@ def __init__(self):
     try:
         app = App()
         app.listen(ts.LISTENING_PORT)
-        print "Onionoo-ng listening on {0}".format(ts.LISTENING_PORT)
+        print "Onionoo-ng ({0}) listening on {1}".format(ts.VERSION, ts.LISTENING_PORT)
         IOLoop.current().start()
     except KeyboardInterrupt:
         IOLoop.current().stop()

settings.py

@@ -7,7 +7,9 @@
 
 ES_RESULT_SIZE = 10000
 
-VERSION = 3.1
+PROTOCOL_VERSION = 3.1
+
+VERSION = 1.1
 
 OOO_QUERYPARAMS = [
     'type',
@@ -33,4 +35,4 @@
 
 DB_NAME = "onionoo-ng"
 COLL_RELAYS = "relay"
-COLL_BRIDGES = "bridge"
+COLL_BRIDGES = "bridge"