Merge branch 'release-0.5.1'

docs/RELEASE-NOTES

@@ -1,3 +1,43 @@
+version 0.5.1
+-------------
+
+- 0.5 changed some cache defaults, only partially intentionally.  For
+  some users, including me, the result was a regression (a 15hr HDD
+  sync became a 20hr sync).  Another user reported their fastest sync
+  yet (sub 10hr SSD sync).  What changed was memory accounting - all
+  releases until 0.5 were not properly accounting for memory usage of
+  unflushed transaction hashes.  In 0.5 they were accounted for in the
+  UTXO cache, which resulted in much earlier flushes.  0.5.1 flushes
+  the hashes at the same time as history so I now account for it
+  towards the history cache limit.  To get a reasonable comparison
+  with prior releases your HIST_MB environment variable should be
+  bumped by about 15% from 0.4 and earlier values.  This will not
+  result in greater memory consumption - the additional memory
+  consumption was being ignored before but is now being included.
+- 0.5.1 is the first release where Electrum client requests are queued
+  on a per-session basis.  Previously they were in a global queue.
+  This is the beginning of ensuring that expensive / DOS requests
+  mostly affect that user's session and not those of other users.  The
+  goal is that each session's requests run asynchronously parallel to
+  every other sessions's requests.  The missing part of the puzzle is
+  that Python's asyncio is co-operative, however at the moment
+  ElectrumX does not yield during expensive requests.  I intend that a
+  near upcoming release will ensure expensive requests yield the CPU
+  at regular fine-grained intervals.  The intended result is that, to
+  some extent, expensive requests mainly delay that and later requests
+  from the same session, and have minimal impact on the legitimate
+  requests of other sessions.  The extent to which this goal is
+  achieved will only be verifiable in practice.
+- more robust tracking and handling of asynchronous tasks.  I hope
+  this will reduce asyncio's logging messages, some of which I'm
+  becoming increasingly convinced I have no control over.  In
+  particular I learned earlier releases were unintentionally limiting
+  the universe of acceptable SSL protocols, and so I made them the
+  default that had been intended.
+- I added logging of expensive tasks, though I don't expect much real
+  information from this
+- various RPC improvements
+
 version 0.5
 -----------
 

electrumx_rpc.py

@@ -13,7 +13,6 @@
 import argparse
 import asyncio
 import json
-import pprint
 from functools import partial
 from os import environ
 
@@ -36,38 +35,39 @@ def send(self, method, params):
         data = json.dumps(payload) + '\n'
         self.transport.write(data.encode())
 
+    def print_sessions(self, result):
+        def data_fmt(count, size):
+            return '{:,d}/{:,d}KB'.format(count, size // 1024)
+        def time_fmt(t):
+            t = int(t)
+            return ('{:3d}:{:02d}:{:02d}'
+                    .format(t // 3600, (t % 3600) // 60, t % 60))
+
+        fmt = ('{:<4} {:>23} {:>15} {:>5} '
+               '{:>7} {:>7} {:>7} {:>7} {:>5} {:>9}')
+        print(fmt.format('Type', 'Peer', 'Client', 'Subs',
+                         'Recv #', 'Recv KB', 'Sent #', 'Sent KB',
+                         'Errs', 'Time'))
+        for (kind, peer, subs, client, recv_count, recv_size,
+             send_count, send_size, error_count, time) in result:
+            print(fmt.format(kind, peer, client, '{:,d}'.format(subs),
+                             '{:,d}'.format(recv_count),
+                             '{:,d}'.format(recv_size // 1024),
+                             '{:,d}'.format(send_count),
+                             '{:,d}'.format(send_size // 1024),
+                             '{:,d}'.format(error_count),
+                             time_fmt(time)))
+
     def data_received(self, data):
         payload = json.loads(data.decode())
         self.transport.close()
         result = payload['result']
         error = payload['error']
-        if error:
-            print("ERROR: {}".format(error))
+        if not error and self.method == 'sessions':
+            self.print_sessions(result)
         else:
-            def data_fmt(count, size):
-                return '{:,d}/{:,d}KB'.format(count, size // 1024)
-            def time_fmt(t):
-                t = int(t)
-                return ('{:3d}:{:02d}:{:02d}'
-                        .format(t // 3600, (t % 3600) // 60, t % 60))
-
-            if self.method == 'sessions':
-                fmt = ('{:<4} {:>23} {:>15} {:>5} '
-                       '{:>7} {:>7} {:>7} {:>7} {:>5} {:>9}')
-                print(fmt.format('Type', 'Peer', 'Client', 'Subs',
-                                 'Snt #', 'Snt MB', 'Rcv #', 'Rcv MB',
-                                 'Errs', 'Time'))
-                for (kind, peer, subs, client, recv_count, recv_size,
-                     send_count, send_size, error_count, time) in result:
-                    print(fmt.format(kind, peer, client, '{:,d}'.format(subs),
-                                     '{:,d}'.format(recv_count),
-                                     '{:,.1f}'.format(recv_size / 1048576),
-                                     '{:,d}'.format(send_count),
-                                     '{:,.1f}'.format(send_size / 1048576),
-                                     '{:,d}'.format(error_count),
-                                     time_fmt(time)))
-            else:
-                pprint.pprint(result, indent=4)
+            value = {'error': error} if error else result
+            print(json.dumps(value, indent=4, sort_keys=True))
 
 def main():
     '''Send the RPC command to the server and print the result.'''

electrumx_server.py

@@ -32,23 +32,18 @@ def main_loop():
     def on_signal(signame):
         '''Call on receipt of a signal to cleanly shutdown.'''
         logging.warning('received {} signal, shutting down'.format(signame))
-        for task in asyncio.Task.all_tasks():
-            task.cancel()
+        future.cancel()
+
+    server = BlockServer(Env())
+    future = asyncio.ensure_future(server.main_loop())
 
     # Install signal handlers
     for signame in ('SIGINT', 'SIGTERM'):
         loop.add_signal_handler(getattr(signal, signame),
                                 partial(on_signal, signame))
 
-    server = BlockServer(Env())
-    future = server.start()
-    try:
-        loop.run_until_complete(future)
-    except asyncio.CancelledError:
-        pass
-    finally:
-        server.stop()
-        loop.close()
+    loop.run_until_complete(future)
+    loop.close()
 
 
 def main():

lib/jsonrpc.py

@@ -63,7 +63,7 @@ class RPCError(Exception):
         def __init__(self, msg, code=-1, **kw_args):
             super().__init__(**kw_args)
             self.msg = msg
-            self.code
+            self.code = code
 
 
     def __init__(self):
@@ -172,7 +172,7 @@ async def handle_json_request(self, request):
         the connection is closing.
         '''
         if isinstance(request, list):
-            payload = self.batch_request_payload(request)
+            payload = await self.batch_request_payload(request)
         else:
             payload = await self.single_request_payload(request)
 
@@ -231,7 +231,7 @@ async def method_result(self, method, params):
 
         handler = self.method_handler(method)
         if not handler:
-            raise self.RPCError('unknown method: {}'.format(method),
+            raise self.RPCError('unknown method: "{}"'.format(method),
                                 self.METHOD_NOT_FOUND)
 
         return await handler(params)