diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst index 2681e2756..7e5286277 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.rst @@ -19,7 +19,6 @@ Python3 ElectrumX uses asyncio. Python version >= 3.8 is **required**. `aiohttp`_ Python library for asynchronous HTTP. Version >= 2.0 required. -`pylru`_ Python LRU cache package. DB Engine A database engine package is required; two are supported (see `Database Engine`_ below). ================ ======================== @@ -443,7 +442,6 @@ You can then set the port as follows and advertise the service externally on the .. _`daemontools`: http://cr.yp.to/daemontools.html .. _`runit`: http://smarden.org/runit/index.html .. _`aiohttp`: https://pypi.python.org/pypi/aiohttp -.. _`pylru`: https://pypi.python.org/pypi/pylru .. _`dash_hash`: https://pypi.python.org/pypi/dash_hash .. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/spesmilo/electrumx/blob/master/contrib/raspberrypi3/install_electrumx.sh .. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/spesmilo/electrumx/blob/master/contrib/raspberrypi3/run_electrumx.sh diff --git a/electrumx/lib/lrucache.py b/electrumx/lib/lrucache.py new file mode 100644 index 000000000..f6c09565a --- /dev/null +++ b/electrumx/lib/lrucache.py @@ -0,0 +1,181 @@ +# The MIT License (MIT) +# +# Copyright (c) 2014-2022 Thomas Kemmer +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# ----- +# +# This is a stripped down LRU-cache from the "cachetools" library. +# https://github.com/tkem/cachetools/blob/d991ac71b4eb6394be5ec572b835434081393215/src/cachetools/__init__.py + +import collections +import collections.abc + + +class _DefaultSize: + + __slots__ = () + + def __getitem__(self, _): + return 1 + + def __setitem__(self, _, value): + assert value == 1 + + def pop(self, _): + return 1 + + +class Cache(collections.abc.MutableMapping): + """Mutable mapping to serve as a simple cache or cache base class.""" + + __marker = object() + + __size = _DefaultSize() + + def __init__(self, maxsize, getsizeof=None): + if getsizeof: + self.getsizeof = getsizeof + if self.getsizeof is not Cache.getsizeof: + self.__size = dict() + self.__data = dict() + self.__currsize = 0 + self.__maxsize = maxsize + + def __repr__(self): + return "%s(%s, maxsize=%r, currsize=%r)" % ( + self.__class__.__name__, + repr(self.__data), + self.__maxsize, + self.__currsize, + ) + + def __getitem__(self, key): + try: + return self.__data[key] + except KeyError: + return self.__missing__(key) + + def __setitem__(self, key, value): + maxsize = self.__maxsize + size = self.getsizeof(value) + if size > maxsize: + raise ValueError("value too large") + if key not in self.__data or self.__size[key] < size: + while self.__currsize + size > maxsize: + self.popitem() + if key in self.__data: + diffsize = size - self.__size[key] + else: + diffsize = size + self.__data[key] = value + self.__size[key] = size + self.__currsize += diffsize + + def __delitem__(self, key): + size = self.__size.pop(key) + del self.__data[key] + self.__currsize -= size + + def __contains__(self, key): + return key in self.__data + + def __missing__(self, key): + raise KeyError(key) + + def __iter__(self): + return iter(self.__data) + + def __len__(self): + return len(self.__data) + + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default + + def pop(self, key, default=__marker): + if key in self: + value = self[key] + del self[key] + elif default is self.__marker: + raise KeyError(key) + else: + value = default + return value + + def setdefault(self, key, default=None): + if key in self: + value = self[key] + else: + self[key] = value = default + return value + + @property + def maxsize(self): + """The maximum size of the cache.""" + return self.__maxsize + + @property + def currsize(self): + """The current size of the cache.""" + return self.__currsize + + @staticmethod + def getsizeof(value): + """Return the size of a cache element's value.""" + return 1 + + +class LRUCache(Cache): + """Least Recently Used (LRU) cache implementation.""" + + def __init__(self, maxsize, getsizeof=None): + Cache.__init__(self, maxsize, getsizeof) + self.__order = collections.OrderedDict() + + def __getitem__(self, key, cache_getitem=Cache.__getitem__): + value = cache_getitem(self, key) + if key in self: # __missing__ may not store item + self.__update(key) + return value + + def __setitem__(self, key, value, cache_setitem=Cache.__setitem__): + cache_setitem(self, key, value) + self.__update(key) + + def __delitem__(self, key, cache_delitem=Cache.__delitem__): + cache_delitem(self, key) + del self.__order[key] + + def popitem(self): + """Remove and return the `(key, value)` pair least recently used.""" + try: + key = next(iter(self.__order)) + except StopIteration: + raise KeyError("%s is empty" % type(self).__name__) from None + else: + return (key, self.pop(key)) + + def __update(self, key): + try: + self.__order.move_to_end(key) + except KeyError: + self.__order[key] = None diff --git a/electrumx/server/session.py b/electrumx/server/session.py index ce782edbd..75f83f3ee 100644 --- a/electrumx/server/session.py +++ b/electrumx/server/session.py @@ -22,7 +22,6 @@ import asyncio import attr -import pylru from aiorpcx import (Event, JSONRPCAutoDetect, JSONRPCConnection, ReplyAndDisconnect, Request, RPCError, RPCSession, handler_invocation, serve_rs, serve_ws, sleep, @@ -30,6 +29,7 @@ import electrumx import electrumx.lib.util as util +from electrumx.lib.lrucache import LRUCache from electrumx.lib.util import OldTaskGroup from electrumx.lib.hash import (HASHX_LEN, Base58Error, hash_to_hex_str, hex_str_to_hash, sha256) @@ -146,17 +146,17 @@ def __init__( self.start_time = time.time() self._method_counts = defaultdict(int) self._reorg_count = 0 - self._history_cache = pylru.lrucache(1000) + self._history_cache = LRUCache(maxsize=1000) self._history_lookups = 0 self._history_hits = 0 - self._tx_hashes_cache = pylru.lrucache(1000) + self._tx_hashes_cache = LRUCache(maxsize=1000) self._tx_hashes_lookups = 0 self._tx_hashes_hits = 0 # Really a MerkleCache cache - self._merkle_cache = pylru.lrucache(1000) + self._merkle_cache = LRUCache(maxsize=1000) self._merkle_lookups = 0 self._merkle_hits = 0 - self.estimatefee_cache = pylru.lrucache(1000) + self.estimatefee_cache = LRUCache(maxsize=1000) self.notified_height = None self.hsub_results = None self._task_group = OldTaskGroup() diff --git a/requirements.txt b/requirements.txt index 958e29af2..c221213ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ aiorpcX[ws]>=0.22.0,<0.23 attrs plyvel -pylru aiohttp>=3.3,<4