From d85b8f42038a3f31f5226311f31ca1b1e81ff99a Mon Sep 17 00:00:00 2001 From: Veselin Penev Date: Sat, 2 Mar 2019 20:32:53 +0100 Subject: [PATCH 1/8] passing text arguments to child_process() --- lib/misc.py | 2 +- storage/backup_tar.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/misc.py b/lib/misc.py index b71b8bf34..55d433b7e 100644 --- a/lib/misc.py +++ b/lib/misc.py @@ -1240,7 +1240,7 @@ def DoRestart(param='', detach=False, std_out='/dev/null', std_err='/dev/null'): cmdargs.remove('daemon') if detach: from system import child_process - cmdargs = [strng.to_bin(a) for a in cmdargs] + cmdargs = [strng.to_text(a) for a in cmdargs] lg.out(0, 'run : %r' % cmdargs) return child_process.detach(cmdargs) lg.out(2, "misc.DoRestart cmdargs=" + str(cmdargs)) diff --git a/storage/backup_tar.py b/storage/backup_tar.py index 5f1987785..6e4410085 100644 --- a/storage/backup_tar.py +++ b/storage/backup_tar.py @@ -100,9 +100,7 @@ def backuptardir(directorypath, arcname=None, recursive_subfolders=True, compres if not os.path.isfile(commandpath): lg.out(1, 'backup_tar.backuptar ERROR %s not found' % commandpath) return None - # lg.out(14, "backup_tar.backuptar going to execute %s" % str(cmdargs)) - # p = child_process.run('bppipe', cmdargs[2:]) - cmdargs = [strng.to_bin(a) for a in cmdargs] + cmdargs = [strng.to_text(a) for a in cmdargs] p = child_process.pipe(cmdargs) return p @@ -135,7 +133,7 @@ def backuptarfile(filepath, arcname=None, compress=None): return None # lg.out(12, "backup_tar.backuptarfile going to execute %s" % str(cmdargs)) # p = run(cmdargs) - cmdargs = [strng.to_bin(a) for a in cmdargs] + cmdargs = [strng.to_text(a) for a in cmdargs] p = child_process.pipe(cmdargs) return p @@ -162,7 +160,7 @@ def extracttar(tarfile, outdir): lg.out(1, 'backup_tar.extracttar ERROR %s is not found' % commandpath) return None # p = run(cmdargs) - cmdargs = [strng.to_bin(a) for a in cmdargs] + cmdargs = [strng.to_text(a) for a in cmdargs] p = child_process.pipe(cmdargs) return p From 2fbfd6e63053719d0e181e04940326eb9c74b9ce Mon Sep 17 00:00:00 2001 From: Veselin Penev Date: Sun, 3 Mar 2019 16:55:03 +0100 Subject: [PATCH 2/8] try to solve multiprocessing issue on Windows, replaced multiprocessing with parallelp --- raid/raid_worker.py | 13 +++++++++---- raid/worker.py | 29 ++++++++++------------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/raid/raid_worker.py b/raid/raid_worker.py index ec7d73616..ba78b1a0c 100644 --- a/raid/raid_worker.py +++ b/raid/raid_worker.py @@ -332,8 +332,7 @@ def doStartTask(self, *args, **kwargs): global _MODULES if len(self.activetasks) >= self.processor.ncpus: - if _Debug: - lg.out(_DebugLevel, 'raid_worker.doStartTask SKIP active=%d cpus=%d' % ( + lg.warn('SKIP active=%d cpus=%d' % ( len(self.activetasks), self.processor.ncpus)) return @@ -348,6 +347,7 @@ def doStartTask(self, *args, **kwargs): func, params, callback=lambda result: self._job_done(task_id, cmd, params, result), + error_callback=lambda err: self._job_failed(task_id, cmd, params, err), ) self.activetasks[task_id] = (proc, cmd, params) @@ -399,10 +399,15 @@ def doDestroyMe(self, *args, **kwargs): def _job_done(self, task_id, cmd, params, result): if _Debug: - lg.out(_DebugLevel, 'raid_worker._job_done %r : %r active:%r' % ( - task_id, result, list(self.activetasks.keys()))) + lg.out(_DebugLevel, 'raid_worker._job_done %r : %r active:%r cmd=%r params=%r' % ( + task_id, result, list(self.activetasks.keys()), cmd, params)) self.automat('task-done', (task_id, cmd, params, result)) + def _job_failed(self, task_id, cmd, params, err): + lg.err('task %r FAILED : %r active:%r cmd=%r params=%r' % ( + task_id, err, list(self.activetasks.keys()), cmd, params)) + self.automat('shutdown') + def _kill_processor(self): if self.processor: self.processor.terminate() diff --git a/raid/worker.py b/raid/worker.py index 565e461d8..c6b105914 100644 --- a/raid/worker.py +++ b/raid/worker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python + #!/usr/bin/env python # rebuild.py # # Copyright (C) 2008-2018 Stanislav Evseev, Veselin Penev https://bitdust.io @@ -76,24 +76,6 @@ def __call__(self, *args, **kwargs): return res -# def _initializer_worker(queue_cancel): -# print(os.getpid(), '_initializer_worker', queue_cancel) -# -# def func(): -# while True: -# value = pipea.recv() -# print(os.getpid(), 'value', value) -# # tid = joinable_cancel_task.get() -# process = multiprocessing.current_process() -# print('kill process %s. tid - %s' % (process.pid, value)) -# os.kill(process.pid, signal.SIGTERM) -# time.sleep(1) -# -# thread = Thread(target=func) -# thread.daemon = True -# thread.start() - - def func_thread(tasks, pool): while True: try: @@ -145,6 +127,14 @@ class Manager(object): def __init__(self, ncpus): self._ncpus = ncpus + if six.PY34: + try: + multiprocessing.set_start_method('spawn') + except RuntimeError: + pass + + multiprocessing.util.log_to_stderr(multiprocessing.util.SUBDEBUG) + from system import bpio if bpio.Windows(): from system import deploy @@ -154,6 +144,7 @@ def __init__(self, ncpus): multiprocessing.set_executable(venv_python_path) self.processor = multiprocessing.Pool(ncpus) + #: implement queue per Manager instance # self.queue = multiprocessing.Queue() From 779bac5815c054575d825ca93a203e56413e780a Mon Sep 17 00:00:00 2001 From: Veselin Penev Date: Fri, 8 Mar 2019 14:25:31 +0100 Subject: [PATCH 3/8] removed pybc blockchain files, we are moving towards Bismuth project --- blockchain/pybc/AuthenticatedDictionary.py | 1607 -------------- blockchain/pybc/Block.py | 353 --- blockchain/pybc/Blockchain.py | 1963 ----------------- blockchain/pybc/BlockchainProtocol.py | 512 ----- blockchain/pybc/ClientFactory.py | 53 - blockchain/pybc/Peer.py | 620 ------ blockchain/pybc/PowAlgorithm.py | 134 -- blockchain/pybc/ServerFactory.py | 41 - blockchain/pybc/State.py | 148 -- blockchain/pybc/StateComponent.py | 70 - blockchain/pybc/StateMachine.py | 294 --- blockchain/pybc/TransactionalBlockchain.py | 64 - blockchain/pybc/__init__.py | 16 - blockchain/pybc/block_explorer.py | 184 -- blockchain/pybc/coin.py | 1548 ------------- blockchain/pybc/emergency_crypto_munitions.py | 1675 -------------- blockchain/pybc/json_coin.py | 1258 ----------- blockchain/pybc/science.py | 177 -- blockchain/pybc/sqliteshelf.py | 371 ---- blockchain/pybc/token.py | 356 --- blockchain/pybc/transactions.py | 80 - blockchain/pybc/util.py | 106 - blockchain/pybc/wallet.py | 430 ---- blockchain/pybc_service.py | 525 ----- blockchain/trinity/deploy_linux.sh | 114 - 25 files changed, 12699 deletions(-) delete mode 100644 blockchain/pybc/AuthenticatedDictionary.py delete mode 100644 blockchain/pybc/Block.py delete mode 100644 blockchain/pybc/Blockchain.py delete mode 100644 blockchain/pybc/BlockchainProtocol.py delete mode 100644 blockchain/pybc/ClientFactory.py delete mode 100644 blockchain/pybc/Peer.py delete mode 100644 blockchain/pybc/PowAlgorithm.py delete mode 100644 blockchain/pybc/ServerFactory.py delete mode 100644 blockchain/pybc/State.py delete mode 100644 blockchain/pybc/StateComponent.py delete mode 100644 blockchain/pybc/StateMachine.py delete mode 100644 blockchain/pybc/TransactionalBlockchain.py delete mode 100644 blockchain/pybc/__init__.py delete mode 100644 blockchain/pybc/block_explorer.py delete mode 100644 blockchain/pybc/coin.py delete mode 100644 blockchain/pybc/emergency_crypto_munitions.py delete mode 100644 blockchain/pybc/json_coin.py delete mode 100644 blockchain/pybc/science.py delete mode 100644 blockchain/pybc/sqliteshelf.py delete mode 100644 blockchain/pybc/token.py delete mode 100644 blockchain/pybc/transactions.py delete mode 100644 blockchain/pybc/util.py delete mode 100644 blockchain/pybc/wallet.py delete mode 100644 blockchain/pybc_service.py delete mode 100755 blockchain/trinity/deploy_linux.sh diff --git a/blockchain/pybc/AuthenticatedDictionary.py b/blockchain/pybc/AuthenticatedDictionary.py deleted file mode 100644 index ac7bc992d..000000000 --- a/blockchain/pybc/AuthenticatedDictionary.py +++ /dev/null @@ -1,1607 +0,0 @@ -""" -AuthenticatedDictionary.py: Contains an authenticated dictionary data structure. -Supports O(log n) insert, find, and delete, and maintains a hash authenticating -the contents. - -The data structure is set-unique; if the same data is in it, it always produces -the same hash, no matter what order it was inserted in. - -The data structure is backed by a SQliteShelf database, and exposes a commit() -method that must be called when you want to commit your changes to disk. - -""" - -from __future__ import absolute_import -from __future__ import print_function -import hashlib -import collections -import struct -import logging - -from collections import MutableMapping -from .sqliteshelf import SQLiteShelf -from .StateComponent import StateComponent -from . import util -import six -from six.moves import range - -# How many children should each MerkleTrieNode be able to have? As many as -# there are hex digits. -ORDER = 16 - - -class MerkleTrieNode(object): - """ - An object that we use to represent a Merkle trie node. Gets pickled and - unpickled, and carries a list of child pointers, a key field, a value field, - and a hash field. - - Keys, values, and hashes must all be byte strings. - - Can't really do anything by itself, since it can't directly access its - children, just their pointer values. - - """ - - def __init__(self, children=None, key=None, value=None, hash=None): - """ - Make a new blank MerkleTrieNode with the given number of child pointer - storage locations. - - Once stored, MerkleTrieNodes should never be changed. - - """ - - # Don't store any children pointer locations until we need to. If we - # need children, this turns into a list of child pointers or Nones. - self.children = children - - # What is our key, if any? - self.key = key - - # What is our value, if any? - self.value = value - - # And our Merkle hash - self.hash = hash - - def copy(self): - """ - Return a deep copy of this MerkleTrieNode. Not in the sense that we - create new MerkleTrieNodes for its children, but in the sense that if we - update the resultant Python object in place it won't affect the - original. - - """ - - # Load the children - children = self.children - - if children is not None: - # It's a list of children and we need to make a copy of it rather - # than just referencing it - children = list(children) - - # Make a new MerkleTrieNode exactly like us. - return MerkleTrieNode(children, self.key, self.value, self.hash) - - def __repr__(self): - """ - Stringify this node for debugging. - - """ - - # Hold all the parts to merge. - parts = ["MerkleTrieNode("] - - if self.key is not None: - parts.append(self.key) - parts.append(" -> ") - if self.value is not None: - parts.append(self.value) - if self.children is not None: - for i, child in enumerate(self.children): - if child is not None: - parts.append("".format(i, child)) - if self.hash is not None: - if len(parts) > 1: - parts.append(", Hash:") - parts.append(util.bytes2string(self.hash)) - parts.append(")") - - return "".join(parts) - - -class AuthenticatedDictionaryStateComponent(StateComponent): - """ - A StateComponent for an AuthenticatedDictionary. Each StateComponent - contains a MerkleTrieNode turned into portable pointer-independent bytes - with node_to_bytes, so the StateComponents have the same hashes as the - MerkleTrieNodes they represent. - - Knows how its dependencies are encoded in the bytestring. - - """ - - def get_dependencies(self): - """ - Yield the Merkle hash of each dependency of this StateComponent. - - """ - - # Children are encoded as follows: - # First byte gives child count - # Then we have that many 65-byte records of child number and child hash. - - if len(self.data) == 0: - raise Exception("No data") - - child_count = struct.unpack(">B", self.data[0])[0] - - if child_count > 16: - # Don't have absurd numbers of children - raise Exception("Too many children: {}".format(child_count)) - - for i in range(child_count): - # Unpack the next 65-byte record - child_index, child_hash = struct.unpack_from(">B64s", self.data, - offset=1 + 65 * i) - - # Say the hash is a dependency. - yield child_hash - - def get_child_list(self): - """ - Return a list of child Merkle hashes, with None everywhere that there is - no child. - - """ - - # We can have up to 16 children - children = [None] * 16 - - # Children are encoded as follows: - # First byte gives child count - # Then we have that many 65-byte records of child number and child hash. - - if len(self.data) == 0: - raise Exception("No data") - - child_count = struct.unpack(">B", self.data[0])[0] - - if child_count > 16: - # Don't have absurd numbers of children - raise Exception("Too many children: {}".format(child_count)) - - for i in range(child_count): - # Unpack the next 65-byte record - child_index, child_hash = struct.unpack_from(">B64s", self.data, - offset=1 + 65 * i) - - # Record it in the list of child hashes at the appropriate index. - children[child_index] = child_hash - - return children - - def get_key(self): - """ - Return the key for thhis AuthenticatedDictionaryStateComponent, or None - if it doesn't carry one. - - """ - - if len(self.data) == 0: - raise Exception("No data") - - # After the children, we have key length (8 bytes), key, and value. - - # How many child records are there? - child_count = struct.unpack(">B", self.data[0])[0] - - # Skip to after the child data - offset = 1 + 65 * child_count - - if len(self.data) > offset: - # We actually do have a key - - # Unpack the key length - key_length = struct.unpack_from(">Q", self.data, offset=offset)[0] - - # Account for the 8 byte key length - offset += 8 - - # And the key itself - key = self.data[offset: offset + key_length] - - return key - - # No key length was given after the children - return None - - def get_value(self): - """ - Return the value for thhis AuthenticatedDictionaryStateComponent, or None - if it doesn't carry one. - - """ - - if len(self.data) == 0: - raise Exception("No data") - - # After the children, we have key length (8 bytes), key, and value. - - # How many child records are there? - child_count = struct.unpack(">B", self.data[0])[0] - - # Skip to after the child data - offset = 1 + 65 * child_count - - if len(self.data) > offset: - # We actually do have a key - - # Unpack the key length - key_length = struct.unpack_from(">Q", self.data, offset=offset)[0] - - # Advance to the start of thre data - offset += 8 + key_length - - # Get the data - data = self.data[offset:] - - return data - - # No key length was given after the children - return None - - -class AuthenticatedDictionary(object): - """ - An authenticated dictionary, based on a Merkle Trie, and stored on disk. - - Nodes are identified by pointers (really strings). - - The whole thing is backed by an SQLite database, but has additional - transaction support. You can make a shallow copy of an - AuthenticatedDictionary, and insert, find, and delete on it without - affecting the original or other shallow copies. Copying a shallow copy with - n changes made is O(N). When you want to save your changes to disk, run - commit(). After one shallow copy (or the updated original) has been - comitted, no accesses are allowed to anything that isn't it or a descendant - of it created after the commit. - - Each chain of an AuthenticatedDictionary and its shallow copy descendants - must have a unique database filename not shared with any other - AuthenticatedDictonary chain. It's fine to share it with other things, as - long as they never sync the database without calling commit on an - AuthenticatedDictionary first. - - This is how you use it: - >>> a = AuthenticatedDictionary(":memory:") - >>> for x in map(str, xrange(100)): - ... a.insert(x, x) - ... a.commit() - >>> a.clear() - >>> a.insert("stuff", "wombats") - >>> components, root = a.dump_state_components() - >>> a.get_hash() == root - True - >>> a.commit() - >>> a.clear() - >>> b = a.copy() - >>> b.import_from_state_components(components, root) - >>> b.get_hash() == root - True - >>> b.find("stuff") - 'wombats' - >>> c = a.copy() - >>> c.update_from_state_components(components, root) - >>> c.get_hash() == b.get_hash() - True - >>> c.find("stuff") - 'wombats' - >>> a.insert("thing", "other thing") - >>> a.commit() - >>> for x in map(str, xrange(100)): - ... a.insert(x, x) - ... a.commit() - >>> a.commit() - >>> util.bytes2hex(a.get_hash()) - '3cf8a21949088a41f405e3f37c90af54e3cc33dce1c7b8226bd4e2450ddf2aff2d8dd089f3\ -d78cd8a6dc5aea757a941868fd369daca22efd87b00d882b6e667d' - >>> a.find("thing") - 'other thing' - >>> print a.find("some third thing") - None - >>> len(set(a.iterkeys())) - 101 - >>> a.insert("some third thing", "foo") - >>> a.remove("thing") - >>> a.remove("some third thing") - >>> a.insert("thing", "other thing") - >>> util.bytes2hex(a.get_hash()) - '3cf8a21949088a41f405e3f37c90af54e3cc33dce1c7b8226bd4e2450ddf2aff2d8dd089f3\ -d78cd8a6dc5aea757a941868fd369daca22efd87b00d882b6e667d' - >>> a.get_node_by_hash(a.get_hash()) - 'root' - >>> a.commit() - >>> a.get_node_by_hash(a.get_hash()) - 'root' - >>> a.insert("thing", "a different thing") - >>> util.bytes2hex(a.get_hash()) - 'af8e3873a0eb172a1aefe78661630f908266e4edbcaae9752ca0fc6a441bad654485\ -43240d17f7ced11d5dad2f72a12ffab1afd7ad600ee4cfb89cdb4a5c64c8' - - This next test depends explicitly on internal pointer values. - >>> a.node_to_state_component("100") # doctest: +NORMALIZE_WHITESPACE - StateComponent gDoDgLKgh+RqaXBf6kdnQ67qkSy4wXwcQ/jdNFApA/rGZFssb87C+4ygOid5\ -l3yNr1m9i9inTKl+pWhOQC0rRg== - \t<13 bytes of data> - - - We also get cool order independence - >>> import random - >>> observed_hashes = set() - >>> for i in xrange(10): - ... items = [str(n) for n in xrange(10)] - ... random.shuffle(items) - ... d = AuthenticatedDictionary(":memory:", table="test{}".format(i)) - ... for item in items: - ... d.insert(item, item) - ... d.commit() - ... random.shuffle(items) - ... for item in items: - ... d.remove(item) - ... d.commit() - ... random.shuffle(items) - ... for item in items: - ... d.insert(item, item) - ... d.commit() - ... random.shuffle(items) - ... for item in items: - ... d.insert(item, "{} updated".format(item)) - ... d.commit() - ... observed_hashes.add(d.get_hash()) - >>> len(observed_hashes) - 1 - - A demo of the shallow copy semantics: - >>> c0 = AuthenticatedDictionary(":memory:", table="copies") - >>> c0.insert("thing", "other thing") - >>> digest = c0.get_hash() - >>> c0.get_hash() == digest - True - - Note that we can copy without committing. This is O(number of updates made - since last commit) - >>> c1 = c0.copy() - >>> c0.get_hash() == digest - True - >>> c1.find("thing") - 'other thing' - >>> c1.insert("thing", "not a thing") - >>> c0.get_hash() == digest - True - >>> c1.find("thing") - 'not a thing' - >>> c0.find("thing") - 'other thing' - >>> c0.get_hash() == digest - True - >>> c1.get_hash() == digest - False - >>> print c1.get_node_by_hash(digest) - None - >>> c0.remove("thing") - >>> print c0.find("thing") - None - >>> c1.find("thing") - 'not a thing' - >>> c1.commit() - - - - """ - - def __init__(self, filename=":memory:", table="AuthenticatedDictionary", - parent=None): - """" - Make a new AuthenticatedDictionary. Store it in the specified file. If a - table is given, use that table name. - - If a parent is given, this AuthenticatedDictionary will be a transaction - based off the parent. If this one is committed, the parent and any other - copies must never be used again. If the parent or some other copy of it - is committed, this copy must never be used again. - - Either a file name and a table, or a parent, should be specified, and - not both or neither. - - """ - - if parent is None: - # We're setting up ourselves. We need to open the database and - # potentially create a root. - - # Get the SQLiteDict we use to store our data. Make sure to use - # transactions. - self.store = SQLiteShelf(filename, table=table, lazy=True) - - # We use another SQLiteDict in the same database to maintain a - # mapping from Merkle hashes to node pointers. - self.hashmap = SQLiteShelf(filename, table="{}hashes".format( - table), lazy=True) - - # Keep track of what to use as the next pointer for the next - # allocated node. If it's None, it will get loaded from the database - # when needed, and it gets saved back to the database on commit. - self.next_pointer = None - - # Keep track of stored or overwritten nodes not yet comitted, by - # pointer - self.updates = {} - - # Keep track of deleted node pointers - self.deletes = set() - - # And the same for the hashmap (these are by Merkle hash) - self.hashmap_updates = {} - self.hashmap_deletes = set() - - # Make sure we have a root node - try: - # Make sure we can load the root node (and thus that it exists) - self.load_node("root") - except BaseException: # We couldn't load the root node, probably since it doesn't - # exist. Fix that. - self.store_node("root", MerkleTrieNode()) - self.update_node_hash("root") - - else: - # We're becoming a transactional copy of a given parent. - - # Steal their database - self.store = parent.store - self.hashmap = parent.hashmap - - # Steal their next available pointer value - self.next_pointer = parent.next_pointer - - # Steal a copy of their uncommitted updates. We can just store - # references to the objects, though, because every time we update a - # MerkleTrieNode we make a copy of it. - self.updates = dict(parent.updates) - - # Copy their uncommitted deletes as well. These can't be updated in - # place, so everything is OK. - self.deletes = set(parent.deletes) - - # The parent took care of making a root node (even if it's - # uncommitted), so we should have it. - - # Copy their uncommitted updates and deletes on the Merkle hash -> - # node pointer database. - self.hashmap_updates = dict(parent.hashmap_updates) - self.hashmap_deletes = set(parent.hashmap_deletes) - - def copy(self): - """ - Return a transactional copy of this AuthenticatedDictionary. Changes may - be made to it and read back from it. If it is ever committed, the - parent, all other copies of the parent, and all copies made from the - copy before the commit must never be used again. If some other copy of - the parent, the parent itself, or a copy of the copy is commited, the - original copy must never be used again. - - """ - - # Just use our awesome copy constructor mode. - return AuthenticatedDictionary(parent=self) - - def get_hash(self): - """ - Return the 64-byte digest of the current state of the dictionary, as a - bytestring. - - """ - - return self.load_node("root").hash - - def iterkeys(self): - """ - Yield each key in the AuthenticatedDictionary. Not key hashes, the - actual key strings. - - The caller may not add to or remove from the AuthenticatedDictionary - while iterating over it. Modifying values shouild be OK. - - """ - - # What do we need to look at? - stack = [] - stack.append("root") - - while len(stack) > 0: - # Pop a node to look at - node = stack.pop() - - # Load the node's key - key = self.get_node_key(node) - - if key is not None: - # It's a data node - yield key - else: - # It's not a data node. Look at its children. - for child in self.get_node_children(node): - if child is not None: - # It actually has this child, so look at it. - stack.append(child) - - def iteritems(self): - """ - Yield each key, value pair in the AuthenticatedDictionary. Not key hashes, the - actual key strings. - - The caller may not add to or remove from the AuthenticatedDictionary - while iterating over it. Modifying values shouild be OK. - - """ - - # What do we need to look at? - stack = [] - stack.append("root") - - while len(stack) > 0: - # Pop a node to look at - node = stack.pop() - - # Load the node - node_obj = self.load_node(node) - key = node_obj.key - value = node_obj.value - - if key is not None: - # It's a data node - yield (key, value) - else: - # It's not a data node. Look at its children. - for child in self.get_node_children(node): - if child is not None: - # It actually has this child, so look at it. - stack.append(child) - - def insert(self, key, value): - """ - Insert the given value into the trie under the given key. The key and - the value must both be strings. - - """ - - # Hash the key - key_hash = util.bytes2hex(hashlib.sha512(key).digest()) - - self.recursive_insert("root", key_hash, key, 0, value) - - def recursive_insert(self, node, key_hash, key, level, value): - """ - Insert the given value under the given key with the given hash (in hex) - into the subtree rooted by the given node. level indicates the character - in key_hash that corresponds to this node. - - """ - - # It goes under the child slot corresponding to the level-th - # character of the key hash. - - if level >= len(key_hash): - raise Exception("Tree deeper ({}) than length of keys.".format( - level)) - - # Which child slot do we use? - child_index = int(key_hash[level], base=16) - - # Get the child pointer value, or None if there is no child there. - child = self.get_node_children(node)[child_index] - # logging.debug('INSERT to [{}:{}] with "{}" {} bytes'.format( - # self.store.table, node, key_hash[:8], len(value))) - - if child is None: - # If that slot is empty, put the value there in a new node. - child = self.create_node() - self.set_node_key(child, key) - self.set_node_value(child, value) - self.update_node_hash(child) - - # Attach the node in the right place - self.set_node_child(node, child_index, child) - else: - # Get the child's key - child_key = self.get_node_key(child) - - if child_key == key: - # If the slot has a node with the same key, overwrite - # it. - self.set_node_value(child, value) - self.update_node_hash(child) - - if self.get_node_by_hash(self.get_node_hash(child)) != child: - raise Exception("Inconsistent insert") - - if self.load_node(child).children is not None: - raise Exception("Updated value on node with children") - - elif child_key is not None: - # If the slot has a node with a different key hash, - # recursively insert both that old value and this new ones - # as children of the node that's there. - - # Hash the child key - child_key_hash = util.bytes2hex(hashlib.sha512( - child_key).digest()) - - # Get the value the child was storing. - child_value = self.get_node_value(child) - - # Blank it out - self.set_node_key(child, None) - self.set_node_value(child, None) - - # Store the value that was there as a child of the child - # node. - self.recursive_insert(child, child_key_hash, child_key, - level + 1, child_value) - - # Store our value as a (hopefully different) child of the - # child node. - self.recursive_insert(child, key_hash, key, level + 1, value) - - if self.get_node_by_hash(self.get_node_hash(child)) != child: - raise Exception("Inconsistent insert") - - if (self.get_node_key(child) is not None or - self.get_node_value(child) is not None): - - raise Exception("Node with children added still has value") - - else: - # If the slot has a node with no key hash (i.e. it has - # children), insert the new value as a child of that node. - - self.recursive_insert(child, key_hash, key, level + 1, value) - - if self.get_node_by_hash(self.get_node_hash(child)) != child: - raise Exception("Inconsistent insert") - - if (self.get_node_key(child) is not None or - self.get_node_value(child) is not None): - - raise Exception("Node with children added still has value") - - # Update our Merkle hash - self.update_node_hash(node) - - if self.get_node_by_hash(self.get_node_hash(node)) != node: - raise Exception("Inconsistent insert") - - def remove(self, key): - """ - Remove the value under the given key from the trie. The key must be in - the trie, and a string. - - """ - - # Hash the key - key_hash = util.bytes2hex(hashlib.sha512(key).digest()) - - # Run the removal - self.recursive_remove("root", key_hash, key, 0) - - def recursive_remove(self, node, key_hash, key, level): - """ - Remove the value with the given key (which has the given hash) from the - subtree rooted at the given node. The key hash is in hex, and level is - the character in that hash being used at this level to decide on a child - storage location. - - The algorithm works by the invariant that every leaf node (i.e. one with - a value) has a sibling. - - If the key to remove is our direct descendant, drop it. This will never - leave us with no children, since every leaf node has a sibling. It may - leave us with one child with a value that now has no siblings. - - If the key to remove is our indirect descendant, we know it has a - sibling. Remove the key we are removing, recursively. If this leaves a - leaf node without any siblings, that value will be promoted to the child - we recursed into. This may leave us with one child with a value that now - has no siblings. - - If we now have only one child, which has a value, promote that value to - this node and drop the child. - - """ - - # The key can be found under the child slot corresponding to the level- - # th character of the key hash. - - if level >= len(key_hash): - raise Exception("Tree deeper ({}) than length of keys.".format( - level)) - - # Which child slot do we use? - child_index = int(key_hash[level], base=16) - - # Get the pointer value, or None if there is no child there. - child = self.get_node_children(node)[child_index] - - # logging.debug('REMOVE from [{}:{}] key {}'.format(self.store.table, node, key_hash[:8])) - - if child is None: - # If that slot is empty, the key can't possibly be in the trie. - raise Exception("Tried to remove key hash {} that wasn't in the " - "trie".format(key_hash)) - - child_key = self.get_node_key(child) - - if child_key == key: - # If the slot has a node with the same key, we've found its leaf - # node. Remove the leaf node. - self.delete_node(child) - - # Set the child pointer in this node to None - self.set_node_child(node, child_index, None) - - elif child_key is not None: - # If the slot has a node with a different key, we're trying to - # remove something not in the trie. - raise Exception("Tried to remove key hash {} that wasn't in " - "the trie".format(key_hash)) - else: - # If the slot has a node with no key (i.e. it has children), recurse - # down on that node - - self.recursive_remove(child, key_hash, key, level + 1) - - if self.get_node_by_hash(self.get_node_hash(child)) != child: - raise Exception("Inconsistent remove") - - # If we now have only one child, and that child has a value, promote the - # value and remove the child. - - # Get a list of all the child pointers, some of which may be None. This - # reflects all the changes made to remove the key we just removed. - child_list = self.get_node_children(node) - - # Get a list of the indices that are filled with children - child_indices = [i for i, child in enumerate(child_list) if child is not - None] - - if len(child_indices) == 1 and level != 0: - # We have an only child, and we aren't the root, so we may need to - # promote its value to us. - - # This holds the pointer for the child - child = child_list[child_indices[0]] - - # Get its key - child_key = self.get_node_key(child) - - if child_key is not None: - # The only child is a leaf node. We need to promote it. - - # Steal the child's key and value - self.set_node_key(node, child_key) - self.set_node_value(node, self.get_node_value(child)) - - # Kill the child - self.delete_node(child) - self.set_node_child(node, child_indices[0], None) - - if self.load_node(node).children is not None: - raise Exception("Node left with children when value " - "promoted") - - # Update our Merkle hash - self.update_node_hash(node) - - if self.get_node_by_hash(self.get_node_hash(node)) != node: - raise Exception("Inconsistent remove") - - def find(self, key): - """ - Return the value string corresponding to the given key string. - - """ - - # Hash the key - key_hash = util.bytes2hex(hashlib.sha512(key).digest()) - - # Run the removal - return self.recursive_find("root", key_hash, key, 0) - - def recursive_find(self, node, key_hash, key, level): - """ - Find the value with the given key, which has the given hash, in the - subtree rooted at the given node. The key hash is in hex, and level is - the character in that hash being used at this level to decide on a child - storage location. - - Returns the value found (a string), or None if no value is found. - - """ - - # The key can be found under the child slot corresponding to the level- - # th character of the key hash. - - if level >= len(key_hash): - raise Exception("Tree deeper ({}) than length of keys.".format( - level)) - - # Which child slot do we use? - child_index = int(key_hash[level], base=16) - - # Get the pointer value, or None if there is no child there. - child = self.get_node_children(node)[child_index] - - if child is None: - # If that slot is empty, the key can't possibly be in the trie. - return None - - child_key = self.get_node_key(child) - - if child_key == key: - # If the slot has a node with the same key, we've found its leaf - # node. Return the value. - return self.get_node_value(child) - - elif child_key is not None: - # If the slot has a node with a different key, we're trying to find - # something not in the trie. - return None - else: - # If the slot has a node with no key (i.e. it has children), recurse - # down on that node - - return self.recursive_find(child, key_hash, key, level + 1) - - def audit(self): - """ - Make sure the AuthenticatedDictionary is in a consistent state. - - """ - - # Put some statistics - logging.debug("TRIE AUDIT: {} nodes on disk, {} updates, {} " - "deletes".format(len(self.store), len(self.updates), - len(self.deletes))) - - # What do we need to look at? - stack = [] - stack.append("root") - - root_hash = self.get_hash() - - while len(stack) > 0: - # Pop a node to look at - node = stack.pop() - - if node in self.updates and node in self.deletes: - logging.error("Node {} both updated and deleted".format(node)) - - # Load the node's struct - node_struct = self.load_node(node) - - # Compute what its hash should be - expected_hash = hashlib.sha512(self.node_to_bytes(node)).digest() - - if expected_hash != node_struct.hash: - logging.error("Hash mismatch on node {}".format(node)) - - if node_struct.children is not None: - if node_struct.key is not None or node_struct.value is not None: - logging.error("Both value and children on node " - "{}".format(node)) - for child in node_struct.children: - if child is not None: - # It actually has this child, so look at it. - stack.append(child) - else: - if node_struct.key is None or node_struct.value is None: - logging.error("Neither value nor key on node {}".format( - node)) - - if self.find(node_struct.key) != node_struct.value: - logging.info("Node {} could not be found by search.".format( - node)) - - def get_node_children(self, node): - """ - Given the pointer of a node, return a sequence of either child - pointers, or Nones if a node does not have a child at that index. - - """ - - # Nodes are stored internally as pickled MerkleTrieNodes - - # Load the children list, which may itself be None - children = self.load_node(node).children - - if children is None: - # We don't store a whole empty child list, but our callers expect to - # get one, so we fake it. - return [None] * ORDER - else: - return children - - def set_node_child(self, node, index, child): - """ - Set the index'th child of the node at the given pointer to the given - child pointer value, which may be None. - - """ - - # Load the node - node_struct = self.load_node(node).copy() - - if node_struct.children is None: - # We need to allocate spme spaces for child pointers - node_struct.children = [None] * ORDER - - # Update the node - node_struct.children[index] = child - - if child is None: - # We may have removed the last real child. - # Count up all the not-None children - actual_children = sum(1 for c in node_struct.children - if c is not None) - - if actual_children == 0: - # No real children remain. Get rid of the list of Nones - node_struct.children = None - - # Save the node again - self.store_node(node, node_struct) - - def get_node_key(self, node): - """ - Returns the key stored in the node with the given pointer, or None if it - has no key. - - """ - - return self.load_node(node).key - - def set_node_key(self, node, key): - """ - Set the key stored at the node with the given pointer. May be set to - None. - - """ - - # Load the node - node_struct = self.load_node(node).copy() - - # Update the node - node_struct.key = key - - # Save the node again - self.store_node(node, node_struct) - - def get_node_value(self, node): - """ - Return the value stored at the node with the given pointer, or None if it - has no value. - - """ - - return self.load_node(node).value - - def set_node_value(self, node, value): - """ - Set the value stored at the node with the given pointer. May be set to - None. - - """ - - # Load the node - node_struct = self.load_node(node).copy() - - # Update the node - node_struct.value = value - - # Save the node again - self.store_node(node, node_struct) - - def get_node_hash(self, node): - """ - Return the Merkle hash of the node with the given pointer. - - """ - - return self.load_node(node).hash - - def get_node_by_hash(self, node_hash): - """ - Given a node Merkle hash, return the node pointer that the hash belongs - to, or None if no node exists with the given Merkle hash. - - """ - - if node_hash in self.hashmap_deletes: - # This node has been deleted - return None - elif node_hash in self.hashmap_updates: - # This node has been updated (almost certainly created) since the - # last commit. - return self.hashmap_updates[node_hash] - elif node_hash in self.hashmap: - # This node exists in the actual database - return self.hashmap[node_hash] - else: - # The node isn't in the database and hasn't been added. - return None - - def node_to_bytes(self, node): - """ - Given a node pointer, return a bytestring containing the node's unique - state and the Merkle hashes of its children. The hash of this is the - node's Merkle hash. - - This is the same regardless of what pointer a node or its children have. - - """ - - # Load the node struct - node_struct = self.load_node(node) - - # Structure: we have child count (1 byte), then that many child number - # (byte) and Merkle hash (64 byte) records, followed optionally by a key - # length (8 bytes), a key, and a value (which is the remainder). If key - # length is not provided, no key is used. - - # This holds the bytestring parts to join together - parts = [] - - if node_struct.children is not None: - # How many non-empty children do we have? - child_count = sum(1 for child in node_struct.children if - child is not None) - - # Put the child couint - parts.append(struct.pack(">B", child_count)) - - for i, child_pointer in enumerate(node_struct.children): - if child_pointer is not None: - # Say we have a child with this number - parts.append(struct.pack(">B", i)) - # Go get the Merkle hash for this child - parts.append(self.get_node_hash(child_pointer)) - else: - # No child list at all. Put 0 children. - parts.append(struct.pack(">B", 0)) - - if node_struct.key is not None: - # We have a key and a value. Put the key length. - parts.append(struct.pack(">Q", len(node_struct.key))) - # Then the key string - parts.append(node_struct.key) - # Then the value string - parts.append(node_struct.value) - - return "".join(parts) - - def update_node_hash(self, node): - """ - Recalculate the Merkle hash for the given node. All of its childrens' - Merkle hashes must be up to date. - - This is just the hash of the node, when the node is converted to bytes. - - """ - - # Load the node - node_struct = self.load_node(node).copy() - - if node_struct.hash is not None: - # Either this node or a new node is listed under the node's current - # hash. If it's this node, we need to remove the listing. - - if node_struct.hash in self.hashmap_updates: - if self.hashmap_updates[node_struct.hash] == node: - - # We're in as an update. Remove the update. - del self.hashmap_updates[node_struct.hash] - else: - # Someone has overwritten us. Do nothing - pass - elif (node_struct.hash in self.hashmap and - self.hashmap[node_struct.hash] == node): - # Nobody has replaced us, and we're still under the old hash in - # the backing database. Delete the old Merkle hash -> node - # pointer mapping that points to us. - self.hashmap_deletes.add(node_struct.hash) - - # If we don't delete anything, it probably means our key and value - # got taken from us and put in some new leaf node that's exactly - # like we used to be and hence has the same hash, and it got its - # hash updated before we could (perhaps it is now our child) - - # Get the new hash. It's OK to hash the thing we copied from since we - # have't changed it yet. - node_struct.hash = hashlib.sha512(self.node_to_bytes(node)).digest() - - # Save the node - self.store_node(node, node_struct) - - # Add the new Merkle hash -> node pointer mapping - self.hashmap_deletes.discard(node_struct.hash) - self.hashmap_updates[node_struct.hash] = node - - def create_node(self): - """ - Return the pointer for a new node. Caller must make sure to update its - hash. - - """ - - if self.next_pointer is None: - if "next_pointer" in self.store: - # We need to load it from the database, since we haven't yet. - self.next_pointer = self.store["next_pointer"] - else: - # It's a fresh database; start at 0 - self.next_pointer = 0 - # Make a string to actually use as the node pointer - pointer = str(self.next_pointer) - - # Store a new node under that pointer - self.store_node(pointer, MerkleTrieNode()) - - # Increment the next pointer counter - self.next_pointer += 1 - - # Return the pointer to the new node - return pointer - - def load_node(self, node): - """ - Return a MerkleTrieNode object for the given node pointer. If you update - it in place, you must store is back with store_node. (It's not that - updating it in place won't take if you don't, it's that it might take - and so we need to know it has happened.) - - Internally, lools at the list of changes since the last commit first. If - the node hasn't been updated or deleted there, looks at the shelf - database. - - """ - - if node in self.updates: - # The node at this pointer has been set since the last commit, so - # use our in-memory version. - return self.updates[node] - elif node in self.deletes: - # The node at this pointer has been deleted since the last commit, - # so complain that someone is trying to use it. - raise Exception("Attempted read of deleted node {}".format(node)) - - # If it hasn't been updated or deleted, read it from the database. - return self.store[node] - - def store_node(self, node, node_struct): - """ - Store the given node struct (a MerkleTrieNode) under the given node - pointer. - - Changes are not written to disk until commit is called, but can be seen - by load_node. - - """ - - # If it was deleted, it isn't anymore. - self.deletes.discard(node) - - # Save the modified node as an update - self.updates[node] = node_struct - - def delete_node(self, node): - """ - Delete the node with the given pointer. It must not be the child of any - node. - - Changes are not written to disk until commit is called, but can be seen - by load_node. In particular, it is an error to try to load a node you - have deleted. - - """ - - # Grab the node hash before we delete it - node_hash = self.get_node_hash(node) - - if node in self.updates: - # If we wrote to it, now we need to delete it - del self.updates[node] - elif node in self.store: - # Mark this key for deletion from the database, since it's there. - self.deletes.add(node) - else: - # Complain about deleting a node that doesn't exist. - raise Exception("Attempt to delete non-existent node {}".format( - node)) - - # Delete the Merkle hash to node pointer mapping - if node_hash in self.hashmap: - # It's in the real database, so mark it for deletion - self.hashmap_deletes.add(node_hash) - if node_hash in self.hashmap_updates: - # It's not yet recorded in the database (potentially also). Don't - # record it. - del self.hashmap_updates[node_hash] - - def clear(self): - """ - Remove all keys and values from the AuthenticatedDictionary. Does it - quickly by emptying the underlying database table, but consequently - invalidates all other copies of the AuthenticatedDictionary. - - """ - - # Throw out our delta from the database - self.deletes = set() - self.updates = {} - - self.hashmap_deletes = set() - self.hashmap_updates = {} - - # Clear the hashmap. This doesn't commit to the database, but empties - # the shared SQLiteShelf. - self.hashmap.clear() - - # Clear the node store - self.store.clear() - - # We don't *need* to do this, but our test cases are happier if we start - # the node pointers over again. - self.next_pointer = None - - # We've deleted our root node. Make a new one. - self.store_node("root", MerkleTrieNode()) - self.update_node_hash("root") - - def commit(self): - """ - Commit changes to disk. Call this when you are done with a transaction. - """ - - # Audit the adds and deletes - for key in six.iterkeys(self.updates): - if key in self.deletes: - raise Exception("Deleting and updating the same key: {}".format( - util.bytes2string(key))) - - for key in self.deletes: - if key in self.updates: - raise Exception("Deleting and updating the same key: {}".format( - key)) - - for key in six.iterkeys(self.hashmap_updates): - if key in self.hashmap_deletes: - raise Exception("Deleting and updating the same key: {}".format( - util.bytes2string(key))) - - for key in self.hashmap_deletes: - if key in self.hashmap_updates: - raise Exception("Deleting and updating the same key: {}".format( - util.bytes2string(key))) - - # First, check up on all our updated nodes - updated_nodes = list(self.updates.keys()) - for node in updated_nodes: - if self.get_node_by_hash(self.get_node_hash(node)) != node: - - print(self.updates) - print(self.deletes) - print({util.bytes2string(key): value for key, value in six.iteritems(self.hashmap_updates)}) - print(set(util.bytes2string(item) for item in self.hashmap_deletes)) - - raise Exception("Node {} with hash {}, key {} not retrievable".format( - node, util.bytes2string(self.get_node_hash(node)), - self.get_node_key(node))) - - if self.next_pointer is not None: - # Save the next unused node pointer to the store, if we ever - # initialized it. - self.store["next_pointer"] = self.next_pointer - - for node, node_struct in six.iteritems(self.updates): - # Update all the updated nodes, pickling them in the process. - self.store[node] = node_struct - - for node in self.deletes: - if node in self.store: - # Delete each deleted node that was actually in the database and - # not, for example, added and deleted since the last commit. - del self.store[node] - - for node_hash, node_pointer in six.iteritems(self.hashmap_updates): - # Record all updated Merkle hash to pointer mappings. These are - # almost certainly all additions. - self.hashmap[node_hash] = node_pointer - - for node_hash in self.hashmap_deletes: - # Record all the deleted Merkle hash to node pointer mappings - del self.hashmap[node_hash] - - # Reset our records of what changes we need to make - self.updates = {} - self.deletes = set() - self.hashmap_updates = {} - self.hashmap_deletes = set() - - # Sync the store to disk, ending the transaction and implicitly starting - # a new one. - self.store.sync() - # This uses the same underlying connection, but it never hurts to be - # thorough. Hopefully. - self.hashmap.sync() - - for node in updated_nodes: - # Make sure the hashmap is still consistent now. - if self.get_node_by_hash(self.get_node_hash(node)) != node: - raise Exception("Node {} with hash {} not retrievable".format( - node, util.bytes2string(self.get_node_hash(node)))) - - def discard(self): - """ - Discard any changes made since the last commit. - - You don't need to call this if you're just getting rid of an - AuthenticatedDictionary. You can just let the garbage collector collect - it. - - """ - - # Don't mess with the database, since we now only ever touch the - # database on commit. Instead, just take the database as being correct. - - self.updates = {} - - self.deletes = set() - - def node_to_state_component(self, node): - """ - Given a node pointer, return an AuthenticatedDictionaryStateComponent - representing the node. - - """ - - return AuthenticatedDictionaryStateComponent(self.node_to_bytes(node)) - - def update_from_state_components(self, state_components, root_hash): - """ - Given a dict from hash to StateComponent, and a root hash, make this - AuthenticatedDictionary have the given hash. Requires that all - dependency StateComponents be in the dict or in this - AuthenticatedDictionary. - - The passed dict must contain the new root StateComponent, and root_hash - must differ from the current root hash of the State. - - Assumes the StateComponents have all been validated. - - """ - - # Conceptually, we're replacing the root and core of a tree, while - # keeping some of the branches - - # Keep a set of re-used subtree root hashes. - reused_roots = set() - - # We start at the root hash of the new tree, and traverse it. For each - # Merkle hash, if we have a node for that hash, put it in the list of - # re-used subtree roots and don't traverse down it. (The nodes for this - # traversal can all come from the input dict.) - - # This holds the StateComponents we're traversing down through - stack = [state_components[root_hash]] - - while len(stack) > 0: - # Grab the top of the stack. - current_component = stack.pop() - - for child_hash in current_component.get_dependencies(): - if self.get_node_by_hash(child_hash) is not None: - # This child is the root of a re-used subtree. Remember not - # to delete it, and don't traverse it. - reused_roots.add(child_hash) - else: - # This child is a new node that came in in the dict. Recurse - # into it. - stack.append(state_components[child_hash]) - - # We traverse our tree. If we find a re-used subtree root, don't - # traverse down that branch. Otherwise, do. Remove the root of each - # subtree we do traverse. - - # Now this holds a list of node pointers that we have to traverse. - stack = ["root"] - - while len(stack) > 0: - # Pop off the node to process - current_pointer = stack.pop() - - if self.get_node_hash(current_pointer) not in reused_roots: - # This is node and its children that aren't themselves reused - # subtree roots need to be removed. - - for child_pointer in self.get_node_children(current_pointer): - if child_pointer is None: - # Skip empty children - continue - - # Put the real children on the stack - stack.append(child_pointer) - - # Get rid of this replaced node (which may be "root") - self.delete_node(current_pointer) - - # Now we only have the forrest of re-used subtrees. - - # Traverse the tree of StateComponents again, adding in pointers to re- - # used subtrees or new nodes, as appropriate. - - # Now this holds the StateComponents we're traversing and turning into - # nodes. - stack = [state_components[root_hash]] - - while len(stack) > 0: - # Grab the top of the stack. - current_component = stack[-1] - - # How many of its children do we still need to make nodes for? - missing_children = 0 - for child_hash in current_component.get_dependencies(): - if self.get_node_by_hash(child_hash) is None: - # Make sure we make a node for this child before truing to - # do its parent. - stack.append(state_components[child_hash]) - missing_children += 1 - - if missing_children > 0: - # We've put all the children we still need on top of the current - # node on the stack. Do them, and then come back to this node - # when they're done. - continue - - # When we get here, we know the subtrees for all our children have - # been built. Build a subtree for us. - - if len(stack) == 1: - # We're adding in the new root. Don't create a node, re-use the - # "root" pointer. - node_pointer = "root" - - # The old "root" node was deleted already above, beacuse it - # wasn't the root of a reused subtree. - - # Store a fresh node as "root" - self.store_node(node_pointer, MerkleTrieNode()) - else: - # Make a new node with a new pointer to realize this non-root - # StateComponent. - node_pointer = self.create_node() - - for i, child_hash in enumerate(current_component.get_child_list()): - if child_hash is not None: - # Grab the node pointer used for the subtree with this hash - child_node = self.get_node_by_hash(child_hash) - - # We definitely should have this node at this point. - assert child_node is not None - - # Make it a child of the new node. - self.set_node_child(node_pointer, i, child_node) - - if current_component.get_key() is not None: - # We need to add a key and a value - self.set_node_key(node_pointer, current_component.get_key()) - self.set_node_value(node_pointer, - current_component.get_value()) - - # Now we have populated this node, and we can calculate its Merkle - # hash. - self.update_node_hash(node_pointer) - - # It really needs to match the hash of the thing we're supposed to - # be adding. - assert (self.get_node_hash(node_pointer) == - current_component.get_hash()) - - # Now we're done putting in the node on top of the stack. Go up and - # do the next one. - stack.pop() - - # Now the AuthenticatedDictionary should be up to date. Since everything - # went through load_node and store_node at some level, we have not even - # invalidated other copies. - - # Make sure we did it right. - assert root_hash == self.get_hash() - - def import_from_state_components(self, state_components, root_hash): - """ - Given a dict from hash to StateComponent, and a root hash, make this - AuthenticatedDictionary have the given hash. Requires that all - dependency StateComponents be in the dict, and that this - AuthenticatedDictionary start out empty. - - Assumes the StateComponents have all been validated. - - """ - - # What component are we currently putting in? - current_component = None - - # What components are stil to add? Use a stack because depth-first is - # limited in depth. - to_add = [root_hash] - - while len(to_add) > 0: - # Now we just insert all the key/value pairs in the properly rooted - # tree. TODO: This may be a bit slow. - - # Look at the top StateComponent - current_component = state_components[to_add.pop()] - - # What are its children? - child_hashes = list(current_component.get_dependencies()) - for child_hash in child_hashes: - # Put its children on top of the stack to process next - to_add.append(child_hash) - - if current_component.get_key() is not None: - # Put in a node holding the key-value mapping - self.insert(current_component.get_key(), - current_component.get_value()) - - def dump_state_components(self): - """ - Return a dict from Merkle hash to StateComponent, and the Merkel hash of - the root StateCmponent, for a complete set of StateComponents describing - this AuthenticatedDictionary. - - """ - - # Make the dict from Merkle hash to state component - state_components = {} - - # This holds a list of node pointers that we have to traverse. - stack = ["root"] - - while len(stack) > 0: - # Pop off the node to process - pointer = stack.pop() - - for child_pointer in self.get_node_children(pointer): - if child_pointer is None: - # Skip empty children - continue - - # Put the real children on the stack - stack.append(child_pointer) - - # Make a StateComponent for the node - component = self.node_to_state_component(pointer) - - # Keep it under its hash - state_components[component.get_hash()] = component - - return state_components, self.get_hash() - - -if __name__ == "__main__": - - # Run doctests. See - import doctest - doctest.testmod() diff --git a/blockchain/pybc/Block.py b/blockchain/pybc/Block.py deleted file mode 100644 index 4660970a8..000000000 --- a/blockchain/pybc/Block.py +++ /dev/null @@ -1,353 +0,0 @@ -""" -Block.py: contains the Block class. - -""" - -from __future__ import absolute_import -import hashlib -import struct -import time -import logging -import traceback - -import pybc.util - - -class Block(object): - """ - Represents a block in a blockchain. Can hold some data and calculate its own - hash. - - Can be pruned down to just a proof chain header, discarding everything but - previous hash, body hash, and nonce. The nonce isn't in the original mini- - blockchain proposal, but having it lets us use a different proof of work - scheme from the hash scheme used to identify blocks, which is a requirement - of the PyBC PowAlgorithm architecture. - - """ - - def __init__(self, previous_hash, body_hash=None, nonce=None): - """ - Make a new block with the given previous block header hash. If you - specify a body_hash, the block is allowed to remain as just a header - (pervious hahs and body hash), with no actual body data. - - If you don't specify a body hash, or if you have new body data, you can - use add_body() (or body_from_bytes()) to set it. If you don't, the block - will serialize down to just a header. If you do, it will serialize to a - whole block with payload. - - To remove the body again, call remove_body(). - - A nonce can be specified if one is known; otherwise, one will have to be - worked out from proof of work before the block will be serializeable. - - """ - - # Save the previous hash - self.previous_hash = previous_hash - - # Store the nonce if specified, or None if we still need to find a valid - # nonce. - self.nonce = nonce - - # Save the body hash - self.body_hash = body_hash - - # Say we don't have a body yet - self.has_body = False - - # Say we haven't had our body explicitly discarded locally yet - self.body_discarded = False - - def add_body(self, target, height, state_hash, payload, - timestamp=int(time.time())): - """ - Add a body to the block, by providing a difficulty target, block height, - system state hash, payload, and timestamp. If no timestamp is provided, - the current time is used. - - """ - - # A body has been added - self.has_body = True - - # Save the target - self.target = target - - # Save the height - self.height = height - - # Save the payload - self.payload = payload - - # Save the state hash - self.state_hash = state_hash - - # Save the timestamp - self.timestamp = timestamp - - # Hash the body bytestring (which is now fully specified) and save that - # hash in the header. - self.body_hash = hashlib.sha512(self.body_to_bytes()).digest() - - def remove_body(self): - """ - Remove all the body data from the block. - - """ - - # We threw out the body on purpose. - self.body_discarded = True - - if self.has_body: - # The body fields are filled, so we can safely del them. - self.has_body = False - - del self.target - del self.height - del self.payload - del self.state_hash - del self.timestamp - - # We keep the body hash, since we need that in the header. - - def body_to_bytes(self): - """ - If the block has its body, return a bytestring containing the body data. - Otherwise, if the body has been discarded, return None. - - You can't do this until you have a nonce for the block, or if the block - hasn't had a body added. - - """ - - # The block body has: - # The block difficulty target (64 bytes) - # The block height in the chain (8 bytes) - # The block timestamp (8 bytes) - # The hash of the current system state - # The payload itself (unspecified length) - - return "".join([self.target, struct.pack(">Q", self.height), - struct.pack(">Q", self.timestamp), self.state_hash, self.payload]) - - def body_from_bytes(self, bytestring): - """ - Given a bytestring representing a packed block body (as from - body_to_bytes), unpack the body and add it to the current block. - - """ - - # Define the layout for the fixed-length data we're unpacking: an 8-byte - # unsigned nonce, a 64-byte target string, an 8-byte unsigned height, an - # 8-byte timestamp, and a 64-byte state hash - layout = ">64sQQ64s" - - # Unpack the block body - target, height, timestamp, state_hash = \ - struct.unpack_from(layout, bytestring) - - # Grab the payload - payload = bytestring[struct.calcsize(layout):] - - # Add the block body, with all its fields. Order is different since - # stuff we might leave optional (timestamp) is last in the argument - # list. - self.add_body(target, height, state_hash, payload, timestamp) - - def to_bytes(self): - """ - Return the block as a byte string. Including the nonce, which must be - filled in. - - Relies on the body_hash field having been filled in, either from - previous deserialization or from having a body added. - - """ - - # A block is a sequence of bytes with a header and a body. - # The header has: - # The nonce (8 bytes) - # The hash of the previous block (SHA-512, 64 bytes) - # The hash of the body (SHA-512, 64 bytes) - # The body is serialized by body_to_bytes, and may not be present. - - # We put the nonce at the front so that the header happens to match what - # the default PoW algorithm hashes, and thus our block hashes happen to - # be the same as our PoW hashes when using that algorithm. - - # This holds all the parts to join up - parts = [struct.pack(">Q", self.nonce), self.previous_hash, - self.body_hash] - - if self.has_body: - # We need to add in the body data - parts.append(self.body_to_bytes()) - - return "".join(parts) - - @classmethod - def from_bytes(cls, bytestring): - """ - Make a new Block from the given bytestring. Uses the entire bytestring. - - Returns the Block, or None if the block could not be unpacked. - - """ - try: - - # Define the layout for the header we're unpacking: 8 byte unsigned - # int (nonce) and two 64-byte strings (previous hash and body hash) - layout = ">Q64s64s" - - # Unpack the block header - nonce, previous_hash, body_hash = \ - struct.unpack_from(layout, bytestring) - - # Get the body data, which comes after the header and runs to the - # end of the block. - body_bytes = bytestring[struct.calcsize(layout):] - - # Make a new block with header filled in - block = cls(previous_hash, body_hash=body_hash, nonce=nonce) - - if len(body_bytes) > 0: - # We also have some data that must be the body. Unpack and add - # that. - block.body_from_bytes(body_bytes) - - # Give it back as our deserialized block - return block - - except BaseException: # Block is malformed and could not be unpacked - - logging.error("Malformed block") - logging.error(traceback.format_exc()) - - return None - - def block_hash(self): - """ - Hash the header (including the body hash) to get the block's full hash, - as referenced by other blocks. This is not always going to be the same - as the PoW hash, depending on the PoW algorithm used, but that hash - never actually has to be stored. - - """ - - return hashlib.sha512("".join([struct.pack(">Q", self.nonce), - self.previous_hash, self.body_hash])).digest() - - def do_work(self, algorithm): - """ - Fill in the block's nonce by doing proof of work sufficient to meet the - block's target (lower is harder) using the given PowAlgorithm. Fills in - the block's nonce. - - """ - - # Fill in the nonce so that hashing it with the previous hash and the - # body hash will produce a proof of work hash that meets the target. - self.nonce = algorithm.do_work(self.target, "".join([self.previous_hash, - self.body_hash])) - - def do_some_work(self, algorithm, iterations=10000): - """ - Try to fill in the block's nonce, starting from the given placeholder, - and doing at most the given number of iterations. - - Returns True if we succeed, or False if we need to run again. - """ - - if self.nonce is None: - self.nonce = 0 - - # Run the algorithm - success, self.nonce = algorithm.do_some_work(self.target, - "".join([self.previous_hash, self.body_hash]), - placeholder=self.nonce, iterations=iterations) - - # TODO: we will pretend to be solved when we aren't in __str__, since we - # set nonce to a number. - - # Return whether we succeeded - return success - - def verify_work(self, algorithm): - """ - Returns True if the block's currently set nonce is sufficient to meet - the block's filled-in target under the given algorithm. - - This does not mean that the block is valid: you still need to make sure - the block's payload is legal, and that the target the block uses is - correct. - - """ - - # Just ast the algorithm if the nonce is good enough for the target, on - # our data_hash. - return algorithm.verify_work(self.target, "".join([self.previous_hash, - self.body_hash]), self.nonce) - - def get_work_points(self, algorithm): - """ - Returns the approximate number of proof-of-work "points" represented by - this block. If the proof of work function is one that requires a hash - below a threshold, this is probably going to be a function of the number - of leading zeros, but we leave it up to the PowAlgorithm to do the - actual calculation. - - Returns a long Python integer which may be quite big. - - """ - - return algorithm.points("".join([self.previous_hash, self.body_hash]), - self.nonce) - - def __str__(self): - """ - Print this block in a human-readable form. - - """ - - # This holds all the lines we want to include - lines = [] - - if self.nonce is None: - # Block hasn't been solved yet - lines.append("----UNSOLVED BLOCK----") - else: - lines.append("----SOLVED BLOCK----") - lines.append("Block hash: {}".format( - pybc.util.bytes2string(self.block_hash()))) - lines.append("Nonce: {}".format(self.nonce)) - - lines.append("Previous hash: {}".format(pybc.util.bytes2string( - self.previous_hash))) - lines.append("Body hash: {}".format(pybc.util.bytes2string( - self.body_hash))) - - if self.has_body: - # We can also put the body - lines.append("Height {}".format(self.height)) - lines.append("Timestamp: {}".format(pybc.util.time2string( - self.timestamp))) - - # Block hash isn't less than target. The hash of the block header under - # the blockchain's POW algorithm is less than target. So don't line them - # up and be misleading. - lines.append("Target: {}".format("".join("{:02x}".format(ord(char)) - for char in self.target))) - - lines.append("State hash: {}".format(pybc.util.bytes2string( - self.state_hash))) - - # No need to dump the payload, probably. - lines.append("<{} byte payload>".format(len(self.payload))) - - else: - # We threw out the body of the block, keeping only the header. - lines.append("") - - return "\n".join(lines) diff --git a/blockchain/pybc/Blockchain.py b/blockchain/pybc/Blockchain.py deleted file mode 100644 index 16422386d..000000000 --- a/blockchain/pybc/Blockchain.py +++ /dev/null @@ -1,1963 +0,0 @@ -""" -Blockchain.py: contains the Blockchain class. - -""" - -from __future__ import absolute_import -import hashlib -import struct -import time -import threading -import collections -import logging -import traceback - -from pybc.Block import Block -from pybc.State import State -from pybc.StateMachine import StateMachine -import pybc.util -import pybc.science -from . import sqliteshelf -import six -from six.moves import range - - -class Blockchain(object): - """ - Represents a Blockchain that uses a particular PowAlgorithm. Starts with a - genesis block (a block that has its previous hash set to 64 null bytes), and - contains a list of blocks based on that genesis block. Responsible for - verifying block payloads and that the rules respecting target updates are - followed. - - We also keep an in-memory dict of transactions, represented as bytestrings. - We include logic to verify transactions against the blockchain, but - transactions are not actually put into blocks. - - We also keep a State that describes some collated state data over all - previous blocks that new blocks might want to know about. - - """ - - def __init__(self, algorithm, store_filename, state, - minification_time=None): - """ - Given a PoW algorithm, a filename in which to store block data (which - may get an extension appended to it) and a State object to use (which - internally is responsible for loading itself from some file), make a - Blockchain. - - If a minification_time is specified, minify blocks that are burried - deeper than that number of blocks. - - """ - - # Set up our retargeting logic. We retarget so that it takes - # retarget_time to generate retarget_period blocks. This specifies a 1 - # minute blocktime. - self.retarget_time = 600 - self.retarget_period = 10 - - # Remember whether we're minifying blocks by discarding their bodies, - # and, if so, at what depth - self.minification_time = minification_time - - if not self.minification_time > self.retarget_period: - # We won't have the block we need to retarget from when it's time to - # retarget. - logging.warning("Minification time of {} no greater than retarget " - " period of {}, so we will not be able to check difficulty " - "retargets or mine retarget blocks.".format( - self.minification_time, self.retarget_period)) - - # Keep the PowAlgorithm around - self.algorithm = algorithm - - # Keep a database of all blocks by block hash. - self.blockstore = sqliteshelf.SQLiteShelf(store_filename, - table="blocks", lazy=True) - - # Keep a database for longest-fork data (really a table in the same - # database; sqlite3 makes multiple accesses to the same database OK) - self.fork_data = sqliteshelf.SQLiteShelf(store_filename, - table="fork", lazy=True) - - # Remember the store filename for periodic filesize checkups - self.store_filename = store_filename - - # Keep the highest Block around, so we can easily get the top of the - # highest fork. - self.highest_block = None - - # We keep the highest block's hash in the fork database under "highest" - if "highest" in self.fork_data: - self.highest_block = self.blockstore[self.fork_data["highest"]] - - # Keep a list of block hashes by height in our current fork, for - # efficient access by height. - self.hashes_by_height = [] - - # We keep that in the fork data too - if "hashes_by_height" in self.fork_data: - self.hashes_by_height = self.fork_data["hashes_by_height"] - - # We need a lock so we can be a monitor and thus thread-safe - self.lock = threading.RLock() - - # This is a dict of lists of pending blocks by the hash we need to - # verify them, and then by their own hash (to detect duplicates) - self.pending_on = collections.defaultdict(dict) - - # This is a dict of callbacks by pending block hash - self.pending_callbacks = {} - - # This holds our pending transactions as bytestrings by hash. - self.transactions = {} - - # This keeps the State at the tip of the longest fork. It should have - # automatically loaded itself from a file. TODO: What if the state gets - # out of sync with the blockchain when we kill the program? - self.state = state - - # This keeps a list of listener functions to call with block and state - # changes. - self.listeners = [] - - # This keeps track of whether the state is actually correct, or whether - # we had to cross a mini-block and need to redownload it. It starts out - # available if we have no blocks. - self.state_available = True - - if "state_available" in self.fork_data: - self.state_available = self.fork_data["state_available"] - - # This holds a StateMachine that we will set up if our State ever - # becomes unavailable. It will take the old, outdated State and download - # the parts it is missing and bring it up to date. When our State is - # available, this is None. - self.state_machine = None - - if not self.state_available: - # Our Sate was unavailable when we stopped last time, so we need to - # invalidate it again to get a new StateMachine. - self.invalidate_state() - - if self.highest_block is not None: - # Tell the StateMachine to start downloading the state for our - # current highest block, if we have one. - self.state_machine.download(self.highest_block.state_hash) - else: - # Somehow we marked our State invalid even though we rightly - # should have the pre-genesis State, since we know no highest - # block. - raise Exception("We know our State is invalid, but we don't " - "have a highest block that we need to get State for.") - - # This keeps the state at the tip of the longest fork, plus any pending - # transactions. - self.transaction_state = self.state.copy() - - # Start a timer measuring how long it takes us to download the - # blockchain. - pybc.science.start_timer("blockchain_download") - - def has_block(self, block_hash): - """ - Return True if we have the given block, and False otherwise. - - """ - - with self.lock: - # Lock since sqliteshelf isn't thread safe - return block_hash in self.blockstore - - def needs_block(self, block_hash): - """ - Return True if we need to get the given block, False otherwise. - - We can have blocks and still need to get them if, for example, we only - have the mini version and it hasn't been burried deeply enough. - - """ - - with self.lock: - if not self.has_block(block_hash): - # We don't have this block, so we need it - return True - - else: - # Load the block - block = self.blockstore[block_hash] - - if not block.has_body and not block.body_discarded: - # We don't have the body, but we didn't intentionally drop - # it. So we need it. - return True - else: - # We used to have the body, but we threw it out. - return False - - def get_block(self, block_hash): - """ - Return the block with the given hash. Raises an error if we don't have - it. - - """ - - with self.lock: - # Lock since sqliteshelf isn't thread safe - return self.blockstore[block_hash] - - def get_block_count(self): - """ - Return the number of blocks we know about (not just the number in the - longest chain). - - """ - - return len(self.blockstore) - - def get_block_locator(self): - """ - Returns a list of block hash bytestrings starting with the most recent - block in the longest chain and going back, with exponentially fewer - hashes the futher back we go. However, we make sure to include the 10 - most recent blocks. - - """ - - # We do need to lock because this is a complicated traversal. We can't - # for example, switch threads in the middle of this. - with self.lock: - - # This holds the list of hashes we use - locator = [] - - # This holds the step we use for how far to go back after including - # each block. - step = 1 - - # This holds our position in the longest chain, from the genesis - # block - position = len(self.hashes_by_height) - 1 - - while position > 0: - # Grab the hash - locator.append(self.hashes_by_height[position]) - - # Go back by the recommended step - position -= step - - if len(locator) > 10: - # After the 10 most recent hashes, start doubling the step - # size. - step *= 2 - - # Always include the genesis block, if available - if len(self.hashes_by_height) > 0: - locator.append(self.hashes_by_height[0]) - - return locator - - def blocks_after_locator(self, locator_hashes): - """ - Takes a "block locator" in the form of a list of hashes. The first - hash is the most recent block that another node has, and subsequent - hashes in the locator are the hashes of earlier blocks, with exponential - backoff. - - We proceed back through our longest fork until we find a block that the - remote node mentions (or we get to the genesis block), and return a list - of the hashes of each of the blocks proceeding upwards from there, in - forward order. - - """ - - # Complicated traversal: need to lock. - with self.lock: - - # Process the locator hashes into a set for easy membership checking - mentioned = set(locator_hashes) - - # This holds whether we found a block they mentioned. If we do find - # one, we can update them from there. If not, we will have to start - # from our genesis block. - found = False - - for block_hash, index_from_end in enumerate(reversed( - self.hashes_by_height)): - - # Go through the longest chain backwards - if block_hash in mentioned: - # This block was mentioned by them. We can update them from - # here. - found = True - break - - if found: - # Get the index from the start of the list of the earliest block - # they have not heard of (the block after index_from_end). - # Because of Python indexing, we don't need to add 1 here. - index = len(self.longest_chain) - index_from_end - else: - # Start from our genesis block. We didn't find anything they - # mentioned. - index = 0 - - # Return a list of all the hashes from that point onwards - return self.hashes_by_height[index:] - - def make_block(self, next_state, payload): - """ - Create a block that would, if solved, put the system into the given next - state, and add the given payload onto the longest chain. - - Does not verify the payload. - - Returns None if the Blockchain is not up to date with the current system - state and thus not able to mine. - - """ - - with self.lock: - - if not self.state_available: - # We can't mine without the latest State - return None - - if not self.next_block_target_available(self.highest_block): - # We're up to date with the blockchain, but we don't know what - # the next target should be. Probably we don't have the block we - # need the time from to compute retargeting. - return None - - if self.highest_block is None: - # This has to be a genesis block. Don't fill in a body hash or - # nonce yet. - new_block = Block("\0" * 64) - - # Now give it a body with the target, height, state hash, and - # payload - new_block.add_body(self.next_block_target(None), 0, - next_state.get_hash(), payload) - else: - # We're adding on to an existing chain. - - # Even if our clock is behind the timestamp of the block, don't - # try to generate a block timestamped older than the parent - # block, because that would be invalid. - min_timestamp = self.highest_block.timestamp - - # Make a new block without filling body hash or nonce - new_block = Block(self.highest_block.block_hash()) - - # Add the body, including previous block hash, height, state - # hash, payload, and a timestamp that will hopefully be valid to - # the network even if our clock is off. - new_block.add_body(self.next_block_target(self.highest_block), - self.highest_block.height + 1, next_state.get_hash(), - payload, timestamp=max(int(time.time()), min_timestamp)) - - return new_block - - def dump_block(self, block): - """ - Turn a Block into a string. If your Blockchain has interesting info in - the payload (like transactions), override this. - - """ - - return str(block) - - def can_verify_block(self, next_block): - """ - Return True if we can determine whether the given block is valid, based - on the blocks we already have. Returns False if we don't have the parent - blocks needed to verify this block. - - """ - - with self.lock: - # We're going to touch our databases, so we have to lock. - - if next_block.previous_hash == "\0" * 64: - # It's a genesis block. We can always check those. - return True - - # The block store never contains any blocks we can't verify, so if - # we have the parent, we can verify this block. - return next_block.previous_hash in self.blockstore - - def verify_block(self, next_block): - """ - Return True if the given block is valid based on the parent block it - references, and false otherwise. We can't just say new blocks are - invalid if a longer chain exists, though, because we might be getting a - new chain longer than the one we have. - - Do not call this unless can_verify_block() is true for the block. - - You probably want to override verify_payload instead of this. - - """ - - with self.lock: - - if not next_block.has_body: - # It's a mini-block. - if self.minification_time is None: - # We aren't running with mini blocks. Don't accept any. - logging.warning("Mini-blocks not allowed in standard " - "blockchain.") - return False - - elif next_block.previous_hash == "\0" * 64: - # We always need to accept mini genesis blocks - return True - elif not self.blockstore[next_block.previous_hash].has_body: - # We also always need to accept mini-blocks on top of other - # mini-blocks - return True - else: - # We can run a special set of checks to validate mini- - # blockchain blocks. They only have a nonce, previous_hash, - # and state_hash. - - # Most of the time we need to assume mini-blocks are valid: - # we don't have their state changes, so we can't say their - # state hashes are wrong, and we don't have their - # difficulty, so we can't say their nonces aren't good - # enough. So if someone comes alogn with a new mini fork, we - # just have to keep it. - - # TODO: We *could* check difficulty levels if it was within - # the retargeting period of somewhere where we know what the - # difficulty should be (a genesis mini-block, or a non-mini - # block). Check in those cases. - - # However, we can say mini-bocks are invalid if they are on - # top of blocks that are too recent to have ben minified. If - # we are current with the latest full block, we shouldn't - # accept a mini-block on top of it. We should only accept a - # mini-block on top of a full block if we're plausibly a - # whole blockchain cycle behind. - - # This check is necessarily heuristic, but it will only - # complain that mini-blocks are invalid for at most a bit - # less than a cycle's worth of real time. And it will only - # say legitimate mini-blocks are invalid if this node has - # gotten an entire cycle's-worth of time behind in much less - # time than a cycle should have taken. And if we do - # eventually say bogus mini-blocks are valid on top of real - # blocks, we will learn otherwise when we make contact with - # the main chain. - - # Basically, this check prevents someone from broadcasting a - # bogus mini-block and then a real block saying whatever - # they want, and convincing all the miners currently mining - # that they have gotten behind and need to skip ahead to the - # new bogus block. - - # How old does the parent block have to be before we believe - # there should be mini-blocks on top of it? Say half of a - # blockchain cycle if we're generating blocks at the optimum - # rate. If you go half a blockchain cycle's worth of time - # without any new blocks, too bad. - min_age_for_miniblocks = 0.5 * self.minification_time * \ - self.retarget_time / self.retarget_period - - now = int(time.time()) - - # How old is the parent? - parent_age = (now - - self.blockstore[next_block.previous_hash].timestamp) - - if parent_age < min_age_for_miniblocks: - # Parent too young to have a mini block on it. - logging.warning("Mini-block not allowed on parent that " - "is only {}/{} seconds old".format(parent_age, - min_age_for_miniblocks)) - return False - - else: - # The parent is old enough for a mini-block to be on top - # of it. Not really much else to check. TODO: minimum - # work? - - return True - - # If we get here, we know we're not checking a mini-block. - - if ((next_block.height == 0) != - (next_block.previous_hash == "\0" * 64)): - # Genesis blocks must have height 0 and a previous hash of all - # 0s. If we have exactly one of those things, the block is - # invalid. - logging.warning("Genesis block height and hash are mismatched.") - return False - - # Get the previous block, or None if this is a genesis block - previous_block = None - if next_block.height != 0: - # This isn't a genesis block, so load its parent, which we must - # have. (Otherwise can_verify_block would be false.) - previous_block = self.blockstore[next_block.previous_hash] - - # Get the state after it. - state = self.state_after(previous_block) - if state is not None: - # We have the previous state, so we can check if this block steps - # forwards correctly. - try: - state.step_forwards(next_block) - except BaseException: # We didn't get to the state we were supposed to - logging.warning(traceback.format_exc()) - logging.warning("Block claims to move to incorrect state.") - return False - - now = int(time.time()) - if next_block.timestamp > (now + 10 * 60): - # The block is from more than 10 minutes in the future. - logging.warning("Block is from too far in the future: {} vs " - "{}".format(pybc.util.time2string(next_block.timestamp), - pybc.util.time2string(now))) - return False - - if (previous_block is not None and previous_block.has_body and - previous_block.timestamp > next_block.timestamp): - # The block is trying to go back in time. - logging.warning("Block is timestamped earlier than parent.") - return False - - if (previous_block is not None and previous_block.has_body and - next_block.height != previous_block.height + 1): - # The block is not at the correct height (1 above its parent) - logging.warning("Block height incorrect.") - return False - - if (self.next_block_target_available(previous_block) and - next_block.target != self.next_block_target(previous_block)): - # We have some idea of what the target should be next, and this - # block is lying about it. - - # The block isn't valid if it cheats at targeting. - logging.warning("Block target incorrect. Should be {}".format( - pybc.util.bytes2hex(self.next_block_target( - previous_block)))) - return False - - if not next_block.verify_work(self.algorithm): - # The block also isn't valid if the PoW isn't good enough - logging.warning("Block PoW isn't correct.") - return False - - if not self.verify_payload(next_block): - # The block can't be valid if the payload isn't. - logging.warning("Block payload is invalid.") - return False - - # Block is valid - logging.debug("Block is valid") - return True - - def verify_payload(self, next_block): - """ - Return True if the payload of the given next block is valid, and false - otherwise. - - Should be overridden to define payload logic. The default implementation - accepts any payloads. - - """ - - return True - - def next_block_target_available(self, previous_block): - """ - Return True if we can calculate the appropriate proof of work target for - the block after the given Block, or False if we can't. - - If the given block is None, the next block is a genesis block, so we can - calculate the target. - - Otherwise, we need previous_block to be unminified. And if - previous_block is at a height that is a multiple of - self.retarget_period, we need the block self.retarget_period blocks - before that to be unminified. - - """ - - if previous_block is None: - # We know it for genesis blocks - return True - - elif not previous_block.has_body: - # Can't get block target on top of a mini block - return False - elif (previous_block.height > 0 and previous_block.height % - self.retarget_period == 0): - - # We need to d a retargeting. We also need the block retarget_period - # ago to be unminified. - - # Go get the time of the block retaregt_preiod blocks ago - block = previous_block - for _ in range(self.retarget_period): - # Go back a block retarget_period times. - # We always have all the blocks, so this will work. - block = self.blockstore[block.previous_hash] - - if not block.has_body: - # We won't be able to know how long the blocks took - return False - - # If we get here, we have our parent unminified and, if needed, we also - # have the block retarget_period before that unminified. - return True - - def get_average_block_time(self): - """ - Compute and return the average block time for the last retarget_period - blocks, if possible. If not possible, return None. - - """ - - with self.lock: - - if len(self.hashes_by_height) < self.retarget_period: - # We don't have blocks old enough. - - return None - - # What block hash do we want to get the timestamp from? - old_block_hash = self.hashes_by_height[-self.retarget_period] - - # What Block is it? - old_block = self.blockstore[old_block_hash] - - if not old_block.has_body: - # We dropped the timestamp, or don't have it - return None - - # What time is it? - now = int(time.time()) - - # The average block time is the time since the first block divided - # by the number of blocks. Return that. - return float(now - old_block.timestamp) / self.retarget_period - - def next_block_target(self, previous_block): - """ - Get the PoW target (64 bytes) that the next block must use, based on the - given previous block Block object (or None). - - The prevous block may not be a mini-block. - - Should be overridden to define PoW difficulty update logic. - """ - - # Lock since we use the blockstore - with self.lock: - - if previous_block is None: - # No blocks yet, so this is the starting target. You get a 0 bit - # as the first bit instead of a 1 bit every other hash. So to - # get n leading 0 bits takes on average 2^n hashes. n leading - # hex 0s takes on average 16^n hashes. - return struct.pack(">Q", 0x00000fffffffffff) + "\xff" * 56 - else: - # Easy default behavior: re-target every 10 blocks to a rate of - # 10 block per minute, but don't change target by more than - # 4x/0.25x - - if (previous_block.height > 0 and - previous_block.height % self.retarget_period == 0): - # This is a re-target block. It's on a multiple of - # retarget_period and not the genesis block. - - # Go get the time of the block retaregt_preiod blocks ago - block = previous_block - for _ in range(self.retarget_period): - # Go back a block retarget_period times. - # We always have all the blocks, so this will work. - block = self.blockstore[block.previous_hash] - - if not block.has_body: - # We can't measure how long the last set of blocks took, - # because the block before then has been minified and - # lost its timestamp. - raise Exception("Block {}, which we need to calculate " - "retarget time, has been minified.".format( - pybc.bytes2string(block.block_hash()))) - - old_time = block.timestamp - new_time = previous_block.timestamp - - # We want new_time - old_time to be retarget_time seconds. - time_taken = new_time - old_time - ideal_time = self.retarget_time - - logging.debug("{} blocks took {}, should have taken " - "{}".format(self.retarget_period, time_taken, - ideal_time)) - - # At constant hashrate, the generation rate scales linearly - # with the target. So if we took a factor of x too long, - # increasing the target by a factor of x should help with - # that. - factor = float(time_taken) / ideal_time - - logging.debug("Want to scale generation rate by {}".format( - factor)) - - # Don't scale too sharply. - if factor > 4: - factor = 4 - if factor < 0.25: - factor = 0.25 - - logging.debug("Will actually scale by: {}".format(factor)) - - # Load the target as a big int - old_target = pybc.util.bytes2long(previous_block.target) - - logging.debug("{} was old target".format( - pybc.util.bytes2hex(previous_block.target))) - - # Multiply it - new_target = int(old_target * factor) - - logging.debug("new / old = {}".format(new_target / - old_target)) - logging.debug("old / new = {}".format(old_target / - new_target)) - - new_target_bytes = pybc.util.long2bytes(new_target) - - while len(new_target_bytes) < 64: - # Padd to the appropriate length with nulls - new_target_bytes = "\0" + new_target_bytes - if len(new_target_bytes) > 64: - # New target would be too long. Don't change - logging.debug("No change because new target is too " - "long.") - return previous_block.target - - logging.debug("{} is new target".format(pybc.util.bytes2hex( - new_target_bytes))) - - return new_target_bytes - - else: - # If it isn't a retarget, don't change the target. - return previous_block.target - - def switch_forks(self, new_tip): - """ - Switch forks from the current fork in self.highest_block and - self.hashes_by_height to the one with the tip Block new_tip. The new - fork may be higher or lower than the old one in absolute block count, - but must be higher in work points. - - Make sure self.hashes_by_height has all the hashes of the blocks - on the new fork, and that these changes are sent to the fork database. - - Also, change our State. - - If we need to cross mini-blocks in order to get to the new fork, we - can't just walk our current state there. Our State will then be marked - for re-download. - - """ - - # Strategy: - # Change our State. - # Find the common ancestor of highest_block and new_tip - # Make sure hashes_by_height has enough spots - # Fill them in from new_tip back to the common ancestor. - - # This is incredibly complicated - with self.lock: - - # Get the sate we should have. If the State can't be calculated, - # this will return None - new_state = self.state_after(new_tip, notify=True) - - if new_state is None: - # We're trying to change forks over mini-blocks. We can't take - # our state with us. Say we need to download a new state, and - # that nobody should use this one until then. - self.invalidate_state() - else: - # Keep the new state - self.state = new_state - - # Find the common ancestor - ancestor = self.common_ancestor(new_tip, self.highest_block) - - if ancestor is None: - # Go back through the whole list, replacing the genesis block. - ancestor_hash = "\0" * 64 - else: - # Go back only to the block on top of the common ancestor. - ancestor_hash = ancestor.block_hash() - - # Make sure hashes_by_height is big enough - - while len(self.hashes_by_height) <= new_tip.height: - # Make empty spots in the hashes by height list until - # hashes_by_height[new_tip.height] is a valid location. - self.hashes_by_height.append(None) - - while len(self.hashes_by_height) > new_tip.height + 1: - # We are switching over to a fork that's lower in actual blocks - # but higher in points. Trim hashes_by_height to be exactly the - # right height. - self.hashes_by_height.pop() - - # Now go back through the blocks in the new fork, filling in - # hashes_by_height. - - # This holds the block we are to put in next - position = new_tip - - # And its height - position_height = new_tip.height - - while (position is not None and - position.block_hash() != ancestor_hash): - # For every block on the new fork back to the common ancestor, - # stick its hash in the right place in hashes by height. - self.hashes_by_height[position_height] = position.block_hash() - - if (self.minification_time is not None and - len(self.hashes_by_height) - position_height > - self.minification_time): - # This block (whether it's already mini or not) is deep - # enough for explicit minification. Record that. - - logging.warning("Minifying block {} burried at height " - "{} vs tip {}".format( - pybc.bytes2string(position.block_hash()), - position_height, len(self.hashes_by_height))) - - # A block burried this deep ought to be minified, and this - # one isn't. Take care of that. - position.remove_body() - # Store the minified block back in the block store. - self.blockstore[position.block_hash()] = position - - if position.previous_hash != "\0" * 64: - # Go back to the previous block and do it - position = self.blockstore[position.previous_hash] - position_height -= 1 - else: - # We've processed the genesis block and are done. - position = None - - # Save our hashes by height changes to the fork database, pending a - # sync. - self.fork_data["hashes_by_height"] = self.hashes_by_height - - def queue_block(self, next_block, callback=None): - """ - We got a block that we think goes in the chain, but we may not have all - the previous blocks that we need to verify it yet. - - Put the block into a receiving queue. - - If the block is eventually verifiable, call the callback with the hash - and True if it's good, or the hash and False if it's bad. - - If the same block is queued multiple times, and it isn't immediately - verifiable, only the last callback for the block will be called. - - If the block is valid but we already have it, and the new block isn't an - improvement on the version we have (i.e. it's a mini-block when we - already have that mini-block, or it's a body for a block we threw out - the body for), does not call the callback at all. - - """ - - # This holds all the callbacks we need to call, so we can call them - # while we're not holding the lock. It's a list of function, argument - # tuple tuples. - to_call = [] - - with self.lock: - - if self.minification_time is None and not next_block.has_body: - # We know right away that we don't want to accept this block. - - logging.debug("Rejecting mini-block on normal blockchain.") - - if callback is not None: - # Fire the callback off right away, since there's only one - callback(next_block.block_hash(), False) - - # Skip out on the rest of the function. Don't put it in the - # queue, since even if we get its parent it won't be acceptable. - return - - if self.has_block(next_block.block_hash()): - # We already have a version of this block. Grab it. - old_version = self.blockstore[next_block.block_hash()] - - if not next_block.has_body or old_version.body_discarded: - # Skip this block because either it's a mini-block for - # something we already have, or it's a body for a block we - # threw out the body of. - - # Don't call the callback at all. - return - - # This is the stack of hashes that we have added to the blockchain. - # We need to check what blocks are wauiting on them. - added_hashes = [] - - if self.can_verify_block(next_block): - # We can verify it right now - if self.verify_block(next_block): - # The block is good! - self.add_block(next_block) - - if callback is not None: - # Later, call the callback. - to_call.append((callback, (next_block.block_hash(), - True))) - - if next_block.has_body: - logging.info("Block height {} immediately verified: " - "{}".format(next_block.height, - pybc.util.bytes2string(next_block.block_hash()))) - else: - logging.info("Mini-block immediately verified: " - "{}".format(pybc.util.bytes2string( - next_block.block_hash()))) - - # Record that we added the block - added_hashes.append(next_block.block_hash()) - else: - logging.warning("Invalid block:\n{}".format(self.dump_block( - next_block))) - if callback is not None: - # Later, call the callback. - to_call.append((callback, (next_block.block_hash(), - False))) - else: - # Add this block to the pending blocks for its parent. If it's - # already there, we just replace it - self.pending_on[next_block.previous_hash][ - next_block.block_hash()] = next_block - # Remember the callback for this block, which chould be called - # when it is verified. - self.pending_callbacks[next_block.block_hash()] = callback - - if next_block.has_body: - # We can report the hash - logging.debug("Block height {}, hash {} pending on parent " - "{}".format(next_block.height, - pybc.util.bytes2string(next_block.block_hash()), - pybc.util.bytes2string(next_block.previous_hash))) - else: - # Say we have a pending mini-block - logging.debug("Mini-block hash {} pending on parent " - "{}".format( - pybc.util.bytes2string(next_block.block_hash()), - pybc.util.bytes2string(next_block.previous_hash))) - - while len(added_hashes) > 0: - # There are blocks that we have added, but we haven't checked - # for other blocks pending on them. Do that. - - # Pop off a hash to check - hash_added = added_hashes.pop() - # Get the dict of waiters by waiter hash - waiters = self.pending_on[hash_added] - - # Remove it - del self.pending_on[hash_added] - - logging.debug("{} waiters were waiting on {}".format( - len(waiters), pybc.util.bytes2string(hash_added))) - - for waiter_hash, waiter in six.iteritems(waiters): - # We ought to be able to verify and add each waiter. - - # Get the callback - waiter_callback = self.pending_callbacks[waiter_hash] - - # Remove it from the collection of remaining callbacks - del self.pending_callbacks[waiter_hash] - - if self.can_verify_block(waiter): - # We can verify the block right now (ought to always be - # true) - if self.verify_block(waiter): - # The block is good! Add it - self.add_block(waiter) - - if waiter_callback is not None: - # Call the callback later - to_call.append((waiter_callback, - (waiter_hash, True))) - - if waiter.has_body: - logging.info("Queued block height {} verified: " - "{}".format(waiter.height, - pybc.util.bytes2string(waiter_hash))) - else: - logging.info("Queued mini-block verified: " - "{}".format(pybc.util.bytes2string( - waiter_hash))) - - # Record that we added the pending block, so things - # pending on it can now be added - added_hashes.append(waiter_hash) - else: - # TODO: throw out blocks waiting on invalid blocks. - # If we have any of those, there's probablt a hard - # fork. - - logging.warning("Queued block invalid: {}".format( - pybc.util.bytes2string(waiter_hash))) - - if waiter_callback is not None: - # Call the callback later - to_call.append((waiter_callback, (waiter_hash, - False))) - else: - # This should never happen - logging.error("Couldn't verify a waiting block {} when " - "its parent came in!".format( - pybc.util.bytes2string(waiter_hash))) - - # Now we're out of the locked section. - logging.debug("Dispatching {} block validity callbacks.".format(len( - to_call))) - for callback, args in to_call: - # Call all the callbacks, in case they need to get a lock that - # another thread has and that thread is waiting on this thread. - callback(*args) - - def state_after(self, block, notify=False): - """ - Given that our State is currently up to date with our current - highest_block, return a copy of our current State, walked to after the - given block. This copy can be updated safely. - - block may be None, in which case we produce the State before the genesis - block. - - Walks the blockchain from the tip of the highest fork to the given - block, which must be in the block store. - - If notify is True, dispatches "backward" events for each block walked - backward, and "forward" events for each block walked forward. - - If we need to cross mini-blocks to get to the given block, return None - instead of a State. - - TODO: can we do real pathfinding? - - """ - - with self.lock: - - if not self.state_available: - # We have no State that we can walk anywhere. - return None - - # What's the hash of the block? - if block is not None: - block_hash = block.block_hash() - else: - block_hash = "\0" * 64 - - if self.highest_block is None: - if block is None: - # We have no blocks, and we want the state after no blocks. - # This is easy. - return self.state.copy() - elif block.has_body: - logging.debug("Making after-genesis state") - # Easy case: this must be a full genesis block. If it had - # any ancestors, they would be longer than our zero-length - # longest fork. Use the current (empty) state updated with - # the new block, which exists and has a body. - state = self.state.copy() - state.step_forwards(block) - if notify: - self.send_event("forward", block) - return state - else: - # We have a mini-block on top of our genesis state. We don't - # know what the state after that should be. - return None - - if block_hash == self.highest_block.block_hash(): - logging.debug("Using current state.") - # We already have the state after this - return self.state.copy() - - if (block is not None and - block.previous_hash == self.highest_block.block_hash()): - - if block.has_body: - logging.debug("Using parent state") - # Special case: this new block comes directly after our old - # one. - - # Advance the State - state = self.state.copy() - state.step_forwards(block) - if notify: - self.send_event("forward", block) - - # Return it - return state - else: - # Block comes directly after our old top block, but we can't - # take the state there because it's a mini-block. - return None - - logging.debug("Walking blockchain for state") - # If we get here, we know we have a highest block, and that we need - # to walk from there to the block we're being asked about. - - # How many blocks did we walk? - blocks_walked = 0 - - # Find the common ancestor of this block and the tip, where we have - # a known State. - ancestor = self.common_ancestor(block, self.highest_block) - - if ancestor is None: - # This block we want the state after is on a completely - # different genesis block. - - # Go back until the state after nothing, and then go forward - # again. - ancestor_hash = "\0" * 64 - else: - # Go back until the state after the common ancestor, and then go - # forward again. - ancestor_hash = ancestor.block_hash() - - logging.debug("Rewinding to common ancestor {}".format( - pybc.bytes2string(ancestor_hash))) - - # Walk the State back along the longest fork to the common ancestor - # (which may be the current top block). - - # TODO: Is there a more efficient way to do this? - - # This holds our scratch state - state = self.state.copy() - - # This holds the hash of the block we're on - position = self.highest_block.block_hash() - - logging.debug("Starting from {}".format(pybc.bytes2string( - position))) - - while position != ancestor_hash: - # Until we reach the common ancestor... - - if not self.blockstore[position].has_body: - # We can't cros this mini-block with out State - logging.debug("Need to cross mini-block on our fork") - return None - - logging.debug("Walking back to {} at height {}".format( - pybc.bytes2string(self.blockstore[position].previous_hash), - self.blockstore[position].height - 1)) - - # Undo the current block - state.step_backwards(self.blockstore[position]) - - if notify: - self.send_event("backward", self.blockstore[position]) - - # Step back a block - position = self.blockstore[position].previous_hash - blocks_walked += 1 - - if position != "\0" * 64 and self.blockstore[position].has_body: - # We can check the current State hash against the hash we're - # supposed to have. - if state.get_hash() != self.blockstore[position].state_hash: - # We stepped back and got an incorrect state. This - # should never happen because these blocks have already - # been validated. - - self.state.audit() - - raise Exception("Stepped back into wrong state after " - "block {}".format(pybc.bytes2string(position))) - - # Now we've reverted to the post-common-ancestor state. - - # Now apply all the blocks from there to block in forwards order. - # First get a list of them - blocks_to_apply = [] - - if block is None: - # We want the state before any blocks, so start at no blocks and - # go back to the common ancestor (which also ought to be no - # blocks). - position = "\0" * 64 - else: - # We want the state after a given block, so we have to grab all - # the blocks between it and the common ancestor, and then run - # them forwards. - position = block.block_hash() - - while position != ancestor_hash: - # For each block back to the common ancestor... (We know we - # won't go off the start of the blockchain, since ancestor is - # known to actually be a common ancestor hash.) - - if not self.blockstore[position].has_body: - # We can't cros this mini-block with out State - logging.debug("Need to cross mini-block on other fork") - return None - - # Collect the blocks on the path from block to the common - # ancestor. - blocks_to_apply.append(self.blockstore[position]) - - # Step back a block - position = self.blockstore[position].previous_hash - blocks_walked += 1 - - # Flip the blocks that need to be applied into chronological order. - blocks_to_apply.reverse() - - for block_to_apply in blocks_to_apply: - - logging.debug("Walking forwards to {} at height {}".format( - pybc.bytes2string(block_to_apply.block_hash()), - block_to_apply.height)) - - # Apply the block to the state - state.step_forwards(block_to_apply) - - if notify: - self.send_event("forward", block_to_apply) - - logging.info("Walked {} blocks".format(blocks_walked)) - - # We've now updated to the state for after the given block. - return state - - def genesis_block_for(self, block): - """ - Return the genesis block for the given Block in the blockstore. - - """ - - while block.previous_hash != "\0" * 64: - # This isn't a genesis block. Go back. - block = self.blockstore[block.previous_hash] - - return block - - def common_ancestor(self, block_a, block_b): - """ - Get the most recent common ancestor of the two Blocks in the blockstore, - or None if they are based on different genesis blocks. - - Either block_a or block_b may be None, in which case the common ancestor - is None as well. - - """ - - # This is incredibly complicated, so lock the blockchain data - # structures. - with self.lock: - - if block_a is None or block_b is None: - # Common ancestor with None is always None (i.e. different - # genesis blocks). - return None - - # This holds our position on the a branch. - position_a = block_a.block_hash() - - # This holds all the hashes we visited tracing back from a - hashes_a = set(position_a) - - # This holds our position on the b branch. - position_b = block_b.block_hash() - - # This holds all the hashes we visited tracing back from b - hashes_b = set(position_b) - - # If we do get all the way back to independent genesis blocks, what - # are they? - genesis_a = None - genesis_b = None - - while position_a != "\0" * 64 or position_b != "\0" * 64: - # While we haven't traced both branches back to independent - # genesis blocks... - if position_a != "\0" * 64: - # Trace the a branch back further, since it's not off the - # end yet. - - if self.blockstore[position_a].previous_hash == "\0" * 64: - # We found a's genesis block - genesis_a = self.blockstore[position_a] - - # Move back a step on the a branch - position_a = self.blockstore[position_a].previous_hash - hashes_a.add(position_a) - - if position_a != "\0" * 64 and position_a in hashes_b: - # We've found a common ancestor. Return the block. - return self.blockstore[position_a] - - if position_b != "\0" * 64: - # Trace the b branch back further, since it's not off the - # end yet. - - if self.blockstore[position_b].previous_hash == "\0" * 64: - # We found b's genesis block - genesis_b = self.blockstore[position_b] - - # Move back a step on the b branch - position_b = self.blockstore[position_b].previous_hash - hashes_b.add(position_b) - - if position_b != "\0" * 64 and position_b in hashes_a: - # We've found a common ancestor. Return the block. - return self.blockstore[position_b] - - if genesis_a.block_hash() == genesis_b.block_hash(): - # Double-check the independence of the independent genesis - # blocks. - raise Exception("Stepped back all the way to independent " - "genesis blocks that were actually the same") - - # We've hit independent genesis blocks. There is no common - # ancestor, so return None. - return None - - def add_block(self, next_block): - """ - Add a block as the most recent block in the blockchain. The block must - be valid, or an Exception is raised. - - """ - - # This is complicated. Lock the blockchain - with self.lock: - - if self.verify_block(next_block): - # Tag the block with its cumulative PoW difficulty, in points - points = next_block.get_work_points(self.algorithm) - - if next_block.previous_hash != "\0" * 64: - # We need to add in the points from the block this is on top - # of. We know we added a cumulative points field already. - points += self.blockstore[next_block.previous_hash].points - - # Tack on the points field. It never goes over the network; we - # just use it locally to work out which fork we belong on. - next_block.points = points - - # Put the block in the block store - self.blockstore[next_block.block_hash()] = next_block - - if (next_block.has_body and (self.highest_block is None or - next_block.points > self.highest_block.points)): - - # We know that this new block we just added is not a mini- - # block, so we can switch to it. - - logging.debug("Switching to new top block") - - # This new block is higher (in total work points) than the - # previously highest block. This means we want to change to - # it. Note that doing it this way creates a more well- - # defined way to resolve competing blocks hat both want to - # be the end of the chain: you should take the one with the - # most points. It also means that someone can come along - # with a super-great block that forks off hlfway down the - # chain and make everyone switch over to that. However, the - # expected work you would need to find that would be greater - # than all the work done on the old fork, so it's just as - # hard to do as coming along with a longer chain (i.e. still - # a 51% attack). - - if (self.highest_block is not None and - next_block.previous_hash != - self.highest_block.block_hash()): - - logging.debug("Switching forks") - - # We had an old highest block, but we need to switch - # forks away from it. - - # This may involve replacing a big part of our - # hashes_by_height list and updating our State in a - # complex way. We have a function for this. - self.switch_forks(next_block) - - elif (next_block.previous_hash != "\0" * 64 and - not self.blockstore[next_block.previous_hash].has_body): - # This is the first non-mini block since the genesis - # block. We need to fill in hashes_by_height, and note - # that we lack a valid State currently, since we had to - # cross mini-blocks to get from genesis to here. - - logging.debug("Switching to first real block after " - "mini-genesis block") - - while (len(self.hashes_by_height) < - next_block.height + 1): - # Make enough slots in hashes_by_height - self.hashes_by_height.append(None) - - # We need a position to walk back and fill in - # hashes_by_height. - position = next_block.block_hash() - - # And a height since we're crossing mini-blocks - height = next_block.height - - while position != "\0" * 64: - # Fill in the correct hashes_by_height spot - self.hashes_by_height[height] = position - # What block are we on now? - block = self.blockstore[position] - # Go to its parent - position = block.previous_hash - height -= 1 - - # Save the updated hashes_by_height - self.fork_data["hashes_by_height"] = \ - self.hashes_by_height - - # Say we lack a state - self.invalidate_state() - - else: - # This is a direct child of the old highest block, or a - # new genesis block when we didn't have one before. - - logging.debug("Switching to child of old top block") - - # Put this block on the end of our hashes by height list - self.hashes_by_height.append(next_block.block_hash()) - # And save that back to the fork database, pending a - # sync. - self.fork_data["hashes_by_height"] = \ - self.hashes_by_height - - if (self.minification_time is not None and - len(self.hashes_by_height) > - self.minification_time): - - # There are enough blocks that we should start - # minifying old ones. What block should we minify? - to_minify = self.blockstore[self.hashes_by_height[ - -self.minification_time - 1]] - - logging.warning("Minifying block {} at depth " - "{}".format(pybc.bytes2string( - to_minify.block_hash()), - self.minification_time)) - - # Minify the block - to_minify.remove_body() - - # Save it back - self.blockstore[to_minify.block_hash()] = to_minify - - if self.state_available: - # We know the last State, so we should keep it - # current. - - # Update the State in place with a simple step. - # There can't be any other State copies floating - # around at this point (we don't call state_after, - # since we decided not to use switch_forks). - self.state.step_forwards(next_block) - - # Notify our listeners of a new block - self.send_event("forward", next_block) - - # Set the highest block to the new block. - self.highest_block = next_block - - # Put the new highest block's hash into the fork database - # "under highest". - self.fork_data["highest"] = self.highest_block.block_hash() - - # We sync to disk periodically; we want to avoid syncing to - # disk after every block. This means that States may need to - # use their slow shallow copy of shallow copy path for a - # while until the periodic commit timer ticks, but it's much - # faster than syncing on every downloaded block. - - # TODO: Separate committing (get off the potentially - # O(updates since last commit) copy of copy path) and - # syncing (write all blocks, fork data, and state to disk). - - if self.state_available: - # Now there is a new block on the longest chain, and we - # are up to date with its State. Throw out all - # transactions that are now invalid on top of it. - - # First, make a clean new State for verifying - # transactions against. - self.transaction_state = self.state.copy() - - # Keep a list of the hashes of invalid transactions that - # don't make it after this block. Note that transactions - # are tested in arbitrary order here, not the order we - # saw them or an order that will make the most valid or - # anything. This means everything may be slow if - # transactions on the same block can vary in validity by - # order. - invalid_transaction_hashes = [] - - for tr_hash, transaction in six.iteritems(self.transactions): - if not self.verify_transaction(transaction, - self.highest_block, self.transaction_state, - advance=True): - - # We only see the transactions that don't make - # it. The others quietly update - # transaction_state, so when new transactions - # come in it will reflect the state they have to - # deal with. - - # Mark this transaction as invalid. - invalid_transaction_hashes.append(tr_hash) - - # Now self.transaction_state is up to date with all the - # queued transactions. - - # Then, remove all the invalid transactions we found. - for transaction_hash in invalid_transaction_hashes: - del self.transactions[transaction_hash] - - logging.info("Dropped {} invalid queued " - "transactions".format(len( - invalid_transaction_hashes))) - - else: - # There is a new full block on the longest chain, but we - # aren't up to date with our State. Tell our - # StateMachine that it needs to download the State for - # this block, potentially interrupting its old download, - # but re-using some of its parts. - self.state_machine.download( - self.highest_block.state_hash) - - # So now we have a new full top block. Can we call the - # blockchain "downloaded"? - - # How old is he block - block_age = time.time() - next_block.timestamp - - if (block_age < 2 * float(self.retarget_time) / - self.retarget_period): - - # The block is less than two average block times old. - # Call the blockchain downloaded. - pybc.science.stop_timer("blockchain_download") - - else: - # Block we tried to add failed verification. Complain. - raise Exception("Invalid block (current state {}): {}".format( - pybc.util.bytes2string(self.state.get_hash()), - self.dump_block(next_block))) - - def sync(self): - """ - Save all changes to the blockstore to disk. Since the blockstore is - always in a consistent internal state when none of its methods are - executing, just call this periodically to make sure things get saved. - - """ - - with self.lock: - # TODO: Implement a state after cache. Clear it here. - - # Save the state. Invalidates any shallow copies, and probably - # confuses anyone iterating over it. - self.state.commit() - # Save the actual blocks. Hopefully these are in the same SQLite - # database as the state, so they can't get out of sync. - self.blockstore.sync() - # Save the metadata about what fork we're on, which depends on - # blocks. TODO: It's in the same database as the blockstore, so this - # is redundant; everything goes through the same underlying - # connection and uses the same transaction. - self.fork_data.sync() - - # Tell all our listeners to save to disk so there is less risk of - # getting out of sync with us and needing a reset. - self.send_event("sync", None) - - def longest_chain(self): - """ - An iterator that goes backwards through the currently longest chain to - the genesis block. - - """ - # Locking things in generators is perfectly fine, supposedly. - with self.lock: - - # This is the block we are currently on - current_block = self.highest_block - if current_block is None: - return - - yield current_block - - # This holds the hash of the prevous block - previous_hash = current_block.previous_hash - - while previous_hash != "\0" * 64: - # While the previous hash is a legitimate block hash, keep - # walking back. - current_block = self.blockstore[previous_hash] - yield current_block - previous_hash = current_block.previous_hash - - def verify_transaction(self, transaction_bytes, chain_head, state, - advance=False): - """ - Returns True if the given transaction is valid on top of the given block - (which may be None), when the system is in the given State. Valid - transactions are broadcast to peers, while invalid transactions are - discarded. - - State may be None, in which case only the well-formedness of the - transaction is verified. - - If advance is true, advances the State with the transaction, if the - State supports it and the transaction is valid. - - *Technically* we really ought to only look at the state, but some things - (like the current height) are easy to get from blocks. - - We need to be able to tell if a transaction is valid on non-longest - forks, because otherwise we won't be able to verify transactions in - blocks that are making up a fork that, once we get more blocks, will - become the longest fork. (The default Blockchain implementation doesn't - actually store transactions in blocks, but we need to have a design that - supports it.) - - Along with verify_payload, subclasses probably ought to override this to - specify application-specific behavior. - - """ - - return True - - def transaction_valid_for_relay(self, transaction): - """ - Returns True if we should accept transactions like the given transaction - from peers, False otherwise. - - If you are making a system where block generators put special - transactions into blocks, you don't want those transactions percolating - through the network and stealing block rewards. - - """ - - return True - - def get_transaction(self, transaction_hash): - """ - Given a transaction hash, return the transaction (as a bytestring) with - that hash. If we don't have the transaction, return None instead of - throwing an error (in case the transaction gets removed, perhaps by - being added to a block). - - """ - with self.lock: - if transaction_hash in self.transactions: - return self.transactions[transaction_hash] - else: - return None - - def get_transactions(self): - """ - Iterate over all pending transactions as (hash, Transaction object) - tuples. - - """ - - with self.lock: - for transaction_hash, transaction in six.iteritems(self.transactions): - yield transaction_hash, transaction - - def has_transaction(self, transaction_hash): - """ - Return True if we have the transaction with the given hash, and False - otherwise. The transaction may go away before you can call - get_transaction. - - """ - - # No need to lock. This is atomic and hits against an in-memory dict - # rather than a persistent one. - return transaction_hash in self.transactions - - def add_transaction(self, transaction, callback=None): - """ - Called when someone has a transaction to give to us. Takes the - transaction as a string of bytes. Only transactions that are valid in - light of currently queued transactions are acepted. - - Calls the callback, if specified, with the transaction's hash and its - validity (True or False). If the transaction is not something we are - interested in, doesn't call the callback at all. - - We don't queue transactions waiting on the blocks they depend on, like - we do blocks, because it doesn't matter if we miss having one. - - If the transaction is valid, we will remember it by hash. - - If the blockchain is not up to date with the State for the current top - block, no new transactions will be accepted. - - """ - - with self.lock: - # Hash the transaction - transaction_hash = hashlib.sha512(transaction).digest() - - if transaction_hash in self.transactions: - # We already know about this one. Don't bother trying to verify - # it. It won't verify twice, and will just produce console spam. - return - - if len(self.transactions) >= 100: - # This is too many. TODO: make this configurable - logging.info("Rejecting transaction due to full queue") - return - - if not self.state_available: - # We can't trust our stored state, or the transaction_state that - # comes from it. - verified = False - elif self.verify_transaction(transaction, self.highest_block, - self.transaction_state, advance=True): - - # The transaction is valid in our current fork. - # Keep it around. - self.transactions[transaction_hash] = transaction - - # Our transaction_state has automatically been advanced. - - # Record that it was verified - verified = True - - else: - # Record that it wasn't verified. - verified = False - logging.warn('Invalid transaction: {}'.format(pybc.util.bytes2string(transaction_hash))) - - # Notify the callback outside the critical section. - if callback is not None: - callback(transaction_hash, verified) - - def invalidate_state(self): - """ - Mark the current State as unavailable, and set up a StateMachine that - can be used to bring it up to date. - - While state_available is False, our State will not change, so it's OK - for the StateMachine to use it as a local source of StateComponents - during the download process. - - """ - with self.lock: - # Mark the State as stale and unuseable by anyone - self.state_available = False - - # Save this, pending a sync. - self.fork_data["state_available"] = self.state_available - - # Make the old State make us a StateMachine that uses it as a local - # store and uses the appropriate StateComponent for deserialization. - self.state_machine = self.state.make_state_machine() - - # At some point we will need to fill in the root that we need to - # download. - logging.info("Invaidated current State; need to re-download") - - def validate_state(self): - """ - Check that the current State is correct for the top block that we are - on, and mark it as available, discarding any StateMachine we have. The - current highest_block may not be None. - - """ - - with self.lock: - - if self.state.get_hash() != self.highest_block.state_hash: - logging.critical("Downloaded state hash: {}".format( - pybc.bytes2string(self.state.get_hash()))) - logging.critical("Expected state hash: {}".format( - pybc.bytes2string(self.highest_block.state_hash))) - raise Exception("Downloaded invalid or wrong state!") - - # Say our State is valid - self.state_available = True - - # Save that - self.fork_data["state_available"] = self.state_available - - # Discard the StateMachine - self.state_machine = None - - logging.info("Successfully updated to state {}".format( - pybc.bytes2string(self.state.get_hash()))) - - # Tell our listeners about our newly downloaded state - self.send_event("reset", self.state) - - def get_requested_state_components(self): - """ - If the State is currently unavailable, get a set of StateComponents that - we need to download. This set is guaranteed not to be modified. - - Otherwise, return an empty set. - - """ - - with self.lock: - if self.state_available: - # We catually don't need anything - return set() - - # Let the StateMachine think about what it wants to download - self.state_machine.tick() - - # Get the set of requests - requests = self.state_machine.get_requests() - - logging.debug("Currently trying to get {} StateComponents".format( - len(requests))) - - # Copy the set before returning it, in case new StateComponents come - # in while the caller is working with it. It will never be very big. - return set(requests) - - def get_state_hash(self): - """ - Return the hash of the current State. - - """ - - with self.lock: - - return self.state.get_hash() - - def get_state_component(self, component_hash): - """ - Return the bytrestring for the component with the given hash, or None if - we don't have one. - - """ - - with self.lock: - - return self.state.get_component(component_hash) - - def add_state_component(self, component_bytestring): - """ - Give a StateComponent bytestring to the Blockchain in order to try and - re-build its out of date State. - - """ - - with self.lock: - - if self.state_machine is not None: - # Feed the component to the StateMachine - self.state_machine.add_component(component_bytestring) - - # Let the StateMachine think about it, and potentially complete. - self.state_machine.tick() - - if self.state_machine.is_done(): - # The StateMachine is done! Re-build our State from it and - # get rid of it. - - # The StateComponent must have been set to download the - # state for our current top block, which must exist. - - # Update our State to be in sync with our current top block. - pybc.science.start_timer("state_update") - self.state.update_from_components( - self.state_machine.get_components(), - self.highest_block.state_hash) - pybc.science.stop_timer("state_update") - - # Check it and save it as valid, discarding the StateMachine - self.validate_state() - - else: - logging.warning("Disacrded StateComponent since we're not " - "downloading state.") - - def subscribe(self, listener): - """ - Subscribe a listener to state changes on this Blockchain. - - Listener will be called with: - "forward", block - State has been advanced forward with the given block - - "backward", block - State has been advanced backward with the given - block - - "reset", state - State has been reset to a new, disconnected state - - "sync", None - Blockchain has saved state to disk, and things that - depend on it should do likewise to avoid needing to walk all unspent - outputs. - - The listener will be run under the Blockchain's lock, so it is safe to - iterate over the state, if one is passed. However, it is not safe to - look at fields of the Blockchain, as they may be in an inconsistent - state. - - """ - - with self.lock: - # Add the listener to our list of listeners - self.listeners.append(listener) - - def send_event(self, event, argument): - """ - Dispatch an event to all listeners. Event should be an event name - ("forward", "backward", "reset", or "sync") and argument should be a - Block or State or None, as appropriate. - - """ - - with self.lock: - for listener in self.listeners: - # Call the listener - listener(event, argument) - - def get_disk_usage(self): - """ - Return the disk uage of the Blockchain's blockstore database, in bytes. - Generally, you also put your State's database in here too. - - """ - - with self.lock: - return self.blockstore.get_size() diff --git a/blockchain/pybc/BlockchainProtocol.py b/blockchain/pybc/BlockchainProtocol.py deleted file mode 100644 index 8123434b2..000000000 --- a/blockchain/pybc/BlockchainProtocol.py +++ /dev/null @@ -1,512 +0,0 @@ -""" -BlockchainProtocol.py: contains the BlockchainProtocol class. - -""" - - -from __future__ import absolute_import -import time -import traceback -import collections -import logging -from twisted.internet import reactor # @UnresolvedImport -from twisted.protocols.basic import LineReceiver -from six.moves import map -from six.moves import range - -# Say that we can take up to 10 mb at a time -LineReceiver.MAX_LENGTH = 1024 * 1024 * 10 - -from pybc.Block import Block -from pybc.util import string2bytes, bytes2string - - -class BlockchainProtocol(LineReceiver): - """ - This is a Twisted protocol that exchanges blocks with the peer at the other - end, and floods valid non-block messages (transactons). Also lets state - components be requested. Used by both servers and clients. - - It is a line-oriented protocol with occasional blocks of raw data. - - """ - - def __init__(self, factory, remote_address, incoming=False): - """ - Make a new protocol (i.e. connection handler) belonging to the given - Factory, which in turn belongs to a Peer. - - remote_address should be the (host, port) tuple of the host we are - talking to. - - incoming specifies whether this connection is incoming or not. If it's - not incoming, the port they are using will be advertised. If incoming is - true, this Protocol will kick off the connection by greeting the other - peer. - - """ - - # Keep the factory around, so we can talk to our Peer. - self.factory = factory - - # Remember the address of the host we are talking to - self.remote_address = remote_address - - # Remember if we're supposed to greet or not. - self.incoming = incoming - - # Keep a queue of block hashes to download. Don't get too many at once. - self.inv_queue = collections.deque() - - # Keep a set of things in the inv_queue. - self.inv_queued = set() - - # Keep a watcher callLater to watch it - self.inv_watcher = None - - # Keep a queue of Block objects we have been sent. Don't try and gove - # too many to the blockchain at once. - self.block_queue = collections.deque() - - # Keep a set of block hashes in the block queue - self.block_queued = set() - - # Keep a watcher callLater to watch it - self.block_watcher = None - - def connectionMade(self): - """ - A connection has been made to the remote host. It may be a server or - client. - - """ - - # Let the peer know we are connected. It will tell us if we should keep - # the connection or not. - keep = self.factory.peer.made_connection(self) - - if not keep: - # We alread have too many incoming connections. Drop the connection. - logging.debug("Dropping unwanted connection") - self.disconnect() - return - - # logging.info("Made a connection!") - - if self.incoming: - # They connected to us. Send a greeting, saying what network we're - # in and our protocol version. - self.send_message(["NETWORK", self.factory.peer.network, - self.factory.peer.version]) - - # Make sure to process the inv_queue periodically, to download blocks we - # want. - self.inv_watcher = reactor.callLater(1, self.process_inv_queue) - - # Make sure to process the block_queue periodically, to verify (and - # broadcast) blocks we got. - self.block_watcher = reactor.callLater(1, self.process_block_queue) - - def connectionLost(self, reason): - """ - This connection has been lost. - """ - - # We're no longer a connection that the peer should send things to. - self.factory.peer.lost_connection(self) - - # logging.info("connection LOST with {}".format(self.remote_address)) - - if self.inv_watcher is not None and self.inv_watcher.active(): - # Stop asking for blocks - self.inv_watcher.cancel() - if self.block_watcher is not None and self.block_watcher.active(): - # Stop processing downloaded blocks - self.block_watcher.cancel() - - def disconnect(self): - """ - Drop the connection from the Twisted thread. - - """ - - reactor.callFromThread(self.transport.loseConnection) - - # TODO: Do something to stop us trying to send messages over this - # connection. - - def send_message(self, parts): - """ - Given a message as a list of parts, send it from the Twisted thread. - - """ - - # Compose the message string and send it from the Twisted thread - reactor.callFromThread(self.sendLine, " ".join(map(str, parts))) - - def handle_message(self, parts): - """ - Given a message as a list of string parts, handle it. Meant to run non- - blocking in its own thread. - - Schedules any reply and any actions to be taken to be done in the main - Twisted thread. - """ - - try: - if len(parts) == 0: - # Skip empty lines - return - - if parts[0] == "NETWORK": - # This is a network command, telling us the network and version - # of the remote host. If we like them (i.e. they are equal to - # ours), send back an acknowledgement with our network info. - # Also start requesting peers and blocks from them. - - if (parts[1] == self.factory.peer.network and - int(parts[2]) == self.factory.peer.version): - # We like their network and version number. - # Send back a network OK command with our own. - self.send_message(["NETWORK-OK", self.factory.peer.network, - self.factory.peer.version]) - - # Ask them for peers - self.send_message(["GETADDR"]) - - # Ask them for the blocks we need, given our list of block - # locator hashes. - self.send_message(["GETBLOCKS"] + - [bytes2string(block_hash) for block_hash in - self.factory.peer.blockchain.get_block_locator()]) - - # Send all the pending transactions - for transaction_hash, _ in \ - self.factory.peer.blockchain.get_transactions(): - - self.send_message(["TXINV", bytes2string( - transaction_hash)]) - - else: - # Nope, we don't like them. - - # Disconnect - self.disconnect() - - elif parts[0] == "NETWORK-OK": - # This is a network OK command, telling us that they want to - # talk to us, and giving us their network and version number. If - # we like their network and version number too, we can start - # exchanging peer info. - - if (parts[1] == self.factory.peer.network and - int(parts[2]) == self.factory.peer.version): - - # We like their network and version number. Send them a - # getaddr message requesting a list of peers. The next thing - # they give us might be something completely different, but - # that's OK; they ought to send some peers eventually. - self.send_message(["GETADDR"]) - - # Ask them for the blocks we need, given our list of block - # locator hashes. - self.send_message(["GETBLOCKS"] + - [bytes2string(block_hash) for block_hash in - self.factory.peer.blockchain.get_block_locator()]) - - # Send all the pending transactions - for transaction_hash, _ in \ - self.factory.peer.blockchain.get_transactions(): - - self.send_message(["TXINV", bytes2string( - transaction_hash)]) - else: - # We don't like their network and version. Drop them. - - # Disconnect - self.disconnect() - - elif parts[0] == "GETADDR": - # They want a list of all known peers. - # Send them ADDR messages, one per known peer. - - for host, port, time_seen in self.factory.peer.get_peers(): - if time_seen is None: - # This is a bootstrap peer - continue - # Send the peer's host and port in an ADDR message - self.send_message(["ADDR", host, port, time_seen]) - elif parts[0] == "ADDR": - # They claim that there is a peer. Tell our peer the host and port - # and time seen. - self.factory.peer.peer_seen(parts[1], int(parts[2]), - int(parts[3])) - elif parts[0] == "GETBLOCKS": - # They gave us a block locator. Work out the blocks they need, - # and send INV messages about them. - - # Decode all the hex hashes to bytestring - block_hashes = [string2bytes(part) for part in parts[1:]] - - for needed_hash in \ - self.factory.peer.blockchain.blocks_after_locator( - block_hashes): - - # They need this hash. Send an INV message about it. - # TODO: consolidate and limit these. - self.send_message(["INV", bytes2string(needed_hash)]) - elif parts[0] == "INV": - # They have a block. If we don't have it, ask for it. - - # TODO: allow advertising multiple things at once. - - # Decode the hash they have - block_hash = string2bytes(parts[1]) - - if (self.factory.peer.blockchain.needs_block(block_hash) and - block_hash not in self.inv_queued and - block_hash not in self.block_queued): - - # We need this block, and it isn't in any queues. - self.inv_queue.append(block_hash) - self.inv_queued.add(block_hash) - - elif parts[0] == "GETDATA": - # They want the data for a block. Send it to them if we have - # it. - - # Decode the hash they want - block_hash = string2bytes(parts[1]) - - if self.factory.peer.blockchain.has_block(block_hash): - # Get the block to send - block = self.factory.peer.blockchain.get_block(block_hash) - - # Send them the block. TODO: This encoding is terribly - # inefficient, but we can't send it as binary without them - # switching out of line mode, and they don't know to do that - # because messages queue. - self.send_message(["BLOCK", bytes2string(block.to_bytes())]) - else: - logging.error("Can't send missing block: '{}'".format( - bytes2string(block_hash))) - - elif parts[0] == "BLOCK": - # They have sent us a block. Add it if it is valid. - - # Decode the block bytes - block_bytes = string2bytes(parts[1]) - - # Make a Block object - block = Block.from_bytes(block_bytes) - - if block.block_hash() not in self.block_queued: - # Queue it if it's not already queued. Because of the set it - # can only be queued once. - self.block_queue.append(block) - self.block_queued.add(block.block_hash()) - - elif parts[0] == "TXINV": - # They have sent us a hash of a transaction that they have. If - # we don't have it, we should get it and pass it on. - - # Decode the hash they have - transaction_hash = string2bytes(parts[1]) - - if not self.factory.peer.blockchain.has_transaction( - transaction_hash): - - # We need this transaction! - self.send_message(["GETTX", parts[1]]) - elif parts[0] == "GETTX": - # They want a transaction from our blockchain. - - # Decode the hash they want - transaction_hash = string2bytes(parts[1]) - - if self.factory.peer.blockchain.has_transaction(transaction_hash): - # Get the transaction to send - transaction = self.factory.peer.blockchain.get_transaction( - transaction_hash) - - if transaction is not None: - # We have it (still). Send them the block transaction. - self.send_message(["TX", bytes2string(transaction)]) - else: - logging.error("Lost transaction: '{}'".format( - bytes2string(transaction_hash))) - else: - logging.error("Can't send missing transaction: '{}'".format( - bytes2string(transaction_hash))) - elif parts[0] == "TX": - # They have sent us a transaction. Add it to our blockchain. - - # Decode the transaction bytes - transaction_bytes = string2bytes(parts[1]) - - logging.debug("Incoming transaction.") - - if self.factory.peer.blockchain.transaction_valid_for_relay( - transaction_bytes): - - # This is a legal transaction to accept from a peer (not - # something like a block reward). - logging.debug("Transaction acceptable from peer.") - - # Give it to the blockchain as bytes. The blockchain can - # determine whether to forward it on or not and call the - # callback with transaction hash and transaction status - # (True or False). - self.factory.peer.send_transaction( - transaction_bytes) - - elif parts[0] == "GETSTATE": - # They're asking us for a StateComponent from our State, with - # the given hash. We just serve these like a dumb server. - - # We assume this is under our current State. If it's not, we'll - # return a nort found message and make them start over again - # from the top. - - # What StateComponent do they want? - state_hash = string2bytes(parts[1]) - - # Go get it - state_component = \ - self.factory.peer.blockchain.get_state_component(state_hash) - - if state_component is None: - # Complain we don't have that. They need to start over from - # the state hash in the latest block. - logging.warning("Peer requested nonexistent StateComponent " - "{} vs. root hash {}".format(parts[1], - bytes2string( - self.factory.peer.blockchain.get_state_hash()))) - self.send_message(["NOSTATE", bytes2string(state_hash)]) - else: - - logging.debug("Fulfilling request for StateComponent " - "{}".format(parts[1])) - - # Pack up the StateComponent's data as bytes and send it - # along. They'll know which one it was when they hash it. - self.send_message(["STATE", - bytes2string(state_component.data)]) - - elif parts[0] == "NOSTATE": - # They're saying they don't have a StateComponent we asked for. - logging.warning("Peer says they do not have StateComponent: " - "{}".format(parts[1])) - - elif parts[0] == "STATE": - # They're sending us a StateComponent we probably asked for. - - # Unpack the data - component_bytestring = string2bytes(parts[1]) - - # Give it to the Blockchain. It knows how to handle these - # things. TODO: Maybe put a queue here, since this potentially - # does the whole state rebuilding operation. - self.factory.peer.blockchain.add_state_component( - component_bytestring) - - # Tell the peer to request more StateComponents. - # TODO: This is going to blow up. - self.factory.peer.request_state_components() - - elif parts[0] == "ERROR": - # The remote peer didn't like something. - # Print debugging output. - logging.error("Error from remote peer: {}".format(" ".join( - parts[1:]))) - - else: - # They're trying to send a command we don't know about. - # Complain. - logging.error("Remote host tried unknown command {}".format( - parts[1])) - self.send_message(["ERROR", parts[0]]) - - if not self.incoming: - # We processed a valid message from a peer we connected out to. - # Record that we've seen them for anouncement purposes. - self.factory.peer.peer_seen(self.remote_address[0], - self.remote_address[1], int(time.time())) - - except BaseException: - logging.error("Exception processing command: {}".format(parts)) - logging.error(traceback.format_exc()) - - # Disconnect from people who send us garbage - self.disconnect() - - def lineReceived(self, data): - """ - We got a command from the remote peer. Handle it. - - TODO: Enforce that any of these happen in the correct order. - - """ - - # Split the command into parts on spaces. - parts = [part.strip() for part in data.split()] - - # Handle it in its own thread. This is the only other thread that ever - # runs. - reactor.callInThread(self.handle_message, parts) - - def lineLengthExceeded(self, line): - """ - The peer sent us a line that is too long. - - Complain about it so the user knows that's why we're dropping - connections. - - """ - - logging.error("Peer sent excessively long line.") - - def process_inv_queue(self): - """ - Download some blocks from the queue of blocks we know we need. - - """ - - for _ in range(100): - # Only ask for 100 blocks at a time. - if len(self.inv_queue) > 0: - # Get the bytestring hash of the next block to ask for - block_hash = self.inv_queue.popleft() - self.inv_queued.remove(block_hash) - - if (self.factory.peer.blockchain.needs_block(block_hash) and - block_hash not in self.block_queued): - # We need this block and don't have it in our queue to add - # to the blockchain, so get it. TODO: Check to see if we - # requested it recently. - - self.send_message(["GETDATA", bytes2string(block_hash)]) - - # Process the inv_queue again. - self.inv_watcher = reactor.callLater(1, self.process_inv_queue) - - def process_block_queue(self): - """ - Try to verify some blocks from the block queue. - - """ - - for i in range(10): - if len(self.block_queue) > 0: - # Get the Block object we need to check out. - block = self.block_queue.popleft() - self.block_queued.remove(block.block_hash()) - - # Give it to the blockchain to add when it can. If it does get - # added, we announce it. - self.factory.peer.send_block(block) - else: - break - - # Process the block queue again. - self.block_watcher = reactor.callLater(0.1, self.process_block_queue) diff --git a/blockchain/pybc/ClientFactory.py b/blockchain/pybc/ClientFactory.py deleted file mode 100644 index 73d84902a..000000000 --- a/blockchain/pybc/ClientFactory.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -ClientFactory.py: contains the ClientFactory class. - -""" -from __future__ import absolute_import -import logging -from twisted.internet import protocol - -from pybc.BlockchainProtocol import BlockchainProtocol - - -class ClientFactory(protocol.ClientFactory): - """ - This is a Twisted client factory, responsible for making outgoing - connections and starting Clients to run them. It is part of a Peer. - - """ - - def __init__(self, peer): - """ - Make a new ClientFactory with a reference to the Peer it is producing - outgoing connections for. - """ - # Keep the peer around - self.peer = peer - - def buildProtocol(self, addr): - """ - Make a new client protocol. It's going to be connecting to the given - address. - """ - logging.info("CONNECTED with {}:{}".format(addr.host, addr.port)) - # Make a new BlockchainProtocol that knows we are its factory. It will - # then talk to our peer. - return BlockchainProtocol(self, (addr.host, addr.port)) - - def clientConnectionLost(self, connector, reason): - """ - We've lost a connection we made. - """ - # Record that the outgoing connection is gone - self.peer.lost_outgoing_connection(connector.getDestination().host) - logging.info("connection LOST with {}:{}".format( - connector.getDestination().host, connector.getDestination().port)) - - def clientConnectionFailed(self, connector, reason): - """ - We failed to make an outgoing connection. - """ - # Record that the outgoing connection is gone - self.peer.lost_outgoing_connection(connector.getDestination().host) - logging.info("connection FAILED with {}:{}".format( - connector.getDestination().host, connector.getDestination().port)) diff --git a/blockchain/pybc/Peer.py b/blockchain/pybc/Peer.py deleted file mode 100644 index 733096679..000000000 --- a/blockchain/pybc/Peer.py +++ /dev/null @@ -1,620 +0,0 @@ -""" -Peer.py: contains the Peer class. - -""" - -from __future__ import absolute_import -import random -import time -import threading -import logging -import socket - -from twisted.internet import reactor # @UnresolvedImport -from twisted.internet import endpoints - -from pybc.ClientFactory import ClientFactory -from pybc.ServerFactory import ServerFactory -from pybc.util import time2string, bytes2string -import pybc.science - -from . import sqliteshelf -import six -from six.moves import range - - -class Peer(object): - """ - Represents a peer in the p2p blockchain network. Implements a protocol bsed - on the Bitcoin p2p protocol. Keeps a Blockchain of valid blocks, and handles - downloading new blocks and broadcasting new blocks to peers. - """ - - def __init__(self, network, version, blockchain, peer_file=":memory:", - external_address=None, port=8007, optimal_connections=10, - connections_per_batch=5, tick_period=60, peer_timeout=60 * 60 * 48): - """ - Make a new Peer in the given network (identified by a string), with the - given version integer, that keeps its blocks in the given Blockchain. - Will only connect to other peers in the same network with the same - version integer. - - network must be a printable string with no spaces or newlines. - - version must be an integer. - - peer_file gives a filename to store a persistent peer database in. It - defaults to ":memory:", which keeps the database in memory where it - isn't really persistent. - - external_address, if specified, gives an address or hostname at which we - can announce our presence on every tick. - - port gives a port to listen on. If none is specified, a default - port is used. - - optimal_connections is the number of connections we want to have. We - will periodically try to get new connections or drop old ones. - - TODO: Actually drop connections if we have too many. - - tick_period is how often to tick and ping our peers/broadcast our - address/try to make new connections/drop extra connections, in seconds. - - peer_timeout is how long to remember nodes sicne we last heard from - them. - - The caller needs to run the Twisted reactor before this does anything. - This can be doine through run(). - - """ - - # Remember our network name and version number - self.network = network - self.version = version - - # Save the blockchain - self.blockchain = blockchain - - if external_address is not None: - # We have an external address. Make sure it's an address and - # remember it. - self.external_address = socket.gethostbyname(external_address) - else: - # No external address known. - self.external_address = None - - # Remember our port. We may need to tell people about it if we connect - # to them. - self.port = port - - # Remember our optimal number of connections - self.optimal_connections = optimal_connections - - # And how many we should make per batch - self.connections_per_batch = connections_per_batch - - # Remember our peer remembering timeout - self.peer_timeout = peer_timeout - - # Make an endpoint to listen on - self.endpoint = endpoints.TCP4ServerEndpoint(reactor, port) - - # Make a Twisted ServerFactory - self.server = ServerFactory(self) - - # Make a Twisted ClientFactory to make outgoing connections - self.client = ClientFactory(self) - - # Keep an sqliteshelf of known peers: from host to (port, last-seen) - # tuples, as in BitMessage. - self.known_peers = sqliteshelf.SQLiteShelf(peer_file, table="peers", - lazy=True) - - # Keep a set of IPs we have open, outgoing connections to - self.outgoing_hosts = set() - - # And one of IPs we have active, incomming connections from - self.incoming_hosts = set() - - # Keep a set of open connections (Protocol objects) - self.connections = set() - - # Remember our tick frequency - self.tick_period = tick_period - - # Make a lock so this whole thing can basically be a monitor - self.lock = threading.RLock() - - # Remember if we need to repoll our peers for blocks - self.repoll = False - - # Keep a set of recently requested StateComponents, so we don't ask for - # the same one over and over again. - # TODO: No other queueing stuff is in Peer. Should this be? - self.recently_requested_state_components = set() - - # Remember our total tick count - self.tick_count = 0 - - # A little bit after we get the last block in a string of blocks, check - # to see if we need StateComponents. This holds the Twisted callLater - # call ID. - self.state_download_timer = None - - # Listen with the endpoint, so we can get connections - self.listener = self.endpoint.listen(self.server) - - # Schedule a tick right away. - reactor.callLater(0, self.tick) - - def connect(self, host, port): - """ - Make a new connection to the given host and port, as a client. - - """ - - with self.lock: - # Remember that we are connecting to this host - self.outgoing_hosts.add(host) - - logging.info("Connecting to {} port {}".format(host, port)) - - # Queue making a connection with our ClientFactory in the Twisted - # reactor. - reactor.connectTCP(host, port, self.client) - - def made_connection(self, connection): - """ - Called when a connection is made. Add the connection to the set of - connections. - - Returns True if the connection was added to the set and is to be kept, - or False if we alread have too many connections and want to drop this - one. - - """ - - with self.lock: - if (len(self.connections) > 2 * self.optimal_connections and - connection.incoming): - # Don't accept this incoming connection, we have too many. - return False - else: - # Add the connection to our list of connections - self.connections.add(connection) - - if connection.incoming: - # Remember the host connecting in to us - self.incoming_hosts.add(connection.transport.getPeer().host) - - return True - - def lost_connection(self, connection): - """ - Called when a connection is closed (either normally or abnormally). - Remove the connection from our set of connections. - - """ - - with self.lock: - # Drop the connection from our list of connections - self.connections.discard(connection) - - if connection.incoming: - # This was a connection we got, so throw it out of that set. - self.incoming_hosts.discard(connection.transport.getPeer().host) - - def lost_outgoing_connection(self, host): - """ - Called when an outgoing connection is lost or fails. - Removes that hostname from the list of open outgoing connections. - - """ - - with self.lock: - # No longer connecting to this host - self.outgoing_hosts.discard(host) - - def get_peers(self): - """ - Return a list of (host, port, time seen) tuples for all peers we know - of. - - """ - - with self.lock: - return [(host, host_data[0], host_data[1]) for host, host_data in - six.iteritems(self.known_peers)] - - def peer_seen(self, host, port, time_seen): - """ - Another peer has informed us of the existence of this peer. Remember it, - and broadcast to to our peers. - - """ - - with self.lock: - - if time_seen > int(time.time()) + 10 * 60: - # Don't accept peers from more than 10 minutes into the future. - return - - # Normalize host to an IPv4 address. TODO: do something to recognize - # IPv6 addresses and pass them unchanged. - if host.count(':'): - host = host.split(':')[0] - try: - host = socket.gethostbyname(host) - except: - logging.exception('socket.gethostbyname failed, will use original host value') - key = '{}:{}'.format(host, port) - - if key in self.known_peers: - # We know of this peer, but we perhaps ought to update the last - # heard from time - - # Get the port we know for that host, and the time we last saw - # them - known_port, our_time_seen = self.known_peers[key] - - if (our_time_seen is None or time_seen > our_time_seen): - # They heard from them more recently. Update our time last - # seen. - self.known_peers[key] = (known_port, time_seen) - logging.info('Known peer updated: {}:{}'.format(host, known_port)) - - else: - # This is a new peer. - # Save it. - logging.info('New peer added: {}:{}'.format(host, port)) - self.known_peers[key] = (port, time_seen) - - # Broadcast it - for connection in self.connections: - connection.send_message(["ADDR", host, port, time_seen]) - - def set_repoll(self): - # We saw an unverifiable block. We may be out of date. - # Next time we tick, do a getblocks to all our peers. - - # Too simple to need to lock - self.repoll = True - - def tick(self): - """ - See if we have the optimal number of outgoing connections. If we have - too few, add some. - """ - - with self.lock: - - # How many connections do we have right now? - current_connections = len(self.connections) - logging.info("Tick {} from localhost port {}: {} of {} " - "connections".format(self.tick_count, self.port, - current_connections, self.optimal_connections)) - - for connection in self.connections: - logging.info("\tTo {} port {}".format( - *connection.remote_address)) - - logging.info("{} outgoing connections:".format(len( - self.outgoing_hosts))) - for host in self.outgoing_hosts: - logging.info("\t To {}".format(host)) - - logging.info("Blockchain height: {}".format( - len(self.blockchain.hashes_by_height))) - logging.info("Blocks known: {}".format( - self.blockchain.get_block_count())) - logging.info("Blocks pending: {}".format( - len(self.blockchain.pending_callbacks))) - logging.info("State available for verification and mining:" - " {}".format(self.blockchain.state_available)) - logging.info("Transactions pending: {}".format(len( - self.blockchain.transactions))) - - # Calculate the average block time in seconds for the last few - # blocks. It may be None, meaning we don't have enough history to - # work it out. - block_time = self.blockchain.get_average_block_time() - if block_time is not None: - logging.info("Average block time: {} seconds".format( - block_time)) - - logging.info("Blockchain disk usage: {} bytes".format( - self.blockchain.get_disk_usage())) - - if self.tick_count % 60 == 0: - # Also log some science stats every 60 ticks (by default, 1 hour) - pybc.science.log_event("chain_height", - len(self.blockchain.hashes_by_height)) - pybc.science.log_event("pending_transactions", - len(self.blockchain.transactions)) - pybc.science.log_event("connections", current_connections) - if block_time is not None: - # We have an average block time, so record that. - pybc.science.log_event("block_time", block_time) - - # We want to log how much space our databases (block store and - # state) take. - pybc.science.log_event("blockchain_usage", - self.blockchain.get_disk_usage()) - pybc.science.log_filesize("blockchain_file_usage", - self.blockchain.store_filename) - - # Request whatever state components we need, egardless if whether we - # recently requested them, in case our download got stalled - self.recently_requested_state_components = set() - self.request_state_components() - - if (len(self.outgoing_hosts) < self.optimal_connections and - len(self.known_peers) > 0): - # We don't have enough outgoing connections, but we do know some - # peers. - - for i in range(min(self.connections_per_batch, - self.optimal_connections - len(self.outgoing_hosts))): - # Try several connections in a batch. - - for _ in range(self.optimal_connections - - len(self.outgoing_hosts)): - - # For each connection we want but don't have - - # Find a peer we aren't connected to and connect to them - key = random.sample(self.known_peers, 1)[0] - if key.count(':'): - host = key.split(':')[0] - else: - host = key - - # Try at most 100 times to find a host we aren't - # connected to, and which isn't trivially obviously us. - attempt = 1 - while (host in self.outgoing_hosts or - host in self.incoming_hosts or - host == self.external_address) and attempt < 100: - # Try a new host - key = random.sample(self.known_peers, 1)[0] - if key.count(':'): - host = key.split(':')[0] - else: - host = key - - # Increment attempt - attempt += 1 - - # TODO: This always makes two attempts at least - - if attempt < 100: - # We found one! - # Connect to it. - - # Get the port (and discard the last heard from - # time) - port, _ = self.known_peers[key] - - # Connect to it. - self.connect(host, port) - else: - # No more things we can try connecting to - break - - # Throw out peers that are too old. First compile a list of their - # hostnames. - too_old = [] - - for key, (port, last_seen) in six.iteritems(self.known_peers): - if last_seen is None: - # This is a bootstrap peer. Don't remove it. - continue - - if time.time() - last_seen > self.peer_timeout: - # We haven't heard from/about this node recently enough. - too_old.append(key) - - # Now drop all the too old hosts - for key in too_old: - del self.known_peers[key] - - # Broadcast all our hosts. - logging.info("{} known peers".format(len(self.known_peers))) - for key, (port, last_seen) in six.iteritems(self.known_peers): - if last_seen is None: - # This is a bootstrap peer, so don't announce it. - continue - if key.count(':'): - host, _ = key.split(':') - logging.debug("\tPeer {} port {} last seen {}".format(host, - port, time2string(last_seen))) - for connection in self.connections: - connection.send_message(["ADDR", host, port, last_seen]) - - if self.external_address is not None: - # Broadcast ourselves, since we know our address. - for connection in self.connections: - connection.send_message(["ADDR", self.external_address, - self.port, int(time.time())]) - - # Do we need to re-poll for blocks? - if self.repoll: - self.repoll = False - logging.warning("Repolling due to unverifiable block.") - - # Compose just one message - message = (["GETBLOCKS"] + - [bytes2string(block_hash) for block_hash in - self.blockchain.get_block_locator()]) - for connection in self.connections: - connection.send_message(message) - - logging.info("Saving...") - # Keep track of how long this takes. - save_start = time.clock() - - if self.tick_count % 60 == 0: - pybc.science.start_timer("sync") - - # Sync the blockchain to disk. This needs to lock the blockchain, so - # the blockchain must never wait on the peer while holding its lock. - # Hence the complicated deferred callback system. TODO: replace the - # whole business with events. - self.blockchain.sync() - - # Sync the known peers to disk - self.known_peers.sync() - - if self.tick_count % 60 == 0: - pybc.science.stop_timer("sync") - - # How long did it take? - save_end = time.clock() - - logging.info("Saved to disk in {:.2} seconds".format(save_end - - save_start)) - - # Count this tick as having happened - self.tick_count += 1 - - # Tick again later - reactor.callLater(self.tick_period, self.tick) - - logging.info("Tick complete.") - - def announce_block(self, block_hash): - """ - Tell all of our connected peers about a block we have. - """ - - with self.lock: - for connection in self.connections: - # Send an INV message with the hash of the thing we have, in - # case they want it. - connection.send_message(["INV", bytes2string(block_hash)]) - - def announce_transaction(self, transaction_hash): - """ - Tell all of our connected peers about a transaction we have. - """ - - with self.lock: - for connection in self.connections: - # Send a TXINV message with the hash of the thing we have, in - # case they want it. - connection.send_message(["TXINV", bytes2string( - transaction_hash)]) - - def was_block_valid(self, block_hash, status): - """ - Called when a block we received becomes verifiable. Takes the block - hash and a status of True if the block was valid, or False if it - wasn't. - - """ - - if status: - # Tell all our peers about this awesome new block - self.announce_block(block_hash) - else: - # Re-poll for blocks when we get a chance. Maybe it was too far in - # the future or something. - self.set_repoll() - - def was_transaction_valid(self, transaction_hash, status): - """ - Called when a transaction is verified and added to the Blockchain's - collection of transactions (i.e. those which one might want to include - in a block), or rejected. Status is True for valid transactions, and - false for invalid ones. - - Broadcasts transactions which are valid. - """ - - if status: - # The transaction was valid. Broadcast it. - self.announce_transaction(transaction_hash) - - def send_block(self, block): - """ - Given a (possibly new) block, will add it to our blockchain and send it - on its way to all peers, if appropriate. - - """ - - if self.blockchain.needs_block(block.block_hash()): - # It's something the blockchain is interested in. - # Queue the block, and, if valid, announce it - self.blockchain.queue_block(block, - callback=self.was_block_valid) - - if self.state_download_timer is not None: - try: - # We started a timer earlier. Cancel it if it hasn't run - self.state_download_timer.cancel() - except BaseException: # It probably already happened - pass - - # In a little bit, if thre are no new blocks before then, download - # StateComponents that we need (if any) - self.state_download_timer = reactor.callLater(0.5, - self.request_state_components) - - def send_transaction(self, transaction): - """ - Given a transaction bytestring, will put it in our queue of transactions - and send it off to our peers, if appropriate. - - """ - logging.info('Sending transaction of %d bytes:\n{}'.format(len(str(transaction)))) - # Add the transaction, and, if valid, announce it - self.blockchain.add_transaction(transaction, - callback=self.was_transaction_valid) - - def request_state_components(self): - """ - Ask all our peers for any StateComponents that our Blockchain needs, and - that we have not recently requested. - - """ - - with self.lock: - if not self.blockchain.state_available: - # We need to ask for StateComponents from our peers until we get - # all of them. - for requested_component in \ - self.blockchain.get_requested_state_components(): - - if (requested_component in - self.recently_requested_state_components): - - # We just asked for this one - continue - - logging.debug("Requesting StateComponent {} from all " - "peers".format(bytes2string(requested_component))) - - # Compose just one message per request - message = ["GETSTATE", bytes2string(requested_component)] - - for connection in self.connections: - # Send it to all peers. Hopefully one or more of them - # will send us a StateComponent back. - connection.send_message(message) - - # Remember having asked for this component - self.recently_requested_state_components.add( - requested_component) - - def run(self): - """ - Run the Twisted reactor and actually make this peer do stuff. Never - returns. - - Exists so people using our module don't need to touch Twisted directly. - They can just do their own IPC and keep us in a separate process. - - """ - - reactor.run() diff --git a/blockchain/pybc/PowAlgorithm.py b/blockchain/pybc/PowAlgorithm.py deleted file mode 100644 index 1e6fcaa03..000000000 --- a/blockchain/pybc/PowAlgorithm.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -PowAlgorithm.py: contains the PowAlgorithm class. - -""" - -from __future__ import absolute_import -import hashlib -import struct - - -class PowAlgorithm(object): - """ - Represents a Strategy pattern object for doing proof-of-work under some - scheme. Can do work on an item to a given target (lower is harder) and - produce a nonce, verify that a nonce meets a given target for an item. - Defines the hash function. - - """ - - def __init__(self): - """ - Make a new PowAlgorithm. Nothing to do, really. - - """ - pass - - def hash(self, data, nonce): - """ - Given a byte string of data (i.e. the thing that you are trying to prove - work on, which ought to be a hash) and a nonce integer of 8 bytes (the - value which may or may not constitute proof of work), compute the hash - that you get by hashing them both together (a string of bytes). This - will be compared against the target to see if this nonce is a valid - proof of work on this data at this target difficulty. - - To change the proof of work function, override this. - - Cribbed from the BitMessage source. - """ - - # By default, we'll use double-SHA512 of the nonce, then the data. This - # is what BitMessage uses, which is where we stole the code from. - return hashlib.sha512(hashlib.sha512(struct.pack('>Q', nonce) + - data).digest()).digest() - - def points(self, data, nonce): - """ - Return the approximate number of proof of work "points" that the given - nonce is worth on the given data. By default this is 2 to the power of - the number of leading zero bits it has. Having another leading zero is - twice as hard, so it should get twice as many points. - - We use Python longs so we can plausibly keep and work with these huge - numbers. - - TODO: Make this more accurate by counting up exactly how many hashes - this hash is smaller than. - - """ - - # Take the hash - hash = self.hash(data, nonce) - - # Count up the leading zeros - leading_zeros = 0 - for char in hash: - for bit in bin(ord(char)): - if bit == "1": - break - - leading_zeros += 1 - - # Count up all the points: double for every zero in the combo - return 2 ** leading_zeros - - def do_work(self, target, data): - """ - Given a target bytestring (lower is harder) and a byte string of data to - work on (probably a hash), find a nonce that, when hashed with the data, - gives a hash where the first 8 bytes are less than the target. Returns - that nonce. - - This function blocks until proof of work is completed, which may be some - time. - - """ - - # This holds the current nonce we're trying. When we get one that's good - # enough, we return it. - nonce = 0 - - while not self.verify_work(target, data, nonce): - # Keep trying nonces until we get one that works. - nonce += 1 - - # We found a good enough nonce! Return it; it is our proof of work. - return nonce - - def do_some_work(self, target, data, placeholder=0, iterations=10000): - """ - Do some work towards finding a proof of work on the data that meets (is - less than) the target. Returns True and a valid nonce if it succeeds, of - False and a placeholder value to be passed back on the next call if it - doesn't find one in fewer than iterations iterations. - - Lets you do proof of work inside an event loop. - """ - - # This holds the nonce we're trying - nonce = placeholder - - while nonce < placeholder + iterations: - # TODO: overflow? - if self.verify_work(target, data, nonce): - # We solved a block! Hooray! - return True, nonce - nonce += 1 - - # We haven't solved a block, but start from here next time. - return False, nonce - - def verify_work(self, target, data, nonce): - """ - Returns True if the given nonce represents at least target work on the - given data, or False if it is invalid. Lower targets are harder. - - This is used both to verify received blocks and check if we've succeeded - in proving work for new ones. - - """ - - # Return whether it's low enough. We do string comparison on - # bytestrings. - return self.hash(data, nonce) <= target diff --git a/blockchain/pybc/ServerFactory.py b/blockchain/pybc/ServerFactory.py deleted file mode 100644 index dbdfe413a..000000000 --- a/blockchain/pybc/ServerFactory.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -ServerFactory.py: contains the ServerFactory class. - -""" - -from __future__ import absolute_import -import logging -from twisted.internet import protocol - -from pybc.BlockchainProtocol import BlockchainProtocol - - -class ServerFactory(protocol.ServerFactory): - """ - This is a Twisted server factory, responsible for taking incoming - connections and starting Servers to serve them. It is part of a Peer. - - """ - - def __init__(self, peer): - """ - Make a new ServerFactory with a reference to the Peer it is handling - incoming connections for. - - """ - - # Keep the peer around - self.peer = peer - - def buildProtocol(self, addr): - """ - Make a new server protocol. It's talking to the given - address. - - """ - - logging.info("Server got connection from {}".format(addr)) - - # Make a new BlockchainProtocol that knows we are its factory. It will - # then talk to our peer. - return BlockchainProtocol(self, (addr.host, addr.port), incoming=True) diff --git a/blockchain/pybc/State.py b/blockchain/pybc/State.py deleted file mode 100644 index 734585bdb..000000000 --- a/blockchain/pybc/State.py +++ /dev/null @@ -1,148 +0,0 @@ -""" -State.py: contains the base State class. - -""" - -from __future__ import absolute_import -from .StateComponent import StateComponent -from .StateMachine import StateMachine - - -class State(object): - """ - Represents some state data that gets kept up to date with the tip of the - blockchain as new blocks come in. In Bitcoin, this might be used to hold the - currently unspent outputs. - - Can return a copy of itself, and update with a block forwards, or a block - backwards. Because we can run history both forwards and backwards, we can - easily calculate the state at any block by walking a path from the tip of - the longest branch. - - A State can be packed up for transmission over the network into a series of - components. Components can depend on each other in a DAG. When the state - changes, only log(n) components need to be updated, so it's possible to - download components from a node that has a state that's constantly changing - and still make meaningful progress. Every State exposes a means to get - components. - - Responsible for persisting itself to and from a file. It should load from - the file on construction, and save to the file on commit(). - - Also responsible for producing StateMachine instances that can be used to - update it. - - Ought to be replaced by implementations that actually keep state and make - copies. - - """ - - def step_forwards(self, block): - """ - If this was the state before the given block, update to what the state - would be after the block. - - """ - - # No change is possible - pass - - def step_backwards(self, block): - """ - If this was the state after the given block, update to what the state - must have been before the block. - - """ - - # No change is possible - pass - - def copy(self): - """ - Return a shallow copy of this State. The State must have no un-commited - operations. The copy may be stepped forwards and backwards without - affecting the original state, and it may be safely discarded. - - """ - - # Since there can be no change, no need to actually copy. - return State() - - def commit(self): - """ - Mark this State as the parent of all States that will be used from now - on. No other States not descended form this one may be comitted in the - future. Allows the State to clean up internal data structures related to - efficiently allowing independent copies. Saves the state to disk to be - re-loaded later. - - """ - - # Nothing to do here. - pass - - def clear(self): - """ - Reset the State to zero or empty, as would be the case before the - genesis block. When this method is called, all other existing copies of - the State become invalid and may never be used again. - - """ - - pass - - def get_component(self, component_hash): - """ - Return the StateComponent with the given hash from this State, or None - if no StateComponent with that hash exists in the State. - - All StateComponents are descendants of a StateComponent with the same - hash as the State itself. - - """ - - return None - - def update_from_components(self, components, root_hash): - """ - Given a dict of just the StateComponents this State does not already - have, by hash, and the hash of the new root StateComponent that we want - this State to adopt, replace internal parts of the State with the - StateComponents from the dict so that the State will have the given root - hash. - - Subclasses must override this. - - """ - pass - - def get_hash(self): - """ - Return a 64-byte hash of the State that is unique and matches the hashes - of other identical states. - - """ - - # Nothing to hash. - return "\0" * 64 - - def audit(self): - """ - Make sure the State is internally consistent. - - """ - - pass - - def make_state_machine(self): - """ - Create a StateMachine that uses this State as a local store, and which - uses the appropriate StateComponent implementation to deserialize - incoming StateComponents. - - If you change the StateComponent class that your State uses, override - this. - - """ - - return StateMachine(StateComponent, self) diff --git a/blockchain/pybc/StateComponent.py b/blockchain/pybc/StateComponent.py deleted file mode 100644 index ba9228d71..000000000 --- a/blockchain/pybc/StateComponent.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -StateComponent.py: Contains the StateComponent class. - -""" - -from __future__ import absolute_import -import hashlib -import pybc.util - - -class StateComponent(object): - """ - A part of a State's DAG for transmitting over the network. - - Contains some binary data, and knows how to manipulate that data to return a - Merkle hash, and an iterator of Merkle hashes of its dependencies. - - """ - - def __init__(self, data): - """ - Make a StateComponent with the given bytestring of data. - - """ - - # Keep the bytestring data - self.data = data - - def get_hash(self): - """ - Return a hash of the StateComponent. - - Implementations can overide this. - """ - - return hashlib.sha512(self.data).digest() - - def get_dependencies(self): - """ - Iterate over Merkle hashes of the dependencies of this node, that we - should fetch to include with this node. - - Implementations can override this. - - """ - - return [] - - def __repr__(self): - """ - Make a StateComponent into a string. - - """ - - # Make a list of lines to merge together into one string - lines = [] - - # Start with the hash we have - lines.append("StateComponent {}".format(pybc.util.bytes2string( - self.get_hash()))) - - for dependency in self.get_dependencies(): - # Then list all the dependencies we need. - lines.append("\tDependency {}".format(pybc.util.bytes2string( - dependency))) - - # Finally talk about the data - lines.append("\t<{} bytes of data>".format(len(self.data))) - - return "\n".join(lines) diff --git a/blockchain/pybc/StateMachine.py b/blockchain/pybc/StateMachine.py deleted file mode 100644 index 5b075a1b3..000000000 --- a/blockchain/pybc/StateMachine.py +++ /dev/null @@ -1,294 +0,0 @@ -""" -StateMachine.py: contains the StateMachine class. - -""" - -from __future__ import absolute_import -import logging - - -from . import util -from . import science - - -class StateMachine(object): - """ - Represents a StateMachine, which downloads StateComponents until it has a - whole tree of them. The StateMachine keeps track of whrre we are in the - download process, because all block downloads themselves are asynchronous - and cause event loop events, and because at any point we may get a new block - and need to start from the top again and download pieces that have changed. - - If a Peer discovers that its Blockchain has no valid State, it can send us a - download message with the hash of the root StateComponent to download. We - will then go into a state where we can produce StateComponent hashes that - we want to download. - - """ - - def __init__(self, state_component_class, local_store=None): - """ - Make a new StateMachine for downloading StateComponents. Uses the given - state_component_class to deserialize StateComponents from bytestrings. - - If local_store is specified, it is a local source of StateComponents. It - must always have the same StateComponents in it as long as we are in the - downloading state, bust always have complete subtrees, and must expose - them via a get_component(hash) method that returns a StateComponent - object or None. (In practice, this can probably be the old State we are - replacing.) - - """ - - # Remember the StateComponentClass - self.state_component_class = state_component_class - - # Remember the local store - self.local_store = None - - # This holds a dict of downloaded StateComponents, by hash - self.downloaded_components = {} - - # This holds a dict of all the StateComponents that we have along with - # all their children, by hash - self.downloaded_subtrees = {} - - # This holds a set of StateComponent hashes we need to download, which - # are all children of the StateComponent on top of the stack. - self.queued_set = set() - - # This holds the stack of StateComponents we're downloading subtrees - # for. The top thing on it is always a StateComponent that we have, but - # which we don't have all the children for. When it's empty, and we have - # the subtree for the root, our download is done. - self.stack = [] - - # This holds the hash of the root StateComponent that we want to have - # all the children of. - self.root_hash = None - - def have_component(self, hash): - """ - Return True if we've downloaded the component with the given hash, and - False otherwise. - - """ - - if hash in self.downloaded_components: - # We downloaded it - return True - - if (self.local_store is not None and - self.local_store.get_component(hash) is not None): - # We had it to start with - return True - - return False - - def get_component(self, hash): - """ - Return a StateComponent that we have, by hash. - - """ - - if hash in self.downloaded_components: - # We downloaded it - return self.downloaded_components[hash] - - # We know we have it, so it must be in the local store. - return self.local_store.get_component(hash) - - def have_subtree(self, hash): - """ - Return True if we have the subtree rooted by the component with the - given hash, and False otheriwse. - - """ - - if hash in self.downloaded_subtrees: - # We downloaded it - return True - - if (self.local_store is not None and - self.local_store.get_component(hash) is not None): - # We had it to start with - return True - - def process_stack(self): - """ - Put all the children that we next should download into queued_set. - Advances and retracts the stack according to an in-order traversal. - - Returns False if there are children in queued_set that we still need to - download before the stack can move, or True if there is more stack - processing to be done and it should be called again. - - If it returns True, but the stack is empty, we have downloaded - everything on the stack. - - """ - - # Get the StateComponent on top of the stack. - - top = self.stack[-1] - - # Do we have all of its children downloaded? If not, the stack stays - # here. - needed_children = 0 - - for child_hash in top.get_dependencies(): - if not self.have_component(child_hash): - # Go get this child - needed_children += 1 - self.queued_set.add(child_hash) - - if needed_children > 0: - # We already queued up all the missing children of this node. Stop - # processing the stack and go get them. - return False - - # If we get here, we have all the children of the top node. - - # Do we have all of its child subtrees downloaded? If not, advance into - # the first one. - for child_hash in top.get_dependencies(): - if not self.have_subtree(child_hash): - # This is the first, go into this one. - self.stack.append(self.downloaded_components[child_hash]) - - # Start again from the top, and either download the children of - # this child, or advance into its first incomplete subtree, or - # mark it as complete. - return True - - # If we get here, the StateComponent on top of the stack has all its - # child subtrees. So we know we have the subtree rooted at it - self.downloaded_subtrees[top.get_hash()] = top - - # Pop it off the stack, and go back and check on its parent, which may - # either be a complete subtree now, or have other children to get, or - # have other subtrees to download. - self.stack.pop() - return True - - def add_component(self, component_bytestring): - """ - We got a new StateComponent. - - """ - - # Deserialize it - component = self.state_component_class(component_bytestring) - - # Get its hash - component_hash = component.get_hash() - - if component_hash in self.queued_set: - # We asked for this one, and now we have it - self.queued_set.remove(component_hash) - - # Save it - self.downloaded_components[component_hash] = component - - logging.debug("Accepted StateComponent {}".format(util.bytes2string( - component_hash))) - elif component_hash in self.downloaded_components: - # We don't need that one because we have it already. - logging.debug("StateComponent {} was already downloaded.".format( - util.bytes2string(component_hash))) - else: - # We got someting we didn't ask for. - logging.warning("StateComponent {} was unsolicited!".format( - util.bytes2string(component_hash))) - - def tick(self): - """ - Advance the stack until either we have some components to download or - the stack empties and we have the root subtree and we're done with our - download. - - """ - - if len(self.stack) == 0 and not self.have_subtree(self.root_hash): - # We still need the root subtree - if self.have_component(self.root_hash): - # We got the root component, though. Put it on the stack so we - # can go get its children. - self.stack.append(self.get_component(self.root_hash)) - else: - # We need to download the root component first. - self.queued_set.add(self.root_hash) - - logging.debug("Need to download root hash: {}".format( - util.bytes2string(self.root_hash))) - - while len(self.stack) > 0 and self.process_stack(): - # Keep going through this loop - pass - - # When we get here, we either have a set of StateComponent hashes to - # download, or we're done. - - def is_done(self): - """ - Return True if we're done with our download, False otherwise. - - """ - - # Are we done yet? We're done if we have the root hash's subtree. - done = self.have_subtree(self.root_hash) - - if done: - # Stop our state download timer, if it's running. We assume we only - # ever have on SateMachine going at a time, but we do actually only - # ever have one StateMachine going at a time (for a Blockchain). - science.stop_timer("state_download") - - return done - - def get_components(self): - """ - Return a dict from hash to StateComponent object for all the newly - downloaded StateComponents that are subtree roots. - - """ - - return self.downloaded_subtrees - - def get_requests(self): - """ - Returns a set of StateComponent hashes that we would like to download. - - """ - - return self.queued_set - - def download(self, root_hash): - """ - Download the StateComponent tree for the given root hash. Re-sets any - in-progress download and clears the set of requests. Keeps the - downloaded StateCompinents and subtrees since they will probably be re- - used at least partially. Call tick after this so that get_requests will - have something in it. - - """ - - # Throw out the old requested component set - self.queued_set = set() - - # Set the root hash - self.root_hash = root_hash - - # Clear off the stack, since we need to come down from the new root and - # download different branches. In the ideal case we'll have lots of - # subtrees already from our local store or the parts we've been - # downloading. - self.stack = [] - - logging.info("Now downloading state {}".format(util.bytes2string( - self.root_hash))) - - # Start a timer so we know when the state download is done. Don't - # replace a running timer because we may switch to downloading a - # different state in the middle. - science.start_timer("state_download", replace=False) diff --git a/blockchain/pybc/TransactionalBlockchain.py b/blockchain/pybc/TransactionalBlockchain.py deleted file mode 100644 index 72f9b8ad8..000000000 --- a/blockchain/pybc/TransactionalBlockchain.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -TransactionalBlockchain.py: contains the TransactionalBlockchain class. - -""" - -from __future__ import absolute_import -import hashlib -import traceback -import struct -import logging - -from pybc.Blockchain import Blockchain -from pybc.transactions import pack_transactions, unpack_transactions -from pybc.transactions import InvalidPayloadError -import pybc.util - - -class TransactionalBlockchain(Blockchain): - """ - A Blockchain where the Blocks are just lists of transactions. All - Blockchains use transactions, but TransactionalBlockchains have blocks where - payloads consist only of transaction lists. - - """ - - def verify_payload(self, next_block): - """ - Verify the payload by verifying all the transactions in this block's - payload against the parent block of this block. - - """ - - with self.lock: - try: - - if self.has_block(next_block.previous_hash): - # This holds the block that this block is based on, or None if - # it's a genesis block. - parent_block = self.get_block(next_block.previous_hash) - else: - # Don't check whether the parent block makes sense here. Just - # say we're checking a genesis block. - parent_block = None - - # Get a State (a copy) to verify transactions against. This - # State may be None, but verify_transaction knows how to deal - # with that. - state = self.state_after(parent_block) - - for transaction in unpack_transactions(next_block.payload): - if not self.verify_transaction(transaction, parent_block, - state, advance=True): - # We checked the the next transaction against the - # blockchain and all previous transactions in the block. - - # This transaction is invalid - return False - - # We have now verified all the the transactions in this block. - return True - - except InvalidPayloadError: - # Parsing the transactions broke, so the payload is invalid. - return False diff --git a/blockchain/pybc/__init__.py b/blockchain/pybc/__init__.py deleted file mode 100644 index 25ada2bae..000000000 --- a/blockchain/pybc/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# __init__.py: Make pybc a module. - -# Present all our external interface classes -from __future__ import absolute_import -from pybc.BlockchainProtocol import * -from pybc.Blockchain import * -from pybc.Block import * -from pybc.ClientFactory import * -from pybc.Peer import * -from pybc.PowAlgorithm import * -from pybc.ServerFactory import * -from pybc.State import * -from pybc.TransactionalBlockchain import * - -# Define all our submodules for importing. -__all__ = ["coin", "util", "transactions", "science"] diff --git a/blockchain/pybc/block_explorer.py b/blockchain/pybc/block_explorer.py deleted file mode 100644 index a149f1c9c..000000000 --- a/blockchain/pybc/block_explorer.py +++ /dev/null @@ -1,184 +0,0 @@ -from __future__ import absolute_import -import time - -from twisted.web import server, resource -from twisted.internet import reactor # @UnresolvedImport -from twisted.internet import protocol - -from . import json_coin -from . import transactions -from . import util - -#------------------------------------------------------------------------------ - -_BlockIndexByTimestamp = dict() - -#------------------------------------------------------------------------------ - -def build_index(blockchain_instance): - global _BlockIndexByTimestamp - _BlockIndexByTimestamp.clear() - for key, block in blockchain_instance.blockstore.items(): - if block.timestamp not in _BlockIndexByTimestamp: - _BlockIndexByTimestamp[block.timestamp] = [] - _BlockIndexByTimestamp[block.timestamp].append(key) - -#------------------------------------------------------------------------------ - -def css_styles(css_dict): - return """ -body {margin: 0 auto; padding: 0;} -.panel-big {display: block; padding: 5px; margin-bottom: 5px; border-radius: 10px; overflow: auto;} -.panel-medium {display: block; padding: 2px; margin-bottom: 2px; border-radius: 5px; overflow: auto;} -.panel-small {display: block; padding: 1px; margin-bottom: 1px; border-radius: 3px; overflow: auto;} -#content {margin: 0 auto; padding: 0; text-align: justify; line-height: 16px; - width: 90%%; font-size: 14px; text-decoration: none; - font-family: "Century Gothic", Futura, Arial, sans-serif;} -.block {background-color: %(block_background)s;border: 1px solid %(block_border)s;} -.b_header {line-height: 14px;} -.b_body {} -.transaction {background-color: %(transaction_background)s; border: 1px solid %(transaction_border)s;} -.t_header {} -.t_body {} -.t_inputs {position: relative;} -.t_outputs {position: relative;} -.t_authorizations {display: block; margin-top: 3px;} -.t_input {background-color: %(input_background)s; border: 1px solid %(input_border)s;} -.t_output {background-color: %(output_background)s; border: 1px solid %(output_border)s;} -.t_authorization {background-color: %(authorization_background)s; border: 1px solid %(authorization_border)s;} -.field {margin: 0 auto; text-align: left; float: left; padding: 0px 10px; display:inline-block;} -.field-right {margin: 0 auto; text-align: left; float: right; padding: 0px 10px; display:inline-block;} -.field code {font-size: 12px;} -.f_json {background-color: %(f_json_bg)s; border: 1px solid %(f_json_border)s; border-radius: 3px;} -""" % css_dict - -#------------------------------------------------------------------------------ - -class MainPage(resource.Resource): - isLeaf = True - peer = None - - def __init__(self, peer, *args, **kwargs): - resource.Resource.__init__(self) - self.peer = peer - - def render_GET(self, request): - global _BlockIndexByTimestamp - src = ''' - -PyBC Node on %(hostname)s:%(hostinfo)s - - -''' % dict( - hostname=self.peer.external_address, - hostinfo=self.peer.port, - css=css_styles(dict( - block_background='#f0f0f0', - block_border='c0c0c0', - transaction_background='#fdfdfd', - transaction_border='#d0d0d0', - input_background='#e0e0ff', - input_border='#c0c0df', - output_background='#e0ffe0', - output_border='#c0dfc0', - authorization_background='#ffe0e0', - authorization_border='#dfc0c0', - f_json_bg='#FAFAD2', - f_json_border='#dfc0c0', - )), - ) - src += '
\n' - src += '

PyBC Node on {}:{}

\n'.format(self.peer.external_address, self.peer.port) - src += 'network name: {}
\n'.format(self.peer.network) - src += 'software version: {}
\n'.format(self.peer.version) - src += 'number of blocks: {}
\n'.format(len(self.peer.blockchain.blockstore)) - src += 'local disk usage: {}
\n'.format(self.peer.blockchain.get_disk_usage()) - src += '

blocks:

\n' - for block in self.peer.blockchain.longest_chain(): - src += '
\n' - src += '
\n' - src += '{} from {},\n'.format( - util.bytes2string(block.block_hash(), limit=8), time.ctime(block.timestamp)) - src += 'previous hash: {},\n'.format( - util.bytes2string(block.previous_hash, limit=8)) - src += 'state hash: {},\n'.format( - util.bytes2string(block.state_hash, limit=8)) - src += 'body hash: {},\n'.format( - util.bytes2string(block.body_hash, limit=8)) - src += 'nonce: {}, height: {}, payload size: {} bytes \n'.format( - block.nonce, block.height, len(block.payload)) - src += '
\n' # b_header - src += '
\n' - if not block.has_body: - src += 'EMPTY BODY\n' - else: - for transaction_bytes in transactions.unpack_transactions(block.payload): - tr = json_coin.JsonTransaction.from_bytes(transaction_bytes) - src += '
\n' - src += '
\n' - src += 'transaction from {}, hash: {}\n'.format( - time.ctime(tr.timestamp), util.bytes2string(tr.transaction_hash(), limit=8)) - src += '
\n' # t_header - src += '
\n' - src += '
\n' - if not tr.inputs: - src += '
\n' - src += '
no inputs
\n' - src += '
\n' # t_input - else: - for inpt in tr.inputs: - src += '
\n' - src += '
{}
\n'.format(inpt[2]) - src += '
{}
\n'.format( - util.bytes2string(inpt[3], limit=8)) - src += '
#{}
\n'.format(inpt[1]) - src += '
{}
\n'.format( - util.bytes2string(inpt[0], limit=8)) - if inpt[4] is not None: - src += '
{}
\n'.format(inpt[4]) - src += '
\n' # t_input - src += '
\n' # t_inputs - src += '
\n' - if not tr.outputs: - src += '
\n' - src += '
no outputs
\n' - src += '
\n' # t_output - else: - for outpt in tr.outputs: - src += '
\n' - src += '
{}
\n'.format(outpt[0]) - src += '
{}
\n'.format( - util.bytes2string(outpt[1], limit=8)) - if outpt[2] is not None: - src += '
{}
\n'.format(outpt[2]) - src += '
\n' # t_input - src += '
\n' # t_outputs - src += '
\n' - if not tr.authorizations: - src += '
\n' - src += '
no authorizations
\n' - src += '
\n' # t_authorization - else: - for author in tr.authorizations: - src += '
\n' - src += '
{}
\n'.format( - util.bytes2string(author[0], limit=10)) - src += '
{}
\n'.format( - util.bytes2string(author[1], limit=10)) - if author[2] is not None: - src += '
{}
\n'.format(author[2]) - src += '
\n' # t_authorization - src += '
\n' # t_authorizations - src += '
\n' # t_body - src += '
\n' # transaction - src += '
\n' # b_body - src += '
\n' # block - src += '
\n' # content - src += '\n' - src += '' - return src - - -def start(port, peer_instance): - site = server.Site(MainPage(peer_instance)) - return reactor.listenTCP(port, site) diff --git a/blockchain/pybc/coin.py b/blockchain/pybc/coin.py deleted file mode 100644 index 1a754825c..000000000 --- a/blockchain/pybc/coin.py +++ /dev/null @@ -1,1548 +0,0 @@ -#!/usr/bin/env python2.7 -# coin.py: a coin implemented on top of pybc - -from __future__ import absolute_import -from __future__ import print_function -import six -from six.moves import range -from six.moves import zip -if __name__ == '__main__': - import os.path as _p, sys - sys.path.insert(0, _p.abspath(_p.join(_p.dirname(_p.abspath(sys.argv[0])), '..'))) - -import struct -import hashlib -import traceback -import time -import threading -import logging -import itertools -import sys - -try: - import pyelliptic -except BaseException: # pyelliptic didn't load. Either it's not installed or it can't find OpenSSL - from . import emergency_crypto_munitions as pyelliptic - -from . import util -from . import sqliteshelf -from .AuthenticatedDictionary import AuthenticatedDictionary -from .AuthenticatedDictionary import AuthenticatedDictionaryStateComponent -from .State import State -from .StateMachine import StateMachine -from .TransactionalBlockchain import TransactionalBlockchain -from .PowAlgorithm import PowAlgorithm -from .transactions import pack_transactions, unpack_transactions - - -class Transaction(object): - """ - Represents a transaction on the blockchain. - - A transaction is a list of inputs (represented by transaction hash, output - index), a list of outputs (represented by amounts and destination public key - hash), and a list of authorizations (public key, signature) signing the - previous two lists. It also has a timestamp, so that two generation - transactions to the same destination address won't have the same hash. (If - you send two block rewards to the same address, make sure to use different - timestamps!) - - Everything except public keys are hashed sha512 (64 bytes). Public keys are - hashed sha256 (32 bytes). - - A transaction is properly authorized if all of the inputs referred to have - destination hashes that match public keys that signed the transaction. - - A generation transaction (or fee collection transaction) is a transaction - with no inputs. It thus requires no authorizations. - - Any input not sent to an output is used as a transaction fee, and added to - the block reward. - - Has a to_bytes and a from_bytes like Blocks do. - - """ - - def __init__(self): - """ - Make a new Transaction with no inputs or outputs. - - """ - - # Set the timestamp to the transaction's creation time (i.e. now). - # Nobody actually verifies it, so it's really 8 arbitrary bytes. - self.timestamp = int(time.time()) - - # Make a list of input tuples (transaction hash, output index, amount, - # destination). - self.inputs = [] - - # Make a list of output tuples (amount, destination public key hash) - self.outputs = [] - - # Make a list of authorization tuples (public key, signature of inputs - # and outputs) - self.authorizations = [] - - def __str__(self): - """ - Represent this transaction as a string. - - """ - - # These are the lines we will return - lines = [] - - lines.append("---Transaction {}---".format(util.time2string( - self.timestamp))) - lines.append("{} inputs".format(len(self.inputs))) - for transaction, index, amount, destination in self.inputs: - # Put every input (another transaction's output) - lines.append("\t{} addressed to {} from output {} of {}".format( - amount, util.bytes2string(destination), index, - util.bytes2string(transaction))) - lines.append("{} outputs".format(len(self.outputs))) - for amount, destination in self.outputs: - # Put every output (an amount and destination public key hash) - lines.append("\t{} to {}".format(amount, - util.bytes2string(destination))) - lines.append("{} authorizations".format(len(self.authorizations))) - for public_key, signature in self.authorizations: - # Put every authorizing key and signature. - lines.append("\tKey: {}".format(util.bytes2string(public_key))) - lines.append("\tSignature: {}".format( - util.bytes2string(signature))) - - # Put the hash that other transactions use to talk about this one - lines.append("Hash: {}".format(util.bytes2string( - self.transaction_hash()))) - - return "\n".join(lines) - - def transaction_hash(self): - """ - Return the SHA512 hash of this transaction, by which other transactions - may refer to it. - - """ - - return hashlib.sha512(self.to_bytes()).digest() - - def add_input(self, transaction_hash, output_index, amount, destination): - """ - Take the coins from the given output of the given transaction as input - for this transaction. It is necessary to specify and store the amount - and destination public key hash of the output, so that the blockchain - can be efficiently read backwards. - - """ - - self.inputs.append((transaction_hash, output_index, amount, - destination)) - - def add_output(self, amount, destination): - """ - Send the given amount of coins to the public key with the given hash. - - """ - - self.outputs.append((amount, destination)) - - def add_authorization(self, public_key, signature): - """ - Add an authorization to this transaction by the given public key. The - given signature is that key's signature of the transaction header data - (inputs and outputs). - - Both public key and signature must be bytestrings. - - """ - - self.authorizations.append((public_key, signature)) - - def get_leftover(self): - """ - Return the sum of all inputs minus the sum of all outputs. - """ - - # This is where we store our total - leftover = 0 - - for _, _, amount, _ in self.inputs: - # Add all the inputs on the + side - leftover += amount - - for amount, _ in self.outputs: - # Add all the outputs on the - side - leftover -= amount - - return leftover - - def verify_authorizations(self): - """ - Returns True if, for every public key hash in the transaction's inputs, - there is a valid authorization signature of this transaction by a public - key with that hash. - - """ - - # Get the bytestring that verifications need to sign - message_to_sign = self.header_bytes() - - # This holds SHA256 hashes of all the pubkeys with valid signatures - valid_signers = set() - - for public_key, signature in self.authorizations: - # Check if each authorization is valid. - if pyelliptic.ECC(pubkey=public_key).verify(signature, - message_to_sign): - - # The signature is valid. Remember the public key hash. - valid_signers.add(hashlib.sha256(public_key).digest()) - - else: - logging.warning("Invalid signature!") - # We're going to ignore extra invalid signatures on - # transactions. What could go wrong? - - for _, _, _, destination in self.inputs: - if destination not in valid_signers: - # This input was not properly unlocked. - return False - - # If we get here, all inputs were to destination pubkey hashes that has - # authorizing signatures attached. - return True - - def pack_inputs(self): - """ - Return the inputs as a bytestring. - - """ - - # Return the 4-byte number of inputs, followed by a 64-byte transaction - # hash, a 4-byte output index, an 8 byte amount, and a 32 byte - # destination public key hash for each input. - return struct.pack(">I", len(self.inputs)) + "".join( - struct.pack(">64sIQ32s", *source) for source in self.inputs) - - def unpack_inputs(self, bytestring): - """ - Set this transaction's inputs to those encoded by the given bytestring. - - """ - - # Start with a fresh list of inputs. - self.inputs = [] - - # How many inputs are there? - (input_count,) = struct.unpack(">I", bytestring[0:4]) - - # Where are we in the string - index = 4 - - for _ in range(input_count): - # Unpack that many 108-byte records of 64-byte transaction hashes, - # 4-byte output indices, 8-byte amounts, and 32-byte destination - # public key hashes. - self.inputs.append(struct.unpack(">64sIQ32s", - bytestring[index:index + 108])) - index += 108 - - def pack_outputs(self): - """ - Return the outputs as a bytestring. - - """ - - # Return the 4-byte number of outputs, followed by an 8-byte amount and - # a 32-byte destination public key hash for each output - return struct.pack(">I", len(self.outputs)) + "".join( - struct.pack(">Q32s", *destination) for destination in self.outputs) - - def unpack_outputs(self, bytestring): - """ - Set this transaction's outputs to those encoded by the given bytestring. - - """ - - # Start with a fresh list of outputs. - self.outputs = [] - - # How many outputs are there? - (output_count,) = struct.unpack(">I", bytestring[0:4]) - - # Where are we in the string - index = 4 - - for _ in range(output_count): - # Unpack that many 40-byte records of 8-byte amounts and 32-byte - # destination public key hashes. - self.outputs.append(struct.unpack(">Q32s", - bytestring[index:index + 40])) - index += 40 - - def pack_authorizations(self): - """ - Return a bytestring of all the authorizations for this transaction. - - """ - - # We have a 4-byte number of authorization records, and then pairs of 4 - # -byte-length and n-byte-data strings for each record. - - # This holds all our length-delimited bytestrings as we make them - authorization_bytestrings = [] - - for public_key, signature in self.authorizations: - # Add the public key - authorization_bytestrings.append(struct.pack(">I", - len(public_key)) + public_key) - # Add the signature - authorization_bytestrings.append(struct.pack(">I", - len(signature)) + signature) - - # Send back the number of records and all the records. - return (struct.pack(">I", len(self.authorizations)) + - "".join(authorization_bytestrings)) - - def unpack_authorizations(self, bytestring): - """ - Set this transaction's authorizations to those encoded by the given - bytestring. - - """ - - # Start with a fresh list of authorizations. - self.authorizations = [] - - # How many outputs are there? - (authorization_count,) = struct.unpack(">I", bytestring[0:4]) - - # Where are we in the string - index = 4 - - for _ in range(authorization_count): - # Get the length of the authorization's public key - (length,) = struct.unpack(">I", bytestring[index:index + 4]) - index += 4 - - # Get the public key itself - public_key = bytestring[index: index + length] - index += length - - # Get the length of the authorization's signature - (length,) = struct.unpack(">I", bytestring[index:index + 4]) - index += 4 - - # Get the signature itself - signature = bytestring[index: index + length] - index += length - - # Add the authorization - self.authorizations.append((public_key, signature)) - - def header_bytes(self): - """ - Convert the inputs and outputs to a bytestring, for signing and for use - in our encoding. - - Packs timestamp in 8 bytes, length of the inputs in 4 bytes, inputs - bytestring, length of the outputs in 4 bytes, outputs bytestring. - - """ - - # Pack up the inputs - inputs_packed = self.pack_inputs() - # And pack up the outputs - outputs_packed = self.pack_outputs() - - # Return both as length-delimited strings - return "".join([struct.pack(">QI", self.timestamp, len(inputs_packed)), - inputs_packed, struct.pack(">I", len(outputs_packed)), - outputs_packed]) - - def to_bytes(self): - """ - Return this Transaction as a bytestring. - - Packs the inputs, outputs, and authorizations bytestrings as length- - delimited strings. - - """ - - # Pack the authorizations - authorizations_packed = self.pack_authorizations() - - # Return the packed inputs and outputs length-delimited strings with one - # for authorizations on the end. - return "".join([self.header_bytes(), struct.pack(">I", - len(authorizations_packed)), authorizations_packed]) - - @classmethod - def from_bytes(cls, bytestring): - """ - Make a new Transaction object from a transaction bytestring, as encoded - by to_bytes. - - """ - - # Make the transaction - transaction = cls() - - # This holds the index we're unpacking the bytestring at - index = 0 - - # Get the timestamp - (transaction.timestamp,) = struct.unpack(">Q", - bytestring[index:index + 8]) - index += 8 - - # Get the length of the inputs bytestring - (length,) = struct.unpack(">I", bytestring[index:index + 4]) - index += 4 - - # Get the inputs bytestring - inputs_bytestring = bytestring[index: index + length] - index += length - - # Get the length of the outputs bytestring - (length,) = struct.unpack(">I", bytestring[index:index + 4]) - index += 4 - - # Get the outputs bytestring - outputs_bytestring = bytestring[index: index + length] - index += length - - # Get the length of the authorizations bytestring - # TODO: It just runs until the end, so we don't really need this. - (length,) = struct.unpack(">I", bytestring[index:index + 4]) - index += 4 - - # Get the authorizations bytestring - authorizations_bytestring = bytestring[index: index + length] - index += length - - # Unpack all the individual bytestrings - transaction.unpack_inputs(inputs_bytestring) - transaction.unpack_outputs(outputs_bytestring) - transaction.unpack_authorizations(authorizations_bytestring) - - # Return the complete Transaction - return transaction - - -class CoinState(State): - """ - A State that keeps track of all unused outputs in blocks. Can also be used - as a generic persistent set of output tuples. - - """ - - def __init__(self, unused_outputs=None, filename=None, table=None): - """ - Make a new CoinState. If unused_outputs is specified, use that - AuthenticatedDictionary to store the state. Otherwise, filename and - table must be specified; load the state from an AuthenticatedDictionary - with that database filename and table name. - - The AuthenticatedDictionary is used to store unused outputs. Unused - outputs are internally identified by the hash of the transaction that - created them (SHA512), the index of the output (int), the amount of the - output (long), and the hash of the destination public key (SHA256); - since these are all fixed length, they are just packed together into a - single bytestring key. Outwardly, these are represented as tuples of the - above. - - """ - - if unused_outputs is not None: - # This holds the set of unused outputs. Since it's an - # AuthenticatedDictionary, we can cheaply copy it. And the copy has - # already been supplied for us. - self.unused_outputs = unused_outputs - elif filename is not None and table is not None: - # We need to make a new AuthenticatedDictionary pointing to the - # appropriate database and table. - self.unused_outputs = AuthenticatedDictionary(filename=filename, - table=table) - else: - # We need to get our data from somewhere. - raise Exception("Cannot make a CoinState without either an " - "AuthenticatedDictionary to use or a database filename and " - "table name to load an AuthenticatedDictionary from.") - - def output2bytes(self, output): - """ - Turn an unused output tuple of (transaction hash, output index, output - amount, destination) into a bytestring. - - The transaction hash is a SHA512 hash of the transaction (64 bytes), the - output index is an integer (4 bytes), the output amount is a long (8 - bytes), and the destination is a SHA256 hash (32 bytes). - - """ - - return struct.pack(">64sIQ32s", *output) - - def bytes2output(self, bytes): - """ - Turn a bytestring into a tuple of (transaction hash, output index, - output amount, destination). - - The transaction hash is a SHA512 hash of the transaction (64 bytes), the - output index is an integer (4 bytes), the output amount is a long (8 - bytes), and the destination is a SHA256 hash (32 bytes). - - """ - - return struct.unpack(">64sIQ32s", bytes) - - def add_unused_output(self, output): - """ - Add the given output tuple as an unused output. - - """ - - # Add a record for it to the set of unspent outputs. We're using the - # AuthenticatedDictionary as a set, so insert the value "". - self.unused_outputs.insert(self.output2bytes(output), "") - - def remove_unused_output(self, output): - """ - Remove the given output tuple as an unused output. - - """ - - # Turn it into its unique bytestring and remove it. - self.unused_outputs.remove(self.output2bytes(output)) - - def apply_transaction(self, transaction): - """ - Update this state by applying the given Transaction object. The - transaction is assumed to be valid. - - """ - - # Get the hash of the transaction - transaction_hash = transaction.transaction_hash() - - logging.debug("Applying transaction {}".format(util.bytes2string( - transaction_hash))) - - for spent_output in transaction.inputs: - # The inputs are in exactly the same tuple format as we use. - # Remove each of them from the unused output set. - - if not self.has_unused_output(spent_output): - logging.error("Trying to remove nonexistent output: {} {} {} " - "{}".format(util.bytes2string(spent_output[0]), - spent_output[1], spent_output[2], - util.bytes2string(spent_output[3]))) - - self.audit() - - # See if things just work out - else: - self.remove_unused_output(spent_output) - - for i, output in enumerate(transaction.outputs): - # Unpack each output tuple - amount, destination = output - - # Make a tupe for the full unused output - unused_output = (transaction_hash, i, amount, destination) - - if self.has_unused_output(unused_output): - logging.error("Trying to create duplicate output: {} {} {} " - "{}".format(util.bytes2string(unused_output[0]), - unused_output[1], unused_output[2], - util.bytes2string(unused_output[3]))) - - self.audit() - - # See if things just work out - else: - self.add_unused_output(unused_output) - - def remove_transaction(self, transaction): - """ - Update this state by removing the given Transaction object. - - """ - - # Get the hash of the transaction - transaction_hash = transaction.transaction_hash() - - logging.debug("Reverting transaction {}".format(util.bytes2string( - transaction_hash))) - - for spent_output in transaction.inputs: - # The inputs are in exactly the same tuple format as we use. Add - # them back to the unused output set, pointing to our useless value - # of "". - - if self.has_unused_output(spent_output): - logging.error("Trying to create duplicate output: {} {} {} " - "{}".format(util.bytes2string(spent_output[0]), - spent_output[1], spent_output[2], - util.bytes2string(spent_output[3]))) - - self.audit() - - # See if things just work out - else: - self.add_unused_output(spent_output) - - for i, output in enumerate(transaction.outputs): - # Unpack each output tuple - amount, destination = output - - # Make a tupe for the full unused output - unused_output = (transaction_hash, i, amount, destination) - - if not self.has_unused_output(unused_output): - logging.error("Trying to remove nonexistent output: {} {} {} " - "{}".format(util.bytes2string(unused_output[0]), - unused_output[1], unused_output[2], - util.bytes2string(unused_output[3]))) - - # Audit ourselves - self.audit() - - # See if things just work out - else: - # Remove its record from the set of unspent outputs - self.remove_unused_output(unused_output) - - def step_forwards(self, block): - """ - Add any outputs of this block's transactions to the set of unused - outputs, and consume all the inputs. - - Updates the CoinState in place. - """ - - for transaction_bytes in unpack_transactions( - block.payload): - - # Parse the transaction - transaction = Transaction.from_bytes(transaction_bytes) - - self.apply_transaction(transaction) - - if self.get_hash() != block.state_hash: - # We've stepped forwards incorrectly - raise Exception("Stepping forward to state {} instead produced " - "state {}.".format(util.bytes2string(block.state_hash), - util.bytes2string(self.get_hash()))) - - def step_backwards(self, block): - """ - Add any inputs from this block to the set of unused outputs, and remove - all the outputs. - - Updates the CoinState in place. - """ - - if self.get_hash() != block.state_hash: - # We're trying to step back the wrong block - raise Exception("Stepping back block for state {} when actually in " - "state {}.".format(util.bytes2string(block.state_hash), - util.bytes2string(self.get_hash()))) - - logging.debug("Correctly in state {} before going back.".format( - util.bytes2string(self.get_hash()))) - - for transaction_bytes in reversed(list(unpack_transactions( - block.payload))): - - # Parse the transaction - transaction = Transaction.from_bytes(transaction_bytes) - - self.remove_transaction(transaction) - - def get_unused_outputs(self): - """ - Yield each unused output, as a (transaction hash, index, amount, - destination) tuple. - - Internally, goes through our AuthenticatedDict and parses out our key - format. - - """ - - for key in six.iterkeys(self.unused_outputs): - yield self.bytes2output(key) - - def has_unused_output(self, output): - """ - Returns true if the given unised output tuple of of (transaction hash, - output index, output amount, destination) is in the current set of - unused outputs, and false otherwise. - - """ - return self.unused_outputs.find(self.output2bytes(output)) is not None - - def copy(self): - """ - Return a copy of the CoinState that can be independently operated on. - This CoinState must have had no modifications made to it since its - creation or last commit() operation. Modifications made to this - CoinState will invalidate the copy and its descendents. - - """ - - # Make a new CoinState, with its AuthenticatedDictionary a child of - # ours. - return CoinState(unused_outputs=self.unused_outputs.copy()) - - def clear(self): - """ - Reset the CoinState to no unused outputs. Works quickly, but - invalidates all other copies of the CoinState. - - """ - - # Clear the AuthenticatedDictionary so it will now be empty. - self.unused_outputs.clear() - - def get_hash(self): - """ - The hash for this state is just the root Merkle hash of the - AuthenticatedDictionary. - - """ - - return self.unused_outputs.get_hash() - - def get_component(self, component_hash): - """ - Return the StateComponent with the given hash from this State, or None - if no StateComponent with that hash exists in the State. - - All StateComponents are descendants of a StateComponent with the same - hash as the State itself. - - """ - - logging.debug("Getting component {}".format(util.bytes2string( - component_hash))) - - # Get the node pointer for that StateComponent - pointer = self.unused_outputs.get_node_by_hash(component_hash) - - if pointer is None: - # We don't have anything with that hash. - return None - - logging.debug("Pointer is {}".format(pointer)) - - # Go get a StateComponent for the appropriate node. - component = self.unused_outputs.node_to_state_component(pointer) - - for child_pointer, child_hash in zip( - self.unused_outputs.get_node_children(pointer), - component.get_child_list()): - - if (child_pointer is None) != (child_hash is None): - raise Exception("Child without hash, or visa versa") - - if child_pointer is not None: - logging.debug("Child pointer {} should have hash {}".format( - child_pointer, util.bytes2string(child_hash))) - - logging.debug("Actually has hash: {}".format(util.bytes2string( - self.unused_outputs.get_node_hash(child_pointer)))) - - logging.debug("Actual pointer for hash: {}".format( - self.unused_outputs.get_node_by_hash(child_hash))) - - for child_hash in component.get_dependencies(): - if self.unused_outputs.get_node_by_hash(child_hash) is None: - # We have a corrupted AuthenticatedDictionary which has nodes - # but not their children. - raise Exception("Node {} mising child {}".format( - util.bytes2string(component_hash), - util.bytes2string(child_hash))) - - # The component checks out. Return it. - return component - - def update_from_components(self, state_components, root_hash): - """ - Update the CoinState to the given rot hash, using the given dict by hash - of extra StateComponents. - - """ - - # Just pass the state components and root hash along to our - # AuthenticatedDictionary. - self.unused_outputs.update_from_state_components(state_components, - root_hash) - - def make_state_machine(self): - """ - Create a StateMachine that knows how to deserialize StateCOmponents of - the type we produce, and which uses us as a local StateComponent store. - - """ - - return StateMachine(AuthenticatedDictionaryStateComponent, self) - - def audit(self): - """ - Make sure the State is internally consistent. - - """ - - # Audit our AuthenticatedDictionary - self.unused_outputs.audit() - - def commit(self): - """ - Mark this CoinState as the CoinState from which all future CoinStstes - will be derived. - - """ - - # Commit the underlying AuthenticatedDictionary - self.unused_outputs.commit() - - -class CoinBlockchain(TransactionalBlockchain): - """ - Represents a Blockchain for a Bitcoin-like currency. - - """ - - def __init__(self, block_store, minification_time=None, state=None): - """ - Make a new CoinBlockchain that stores blocks and state in the specified - file. If a minification_time is specified, accept mini-blocks and throw - out the bodies of blocks burried deeper than the minification time. - - """ - - # Just make a new Blockchain using the default POW algorithm and a - # CoinState to track unspent outputs. Store/load the state to/from the - # "state" table of the blockstore database. Because the state and the - # other blockstore components use the same database, they can't get out - # of sync; the Blockchain promises never to sync its databases without - # committing its State. - super(CoinBlockchain, self).__init__(PowAlgorithm(), block_store, - state=state or CoinState(filename=block_store, table="state"), - minification_time=minification_time) - - # Set up the blockchain for 1 minute blocks, retargeting every 10 - # blocks - # This is in blocks - self.retarget_period = 10 - # This is in seconds - self.retarget_time = self.retarget_period * 60 - - def transaction_valid_for_relay(self, transaction_bytes): - """ - Say that normal transactions can be accepted from peers, but generation - and fee collection transactions cannot. - - """ - - if len(Transaction.from_bytes(transaction_bytes).inputs) > 0: - # It has an input, so isn't a reward. - return True - - # No inputs. Shouldn't accept this, even if it's valid. It will steal - # our fees. - return False - - def get_block_reward(self, previous_block): - """ - Get the block reward for a block based on the given previous block, - which may be None. - - """ - - # Easy example: 50 coins forever - - # Get the height of this block - if previous_block is not None: - height = previous_block.height + 1 - else: - height = 0 - - # How many coins should we generate? We could do something based on - # height, but just be easy. - coins = 50 - - # Return the number of coins to generate - return coins - - def verify_payload(self, next_block): - """ - Verify all the transactions in the block as a group. Each individually - gets validated the normal way with verify_transaction, but after that's - done we make sure that the total excess coin spent exactly equals the - amount that's supposed to be generated. - - """ - - if not super(CoinBlockchain, self).verify_payload(next_block): - # Some transaction failed basic per-transaction validation. - return False - - # How much left-over coin do we have in this block so far? Start with - # the block reward. - block_leftover = self.get_block_reward(next_block) - - for transaction_bytes in unpack_transactions(next_block.payload): - # Parse a Transaction object out - transaction = Transaction.from_bytes(transaction_bytes) - - # Add (or remove) the transaction's leftover coins from the block's - # leftover coins - block_leftover += transaction.get_leftover() - - if block_leftover == 0: - # All the fees and rewards went to the exact right places. Only - # transactions with no inputs can take from the fees at all (as per - # verify_transaction). - return True - else: - # Reject the block if its transactions have uncollected - # fees/rewards, or if it tries to give out more in fees and rewards - # than it deserves. - logging.warning("Block disburses rewards/fees incorrectly.") - return False - - def verify_transaction(self, transaction_bytes, chain_head, state, - advance=False): - """ - If the given Transaction is valid on top of the given chain head block - (which may be None), in the given State (which may be None), return - True. Otherwise, return False. If advance is True, and the transaction - is valid, advance the State in place. - - Ensures that: - - The transaction's inputs are existing unspent outputs that the other - transactions didn't use. - - The transaction's outputs are not already present as unspent outputs - (in case someone tries to put in the same generation transaction twice). - - The transaction's authorizations are sufficient to unlock its inputs. - from pybc.transactions import pack_transactions, unpack_transactions - The transaction's outputs do not excede its inputs, if it has inputs. - - """ - - try: - # This holds our parsed transaction - transaction = Transaction.from_bytes(transaction_bytes) - except BaseException: # The transaction is uninterpretable - logging.warning("Uninterpretable transaction.") - traceback.print_exc() - return False - - for i, (amount, destination) in enumerate(transaction.outputs): - if amount == 0: - logging.warning("Transaction trying to send a 0 output.") - return False - - # What unused output tuple would result from this? They all need to - # be unique. - unused_output = (transaction.transaction_hash(), i, amount, - destination) - - if state is not None: - if state.has_unused_output(unused_output): - # We have a duplicate transaction. Probably a generation - # transaction, since all others need unspent inputs. - logging.warning("Transaction trying to create a duplicate unused output") - return False - else: - # If the State is None, we have to skip verifying that these - # outputs are unused - - logging.debug("Not checking for duplicate output, since we have no State.") - - # Outputs can never be negative since they are unsigned. So we don't - # need to check that. - - if len(transaction.inputs) == 0: - # This is a fee-collecting/reward-collecting transaction. We can't - # verify them individually, but we can make sure the total they come - # to is not too big or too small in verify_payload. - - if advance and state is not None: - # We're supposed to advance the state since transaction is valid - state.apply_transaction(transaction) - - return True - - # Now we know the transaction has inputs. Check them. - - for source in transaction.inputs: - if state is not None: - # Make sure each input is accounted for by a previous unused - # output. - if not state.has_unused_output(source): - # We're trying to spend something that doesn't exist or is - # already spent. - logging.warning("Transaction trying to use spent or nonexistent input") - return False - else: - logging.debug("Not checking for re-used input, since we have no State.") - - if transaction.get_leftover() < 0: - # Can't spend more than is available to the transaction. - logging.warning("Transaction trying to output more than it inputs") - return False - - if not transaction.verify_authorizations(): - # The transaction isn't signed properly. - logging.warning("Transaction signature(s) invalid") - return False - - # If we get here, the transaction must be valid. All its inputs are - # authorized, and its outputs aren't too large. - - if advance and state is not None: - # We're supposed to advance the state since transaction is valid - state.apply_transaction(transaction) - - return True - - def make_block(self, destination, min_fee=1): - """ - Override the ordinary Blockchain make_block with a make_block that - incorporates pending transactions and sends fees to the public key hash - destination. - - min_fee specifies the minimum trnasaction fee to require. - - Returns None if the Blockchain is not up to date with a State for the - top Block, and thus unable to mine on top of it. - - """ - - # Don't let anybody mess with our transactions and such until we've made - # the block. It can still be rendered invalid after that, though. - with self.lock: - - logging.debug("Making a block") - - if not self.state_available: - # We can't mine without the latest State - logging.debug("Can't make block without State") - return None - - # This holds the list of Transaction objects to include - to_include = [] - - # This holds the total fee available, starting with the block - # reward. - total_fee = self.get_block_reward(self.highest_block) - - # This holds the state that we will move to when we apply all these - # transactions. We already know none of the transactions in - # self.transactions conflict, and that none of them depend on each - # other, so we can safely advance the State with them. - next_state = self.state.copy() - - # How many bytes of transaction data have we used? - transaction_bytes_used = 0 - - for transaction_bytes in self.transactions.values(): - # Parse this transaction out. - transaction = Transaction.from_bytes(transaction_bytes) - - # Get how much it pays - fee = transaction.get_leftover() - - if fee >= min_fee: - # This transaction pays enough. Use it. - - if self.verify_transaction(transaction_bytes, - self.highest_block, next_state, advance=True): - - # The transaction is OK to go in the block. We checked - # it earlier, but we should probably check it again. - to_include.append(transaction) - total_fee += fee - transaction_bytes_used += len(transaction_bytes) - - if transaction_bytes_used >= 1024 * 1024: - # Don't make blocks bigger than 1 MB of transaction data - logging.info("Hit block size limit for generation.") - break - - # Add a transaction that gives all the generated coins and fees to - # us. - reward_transaction = Transaction() - reward_transaction.add_output(total_fee, destination) - - # This may match exactly the reward transaction in the last block if - # we just made and solved that. What unused output does the - # generation transaction produce? - - if not self.verify_transaction(reward_transaction.to_bytes(), - self.highest_block, next_state, advance=True): - - # Our reward-taking transaction is invalid for some reason. We - # probably already generated a block this second or something. - - logging.info("Reward transaction would be invalid (maybe " - "already used). Skipping block creation.") - - return None - - to_include.append(reward_transaction) - - # Make a block moving to the state we have after we apply all those - # transactions, with the transaction packed into its payload - block = super(CoinBlockchain, self).make_block(next_state, - pack_transactions( - [transaction.to_bytes() for transaction in to_include])) - - if block is None: - logging.debug("Base class cound not make block") - return block - - # next_state gets discarded without being committed, which is - # perfectly fine. It won't touch the database at all, not even by - # making unsynced changes, unless we commit. - - def dump_block(self, block): - """ - Return a string version of the block, with string versions of all the - Transactions appended, for easy viewing. - - """ - - # Keep a list of all the parts to join - parts = [str(block)] - - if block.has_body: - for transaction_bytes in unpack_transactions( - block.payload): - - parts.append(str(Transaction.from_bytes(transaction_bytes))) - - return "\n".join(parts) - - -class Wallet(object): - """ - Represents a Wallet that holds keypairs. Interrogates the blockchain to - figure out what coins are available to spend, and manages available unspent - outputs and change sending so that you can send transactions for arbitrary - amounts to arbitrary addresses. - - TODO: This isn't thread safe at all. - - """ - - def __init__(self, blockchain, filename, state=None): - """ - Make a new Wallet, working on the given Blockchain, and storing keypairs - in the given Wallet file. - - """ - - # Use a database to keep pyelliptic ECC objects for our addresses. - # keypairs are stored by public key hash. - self.keystore = sqliteshelf.SQLiteShelf(filename, table="wallet", - lazy=True) - - # Keep a wallet metadata table, really just fof frecording what - # Blockchain state the wallet is up to date with. If they get out of - # sync we force a reset event. - self.wallet_metadata = sqliteshelf.SQLiteShelf(filename, - table="metadata", lazy=True) - - # Keep a persistent set of spendable outputs. We use a CoinState because - # it has methods that easily take and yield output tuples. - self.spendable = state or CoinState(filename=filename, table="spendable") - - # Keep a copy of that that holds the transactions we should draw upon - # when making our next transaction. This lets us use different unspent - # outputs for different transactions. - self.willing_to_spend = self.spendable.copy() - - # Keep the blockchain - self.blockchain = blockchain - - # Listen to it - self.blockchain.subscribe(self.blockchain_event) - - # We need a lock to protect our keystore from multithreaded access. - # TODO: Shrink the critical sections. - self.lock = threading.RLock() - - # Make sure we are in sync with the blockchain - if ("blockchain_state" not in self.wallet_metadata or - self.wallet_metadata["blockchain_state"] != - self.blockchain.state.get_hash()): - - # We're not in sync with the blockchain. Get in sync. - with self.blockchain.lock: - # TODO: Don't steal the blockchain's lock. Write a "tell - # everyone your state" method in Blockchain. - - # This should work even if state_available isn't true. When it - # becomes true, we'll be given the new State to replace this - # one. - self.blockchain.send_event("reset", self.blockchain.state) - else: - logging.info("Wallet is in sync with blockchain.") - - def blockchain_event(self, event, argument): - """ - Called by the Blockchain, with the Blockchain's lock, when an event - happens. The Blockchain is probably in an intermediate state, so only - look at the arguments and not the Blockchain itself. - - """ - - # TODO: Track balance as an int for easy getting. - - with self.lock: - if event == "forward": - logging.debug("Advancing wallet forward") - # We've advanced forward a block. Get any new spendable outputs. - for transaction_bytes in unpack_transactions( - argument.payload): - - # Parse the transaction - transaction = Transaction.from_bytes(transaction_bytes) - - transaction_hash = transaction.transaction_hash() - - for spent_output in transaction.inputs: - if self.spendable.has_unused_output(spent_output): - # This output we had available got spent - self.spendable.remove_unused_output(spent_output) - - for i, output in enumerate(transaction.outputs): - # Unpack each output tuple - amount, destination = output - - if destination in self.keystore: - # This output is spendable by us - self.spendable.add_unused_output((transaction_hash, - i, amount, destination)) - - # Re-set our set of transactions we should draw upon to all the - # transactions available - self.willing_to_spend = self.spendable.copy() - - elif event == "backward": - # We've gone backward a block - - logging.debug("Advancing wallet backward") - - for transaction_bytes in unpack_transactions( - argument.payload): - - # Parse the transaction - transaction = Transaction.from_bytes(transaction_bytes) - - transaction_hash = transaction.transaction_hash() - - for spent_output in transaction.inputs: - if spent_output[3] in self.keystore: - # This output we spent got unspent - self.spendable.add_unused_output(spent_output) - - for i, output in enumerate(transaction.outputs): - # Unpack each output tuple - amount, destination = output - - # Make an output tuple in the full format - spent_output = (transaction_hash, i, amount, - destination) - - if self.spendable.has_unused_output(spent_output): - # This output we had available to spend got un-made - self.spendable.remove_unused_output(spent_output) - - # Re-set our set of transactions we should draw upon to all the - # transactions available - self.willing_to_spend = self.spendable.copy() - - elif event == "reset": - # Argument is a CoinState that's not really related to our - # previous one. - - logging.info("Rebuilding wallet's index of spendable outputs.") - - # Throw out our current idea of our spendable outputs. - self.spendable.clear() - - # How many outputs are for us? - found = 0 - # And how much are they worth? - balance = 0 - - for unused_output in argument.get_unused_outputs(): - if unused_output[3] in self.keystore: - # This output is addressed to us. Say it's spendable - self.spendable.add_unused_output(unused_output) - logging.debug("\t{} to {}".format(unused_output[2], - util.bytes2string(unused_output[3]))) - found += 1 - balance += unused_output[2] - - logging.info("{} outputs available, totaling {}".format(found, - balance)) - - # Re-set our set of transactions we should draw upon to all the - # transactions available - self.willing_to_spend = self.spendable.copy() - - elif event == "sync": - # Save anything to disk that depends on the blockchain. This - # doesn't guarantee that we won't get out of sync with the - # blockchain, but it helps. - - # TODO: We happen to know it's safe to look at state here, but - # in general it's not. Also, the state may be invalid or the top - # block, but if that's true we'll get a reset when we have a - # better state. - - logging.info("Saving wallet") - - # Save the blockchain state that we are up to date with. - self.wallet_metadata["blockchain_state"] = \ - self.blockchain.state.get_hash() - - self.spendable.commit() - self.keystore.sync() - - else: - logging.warning("Unknown event {} from blockchain".format(event)) - - def generate_address(self): - """ - Make a new address and add it to our keystore. - - We won't know about coins to this address sent before it was generated. - - """ - with self.lock: - # This holds the new keypair as a pyelliptic ECC - keypair = pyelliptic.ECC() - - # Save it to the keystore - self.keystore[hashlib.sha256(keypair.get_pubkey()).digest()] = \ - keypair - - # Sync it to disk. TODO: this is an easy way to get out of sync with - # the saved Blockchain state. - self.keystore.sync() - - def get_address(self): - """ - Return the public key hash of an address that we can receive on. - - """ - - with self.lock: - if len(self.keystore) == 0: - # We need to make an address - self.generate_address() - - # Just use the first address we have - return list(self.keystore.keys())[0] - - def get_balance(self): - """ - Return the total balance of all spendable outputs. - - """ - - # This holds the balance so far - balance = 0 - - for _, _, amount, _ in self.spendable.get_unused_outputs(): - # Sum up the amounts over all spendable outputs - balance += amount - - return balance - - def make_simple_transaction(self, amount, destination, fee=1): - """ - Return a Transaction object sending the given amount to the given - destination, and any remaining change back to ourselves, leaving the - specified miner's fee unspent. - - If we don't have enough in outputs that we're willing to spend (i.e. - which we haven't used to make transactiona already, and which aren't - change that hasn't been confirmed yet), return None. - - If the amount isn't strictly positive, also returns None, since such a - transaction would be either useless or impossible depending on the - actual value. - - Destination must be a 32-byte public key SHA256 hash. - - A negative fee can be passed, but the resulting transaction will not be - valid. - - """ - - with self.lock: - - if not amount > 0: - # Transaction is unreasonable: not sending any coins anywhere. - return None - - # Make a transaction - transaction = Transaction() - - # This holds how much we have accumulated from the spendable outputs - # we've added to the transaction's inputs. - coins_collected = 0 - - # This holds the set of public key hashes that we need to sign the - # transaction with. - key_hashes = set() - - # This holds our willing_to_spend set with updates - willing_to_spend = self.willing_to_spend.copy() - - # This holds our outputs we need to take out of - # willing_to_spend when done iterating over it. - spent = [] - - for spendable in willing_to_spend.get_unused_outputs(): - # Unpack the amount we get from this as an input, and the key we - # need to use to spend it. - _, _, input_amount, key_needed = spendable - - # Add the unspent output as an input to the transaction - transaction.add_input(*spendable) - - # Say we've collected that many coins - coins_collected += input_amount - - # Say we need to sign with the appropriate key - key_hashes.add(key_needed) - - # Say we shouldn't spend this again in our next transaction. - spent.append(spendable) - - if coins_collected >= amount + fee: - # We have enough coins. - break - - for output in spent: - # Mark all the outputs we just tried to spend as used until a - # block comes along either confirming or denying this. - willing_to_spend.remove_unused_output(output) - - if coins_collected < amount + fee: - # We couldn't find enough money for this transaction! - # Maybe wait until some change transactions get into blocks. - return None - - # We're going through with the transaction. Don't re-use these - # inputs until after the next block comes in. - self.willing_to_spend = willing_to_spend - - # We've made a transaction with enough inputs! - # Add the outputs. - # First the amount we actually wanted to send. - transaction.add_output(amount, destination) - - if coins_collected - amount - fee > 0: - # Then the change, if any, back to us at some address we can - # receive on. - transaction.add_output(coins_collected - amount - fee, - self.get_address()) - # The fee should be left over. - - # Now do the authorizations. What do we need to sign? - to_sign = transaction.header_bytes() - - for key_hash in key_hashes: - # Load the keypair - keypair = self.keystore[key_hash] - - # Grab the public key - public_key = keypair.get_pubkey() - - # Make the signature - signature = keypair.sign(to_sign) - - # Add the authorization to the transaction - transaction.add_authorization(public_key, signature) - - # TODO: If we have a change output, put it in the willing to spend - # set so we can immediately spend from it. - - # Now the transaction is done! - return transaction - - -if __name__ == "__main__": - # Do a transaction test - - def generate_block(blockchain, destination, min_fee=1): - """ - Given a blockchain, generate a block (synchronously!), sending the - generation reward to the given destination public key hash. - - min_fee specifies the minimum fee to charge. - - TODO: Move this into the Blockchain's get_block method. - - """ - - # Make a block with the transaction as its payload - block = blockchain.make_block(destination, min_fee=min_fee) - - # Now unpack and dump the block for debugging. - print("Block will be:\n{}".format(block)) - - for transaction in unpack_transactions(block.payload): - # Print all the transactions - print("Transaction: {}".format(Transaction.from_bytes(transaction))) - - # Do proof of work on the block to mine it. - block.do_work(blockchain.algorithm) - - print("Successful nonce: {}".format(block.nonce)) - - # See if the work really is enough - print("Work is acceptable: {}".format(block.verify_work( - blockchain.algorithm))) - - # See if the block is good according to the blockchain - print("Block is acceptable: {}".format(blockchain.verify_block(block))) - - # Add it to the blockchain through the complicated queueing mechanism - blockchain.queue_block(block) - - # Make a blockchain - blockchain = CoinBlockchain("coin.blocks") - - # Make a wallet that hits against it - wallet = Wallet(blockchain, "coin.wallet") - - print("Receiving address: {}".format(util.bytes2string( - wallet.get_address()))) - - # Make a block that gives us coins. - generate_block(blockchain, wallet.get_address()) - - # Send some coins to ourselves - print("Sending ourselves 10 coins...") - transaction = wallet.make_simple_transaction(10, wallet.get_address()) - print(transaction) - blockchain.add_transaction(transaction.to_bytes()) - - # Make a block that confirms that transaction. - generate_block(blockchain, wallet.get_address()) diff --git a/blockchain/pybc/emergency_crypto_munitions.py b/blockchain/pybc/emergency_crypto_munitions.py deleted file mode 100644 index 351ab504e..000000000 --- a/blockchain/pybc/emergency_crypto_munitions.py +++ /dev/null @@ -1,1675 +0,0 @@ -""" -emergency_crypto_munitions.py: A hacked-up version of pyelliptic that will load -even when the system OpenSSL is not necesarily the best. It still has to support -elliptic curves, but it can be ay out of date. - -Contains only the parts of pyelliptic needed for elliptic curve cryptography. - -Licensed under the GPL, since pyelliptic is also GPL. The rest of pybc is MIT -licensed. - -""" - -from __future__ import absolute_import -from __future__ import print_function -print("Attempting to load emergency backup crypto munitions.") - -# Copyright (C) 2011 Yann GUIBET - -import traceback - -# Stuf from openssl.py - -import sys -import ctypes -import ctypes.util - -OpenSSL = None - - -class CipherName: - def __init__(self, name, pointer, blocksize): - self._name = name - self._pointer = pointer - self._blocksize = blocksize - - def __str__(self): - return "Cipher : " + self._name + " | Blocksize : " + str(self._blocksize) + " | Fonction pointer : " + str(self._pointer) - - def get_pointer(self): - return self._pointer() - - def get_name(self): - return self._name - - def get_blocksize(self): - return self._blocksize - - -class _OpenSSL: - """ - Wrapper for OpenSSL using ctypes - """ - - def __init__(self, library): - """ - Build the wrapper - """ - print("Loading {}".format(library)) - self._lib = ctypes.CDLL(library) - - self.pointer = ctypes.pointer - self.c_int = ctypes.c_int - self.byref = ctypes.byref - self.create_string_buffer = ctypes.create_string_buffer - - self.BN_new = self._lib.BN_new - self.BN_new.restype = ctypes.c_void_p - self.BN_new.argtypes = [] - - self.BN_free = self._lib.BN_free - self.BN_free.restype = None - self.BN_free.argtypes = [ctypes.c_void_p] - - self.BN_num_bits = self._lib.BN_num_bits - self.BN_num_bits.restype = ctypes.c_int - self.BN_num_bits.argtypes = [ctypes.c_void_p] - - self.BN_bn2bin = self._lib.BN_bn2bin - self.BN_bn2bin.restype = ctypes.c_int - self.BN_bn2bin.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - - self.BN_bin2bn = self._lib.BN_bin2bn - self.BN_bin2bn.restype = ctypes.c_void_p - self.BN_bin2bn.argtypes = [ctypes.c_void_p, ctypes.c_int, - ctypes.c_void_p] - - self.EC_KEY_free = self._lib.EC_KEY_free - self.EC_KEY_free.restype = None - self.EC_KEY_free.argtypes = [ctypes.c_void_p] - - self.EC_KEY_new_by_curve_name = self._lib.EC_KEY_new_by_curve_name - self.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p - self.EC_KEY_new_by_curve_name.argtypes = [ctypes.c_int] - - self.EC_KEY_generate_key = self._lib.EC_KEY_generate_key - self.EC_KEY_generate_key.restype = ctypes.c_int - self.EC_KEY_generate_key.argtypes = [ctypes.c_void_p] - - self.EC_KEY_check_key = self._lib.EC_KEY_check_key - self.EC_KEY_check_key.restype = ctypes.c_int - self.EC_KEY_check_key.argtypes = [ctypes.c_void_p] - - self.EC_KEY_get0_private_key = self._lib.EC_KEY_get0_private_key - self.EC_KEY_get0_private_key.restype = ctypes.c_void_p - self.EC_KEY_get0_private_key.argtypes = [ctypes.c_void_p] - - self.EC_KEY_get0_public_key = self._lib.EC_KEY_get0_public_key - self.EC_KEY_get0_public_key.restype = ctypes.c_void_p - self.EC_KEY_get0_public_key.argtypes = [ctypes.c_void_p] - - self.EC_KEY_get0_group = self._lib.EC_KEY_get0_group - self.EC_KEY_get0_group.restype = ctypes.c_void_p - self.EC_KEY_get0_group.argtypes = [ctypes.c_void_p] - - self.EC_POINT_get_affine_coordinates_GFp = self._lib.EC_POINT_get_affine_coordinates_GFp - self.EC_POINT_get_affine_coordinates_GFp.restype = ctypes.c_int - self.EC_POINT_get_affine_coordinates_GFp.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - - self.EC_KEY_set_private_key = self._lib.EC_KEY_set_private_key - self.EC_KEY_set_private_key.restype = ctypes.c_int - self.EC_KEY_set_private_key.argtypes = [ctypes.c_void_p, - ctypes.c_void_p] - - self.EC_KEY_set_public_key = self._lib.EC_KEY_set_public_key - self.EC_KEY_set_public_key.restype = ctypes.c_int - self.EC_KEY_set_public_key.argtypes = [ctypes.c_void_p, - ctypes.c_void_p] - - self.EC_KEY_set_group = self._lib.EC_KEY_set_group - self.EC_KEY_set_group.restype = ctypes.c_int - self.EC_KEY_set_group.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - - self.EC_POINT_set_affine_coordinates_GFp = self._lib.EC_POINT_set_affine_coordinates_GFp - self.EC_POINT_set_affine_coordinates_GFp.restype = ctypes.c_int - self.EC_POINT_set_affine_coordinates_GFp.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - - self.EC_POINT_new = self._lib.EC_POINT_new - self.EC_POINT_new.restype = ctypes.c_void_p - self.EC_POINT_new.argtypes = [ctypes.c_void_p] - - self.EC_POINT_free = self._lib.EC_POINT_free - self.EC_POINT_free.restype = None - self.EC_POINT_free.argtypes = [ctypes.c_void_p] - - self.EC_KEY_set_private_key = self._lib.EC_KEY_set_private_key - self.EC_KEY_set_private_key.restype = ctypes.c_int - self.EC_KEY_set_private_key.argtypes = [ctypes.c_void_p, - ctypes.c_void_p] - - self.ECDH_OpenSSL = self._lib.ECDH_OpenSSL - self._lib.ECDH_OpenSSL.restype = ctypes.c_void_p - self._lib.ECDH_OpenSSL.argtypes = [] - - self.ECDH_set_method = self._lib.ECDH_set_method - self._lib.ECDH_set_method.restype = ctypes.c_int - self._lib.ECDH_set_method.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - - self.ECDH_compute_key = self._lib.ECDH_compute_key - self.ECDH_compute_key.restype = ctypes.c_int - self.ECDH_compute_key.argtypes = [ctypes.c_void_p, - ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] - - self.EVP_CipherInit_ex = self._lib.EVP_CipherInit_ex - self.EVP_CipherInit_ex.restype = ctypes.c_int - self.EVP_CipherInit_ex.argtypes = [ctypes.c_void_p, - ctypes.c_void_p, ctypes.c_void_p] - - self.EVP_CIPHER_CTX_new = self._lib.EVP_CIPHER_CTX_new - self.EVP_CIPHER_CTX_new.restype = ctypes.c_void_p - self.EVP_CIPHER_CTX_new.argtypes = [] - - # Cipher - self.EVP_aes_128_cfb128 = self._lib.EVP_aes_128_cfb128 - self.EVP_aes_128_cfb128.restype = ctypes.c_void_p - self.EVP_aes_128_cfb128.argtypes = [] - - self.EVP_aes_256_cfb128 = self._lib.EVP_aes_256_cfb128 - self.EVP_aes_256_cfb128.restype = ctypes.c_void_p - self.EVP_aes_256_cfb128.argtypes = [] - - self.EVP_aes_128_cbc = self._lib.EVP_aes_128_cbc - self.EVP_aes_128_cbc.restype = ctypes.c_void_p - self.EVP_aes_128_cbc.argtypes = [] - - self.EVP_aes_256_cbc = self._lib.EVP_aes_256_cbc - self.EVP_aes_256_cbc.restype = ctypes.c_void_p - self.EVP_aes_256_cbc.argtypes = [] - - self.EVP_aes_128_ctr = self._lib.EVP_aes_128_ctr - self.EVP_aes_128_ctr.restype = ctypes.c_void_p - self.EVP_aes_128_ctr.argtypes = [] - - self.EVP_aes_256_ctr = self._lib.EVP_aes_256_ctr - self.EVP_aes_256_ctr.restype = ctypes.c_void_p - self.EVP_aes_256_ctr.argtypes = [] - - self.EVP_aes_128_ofb = self._lib.EVP_aes_128_ofb - self.EVP_aes_128_ofb.restype = ctypes.c_void_p - self.EVP_aes_128_ofb.argtypes = [] - - self.EVP_aes_256_ofb = self._lib.EVP_aes_256_ofb - self.EVP_aes_256_ofb.restype = ctypes.c_void_p - self.EVP_aes_256_ofb.argtypes = [] - - self.EVP_bf_cbc = self._lib.EVP_bf_cbc - self.EVP_bf_cbc.restype = ctypes.c_void_p - self.EVP_bf_cbc.argtypes = [] - - self.EVP_bf_cfb64 = self._lib.EVP_bf_cfb64 - self.EVP_bf_cfb64.restype = ctypes.c_void_p - self.EVP_bf_cfb64.argtypes = [] - - self.EVP_rc4 = self._lib.EVP_rc4 - self.EVP_rc4.restype = ctypes.c_void_p - self.EVP_rc4.argtypes = [] - - self.EVP_CIPHER_CTX_cleanup = self._lib.EVP_CIPHER_CTX_cleanup - self.EVP_CIPHER_CTX_cleanup.restype = ctypes.c_int - self.EVP_CIPHER_CTX_cleanup.argtypes = [ctypes.c_void_p] - - self.EVP_CIPHER_CTX_free = self._lib.EVP_CIPHER_CTX_free - self.EVP_CIPHER_CTX_free.restype = None - self.EVP_CIPHER_CTX_free.argtypes = [ctypes.c_void_p] - - self.EVP_CipherUpdate = self._lib.EVP_CipherUpdate - self.EVP_CipherUpdate.restype = ctypes.c_int - self.EVP_CipherUpdate.argtypes = [ctypes.c_void_p, - ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int] - - self.EVP_CipherFinal_ex = self._lib.EVP_CipherFinal_ex - self.EVP_CipherFinal_ex.restype = ctypes.c_int - self.EVP_CipherFinal_ex.argtypes = [ctypes.c_void_p, - ctypes.c_void_p, ctypes.c_void_p] - - self.EVP_DigestInit = self._lib.EVP_DigestInit - self.EVP_DigestInit.restype = ctypes.c_int - self._lib.EVP_DigestInit.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - - self.EVP_DigestUpdate = self._lib.EVP_DigestUpdate - self.EVP_DigestUpdate.restype = ctypes.c_int - self.EVP_DigestUpdate.argtypes = [ctypes.c_void_p, - ctypes.c_void_p, ctypes.c_int] - - self.EVP_DigestFinal = self._lib.EVP_DigestFinal - self.EVP_DigestFinal.restype = ctypes.c_int - self.EVP_DigestFinal.argtypes = [ctypes.c_void_p, - ctypes.c_void_p, ctypes.c_void_p] - - self.EVP_ecdsa = self._lib.EVP_ecdsa - self._lib.EVP_ecdsa.restype = ctypes.c_void_p - self._lib.EVP_ecdsa.argtypes = [] - - self.ECDSA_sign = self._lib.ECDSA_sign - self.ECDSA_sign.restype = ctypes.c_int - self.ECDSA_sign.argtypes = [ctypes.c_int, ctypes.c_void_p, - ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - - self.ECDSA_verify = self._lib.ECDSA_verify - self.ECDSA_verify.restype = ctypes.c_int - self.ECDSA_verify.argtypes = [ctypes.c_int, ctypes.c_void_p, - ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p] - - self.EVP_MD_CTX_create = self._lib.EVP_MD_CTX_create - self.EVP_MD_CTX_create.restype = ctypes.c_void_p - self.EVP_MD_CTX_create.argtypes = [] - - self.EVP_MD_CTX_init = self._lib.EVP_MD_CTX_init - self.EVP_MD_CTX_init.restype = None - self.EVP_MD_CTX_init.argtypes = [ctypes.c_void_p] - - self.EVP_MD_CTX_destroy = self._lib.EVP_MD_CTX_destroy - self.EVP_MD_CTX_destroy.restype = None - self.EVP_MD_CTX_destroy.argtypes = [ctypes.c_void_p] - - self.RAND_bytes = self._lib.RAND_bytes - self.RAND_bytes.restype = None - self.RAND_bytes.argtypes = [ctypes.c_void_p, ctypes.c_int] - - self.EVP_sha256 = self._lib.EVP_sha256 - self.EVP_sha256.restype = ctypes.c_void_p - self.EVP_sha256.argtypes = [] - - self.EVP_sha512 = self._lib.EVP_sha512 - self.EVP_sha512.restype = ctypes.c_void_p - self.EVP_sha512.argtypes = [] - - self.HMAC = self._lib.HMAC - self.HMAC.restype = ctypes.c_void_p - self.HMAC.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, - ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] - - self.PKCS5_PBKDF2_HMAC = self._lib.PKCS5_PBKDF2_HMAC - self.PKCS5_PBKDF2_HMAC.restype = ctypes.c_int - self.PKCS5_PBKDF2_HMAC.argtypes = [ctypes.c_void_p, ctypes.c_int, - ctypes.c_void_p, ctypes.c_int, - ctypes.c_int, ctypes.c_void_p, - ctypes.c_int, ctypes.c_void_p] - - self._set_ciphers() - self._set_curves() - - def _set_ciphers(self): - self.cipher_algo = { - 'aes-128-cbc': CipherName('aes-128-cbc', self.EVP_aes_128_cbc, 16), - 'aes-256-cbc': CipherName('aes-256-cbc', self.EVP_aes_256_cbc, 16), - 'aes-128-cfb': CipherName('aes-128-cfb', self.EVP_aes_128_cfb128, 16), - 'aes-256-cfb': CipherName('aes-256-cfb', self.EVP_aes_256_cfb128, 16), - 'aes-128-ofb': CipherName('aes-128-ofb', self._lib.EVP_aes_128_ofb, 16), - 'aes-256-ofb': CipherName('aes-256-ofb', self._lib.EVP_aes_256_ofb, 16), - 'aes-128-ctr': CipherName('aes-128-ctr', self._lib.EVP_aes_128_ctr, 16), - 'aes-256-ctr': CipherName('aes-256-ctr', self._lib.EVP_aes_256_ctr, 16), - 'bf-cfb': CipherName('bf-cfb', self.EVP_bf_cfb64, 8), - 'bf-cbc': CipherName('bf-cbc', self.EVP_bf_cbc, 8), - 'rc4': CipherName('rc4', self.EVP_rc4, 128), # 128 is the initialisation size not block size - } - - def _set_curves(self): - self.curves = { - 'secp112r1': 704, - 'secp112r2': 705, - 'secp128r1': 706, - 'secp128r2': 707, - 'secp160k1': 708, - 'secp160r1': 709, - 'secp160r2': 710, - 'secp192k1': 711, - 'secp224k1': 712, - 'secp224r1': 713, - 'secp256k1': 714, - 'secp384r1': 715, - 'secp521r1': 716, - 'sect113r1': 717, - 'sect113r2': 718, - 'sect131r1': 719, - 'sect131r2': 720, - 'sect163k1': 721, - 'sect163r1': 722, - 'sect163r2': 723, - 'sect193r1': 724, - 'sect193r2': 725, - 'sect233k1': 726, - 'sect233r1': 727, - 'sect239k1': 728, - 'sect283k1': 729, - 'sect283r1': 730, - 'sect409k1': 731, - 'sect409r1': 732, - 'sect571k1': 733, - 'sect571r1': 734, - 'prime256v1': 415, - } - - def BN_num_bytes(self, x): - """ - returns the length of a BN (OpenSSl API) - """ - return int((self.BN_num_bits(x) + 7) / 8) - - def get_cipher(self, name): - """ - returns the OpenSSL cipher instance - """ - if name not in self.cipher_algo: - raise Exception("Unknown cipher") - return self.cipher_algo[name] - - def get_curve(self, name): - """ - returns the id of a elliptic curve - """ - if name not in self.curves: - raise Exception("Unknown curve") - return self.curves[name] - - def get_curve_by_id(self, id): - """ - returns the name of a elliptic curve with his id - """ - res = None - for i in self.curves: - if self.curves[i] == id: - res = i - break - if res is None: - raise Exception("Unknown curve") - return res - - def rand(self, size): - """ - OpenSSL random function - """ - buffer = self.malloc(0, size) - self.RAND_bytes(buffer, size) - return buffer.raw - - def malloc(self, data, size): - """ - returns a create_string_buffer (ctypes) - """ - buffer = None - if data != 0: - if sys.version_info.major == 3 and isinstance(data, type('')): - data = data.encode() - buffer = self.create_string_buffer(data, size) - else: - buffer = self.create_string_buffer(size) - return buffer - - -libname = ctypes.util.find_library('crypto') -if libname is None: - # For Windows ... - libname = ctypes.util.find_library('libeay32.dll') - -# Add some extra libs to try if the one ctypes finds doesn't work -openssl_lib_candidates = [libname, "libcrypto.so.1.0.0"] - - -for lib_to_try in openssl_lib_candidates: - try: - # Connect to OpenSSL and grab all the symbols we need - OpenSSL = _OpenSSL(lib_to_try) - - # If we get here, that worked. Just keep going. - break - except BaseException: # Try the next lib - traceback.print_exc() - pass - -if OpenSSL is None: - print(("Could not load a working OpenSSL from: {}".format( - openssl_lib_candidates))) - print("You probably need to do this:") - print("") - print("\t# wget http://www.openssl.org/source/openssl-1.0.1e.tar.gz") - print("\t# tar -xvzf openssl-1.0.1e.tar.gz") - print("\t# cd openssl-1.0.1e") - print("\t# ./config --prefix=/ --openssldir=/etc/ssl --libdir=lib shared") - print("\t# make") - print("\t# make install") - print("") - print ("Or otherwise ensure that either the system OpenSSL has all the " - "functions we need, or OpenSSL 1.0 is installed aslibcrypto.so.1.0.0 " - "in LD_LIBRARY_PATH") - - sys.exit(1) - -else: - print("Successfully loaded emergency backup crypto munitions!") - print("You really should install pyelliptic instead though.") - -# Stuff from cypher.py - - -class Cipher: - """ - Symmetric encryption - - import pyelliptic - iv = pyelliptic.Cipher.gen_IV('aes-256-cfb') - ctx = pyelliptic.Cipher("secretkey", iv, 1, ciphername='aes-256-cfb') - ciphertext = ctx.update('test1') - ciphertext += ctx.update('test2') - ciphertext += ctx.final() - - ctx2 = pyelliptic.Cipher("secretkey", iv, 0, ciphername='aes-256-cfb') - print ctx2.ciphering(ciphertext) - """ - - def __init__(self, key, iv, do, ciphername='aes-256-cbc'): - """ - do == 1 => Encrypt; do == 0 => Decrypt - """ - self.cipher = OpenSSL.get_cipher(ciphername) - self.ctx = OpenSSL.EVP_CIPHER_CTX_new() - if do == 1 or do == 0: - k = OpenSSL.malloc(key, len(key)) - IV = OpenSSL.malloc(iv, len(iv)) - OpenSSL.EVP_CipherInit_ex( - self.ctx, self.cipher.get_pointer(), 0, k, IV, do) - else: - raise Exception("RTFM ...") - - @staticmethod - def get_all_cipher(): - """ - static method, returns all ciphers available - """ - return list(OpenSSL.cipher_algo.keys()) - - @staticmethod - def get_blocksize(ciphername): - cipher = OpenSSL.get_cipher(ciphername) - return cipher.get_blocksize() - - @staticmethod - def gen_IV(ciphername): - cipher = OpenSSL.get_cipher(ciphername) - return OpenSSL.rand(cipher.get_blocksize()) - - def update(self, input): - i = OpenSSL.c_int(0) - buffer = OpenSSL.malloc(b"", len(input) + self.cipher.get_blocksize()) - inp = OpenSSL.malloc(input, len(input)) - if OpenSSL.EVP_CipherUpdate(self.ctx, OpenSSL.byref(buffer), - OpenSSL.byref(i), inp, len(input)) == 0: - raise Exception("[OpenSSL] EVP_CipherUpdate FAIL ...") - return buffer.raw[0:i.value] - - def final(self): - i = OpenSSL.c_int(0) - buffer = OpenSSL.malloc(b"", self.cipher.get_blocksize()) - if (OpenSSL.EVP_CipherFinal_ex(self.ctx, OpenSSL.byref(buffer), - OpenSSL.byref(i))) == 0: - raise Exception("[OpenSSL] EVP_CipherFinal_ex FAIL ...") - return buffer.raw[0:i.value] - - def ciphering(self, input): - """ - Do update and final in one method - """ - buff = self.update(input) - return buff + self.final() - - def __del__(self): - OpenSSL.EVP_CIPHER_CTX_cleanup(self.ctx) - OpenSSL.EVP_CIPHER_CTX_free(self.ctx) - -# Stuff from hash.py - - -def hmac_sha256(k, m): - """ - Compute the key and the message with HMAC SHA5256 - """ - key = OpenSSL.malloc(k, len(k)) - d = OpenSSL.malloc(m, len(m)) - md = OpenSSL.malloc(0, 32) - i = OpenSSL.pointer(OpenSSL.c_int(0)) - OpenSSL.HMAC(OpenSSL.EVP_sha256(), key, len(k), d, len(m), md, i) - return md.raw - - -# Stuff from ecc.py - -from hashlib import sha512 -from struct import pack, unpack - - -class ECC: - """ - Asymmetric encryption with Elliptic Curve Cryptography (ECC) - ECDH, ECDSA and ECIES - - import pyelliptic - - alice = pyelliptic.ECC() # default curve: sect283r1 - bob = pyelliptic.ECC(curve='sect571r1') - - ciphertext = alice.encrypt("Hello Bob", bob.get_pubkey()) - print bob.decrypt(ciphertext) - - signature = bob.sign("Hello Alice") - # alice's job : - print pyelliptic.ECC( - pubkey=bob.get_pubkey()).verify(signature, "Hello Alice") - - # ERROR !!! - try: - key = alice.get_ecdh_key(bob.get_pubkey()) - except: print("For ECDH key agreement,\ - the keys must be defined on the same curve !") - - alice = pyelliptic.ECC(curve='sect571r1') - print alice.get_ecdh_key(bob.get_pubkey()).encode('hex') - print bob.get_ecdh_key(alice.get_pubkey()).encode('hex') - - """ - - def __init__(self, pubkey=None, privkey=None, pubkey_x=None, - pubkey_y=None, raw_privkey=None, curve='sect283r1'): - """ - For a normal and High level use, specifie pubkey, - privkey (if you need) and the curve - """ - if isinstance(curve, str): - self.curve = OpenSSL.get_curve(curve) - else: - self.curve = curve - - if pubkey_x is not None and pubkey_y is not None: - self._set_keys(pubkey_x, pubkey_y, raw_privkey) - elif pubkey is not None: - curve, pubkey_x, pubkey_y, i = ECC._decode_pubkey(pubkey) - if privkey is not None: - curve2, raw_privkey, i = ECC._decode_privkey(privkey) - if curve != curve2: - raise Exception("Bad ECC keys ...") - self.curve = curve - self._set_keys(pubkey_x, pubkey_y, raw_privkey) - else: - self.privkey, self.pubkey_x, self.pubkey_y = self._generate() - - def _set_keys(self, pubkey_x, pubkey_y, privkey): - if self.raw_check_key(privkey, pubkey_x, pubkey_y) < 0: - self.pubkey_x = None - self.pubkey_y = None - self.privkey = None - raise Exception("Bad ECC keys ...") - else: - self.pubkey_x = pubkey_x - self.pubkey_y = pubkey_y - self.privkey = privkey - - @staticmethod - def get_curves(): - """ - static method, returns the list of all the curves available - """ - return list(OpenSSL.curves.keys()) - - def get_curve(self): - return OpenSSL.get_curve_by_id(self.curve) - - def get_curve_id(self): - return self.curve - - def get_pubkey(self): - """ - High level function which returns : - curve(2) + len_of_pubkeyX(2) + pubkeyX + len_of_pubkeyY + pubkeyY - """ - return b''.join((pack('!H', self.curve), - pack('!H', len(self.pubkey_x)), - self.pubkey_x, - pack('!H', len(self.pubkey_y)), - self.pubkey_y - )) - - def get_privkey(self): - """ - High level function which returns - curve(2) + len_of_privkey(2) + privkey - """ - return b''.join((pack('!H', self.curve), - pack('!H', len(self.privkey)), - self.privkey - )) - - @staticmethod - def _decode_pubkey(pubkey): - i = 0 - curve = unpack('!H', pubkey[i:i + 2])[0] - i += 2 - tmplen = unpack('!H', pubkey[i:i + 2])[0] - i += 2 - pubkey_x = pubkey[i:i + tmplen] - i += tmplen - tmplen = unpack('!H', pubkey[i:i + 2])[0] - i += 2 - pubkey_y = pubkey[i:i + tmplen] - i += tmplen - return curve, pubkey_x, pubkey_y, i - - @staticmethod - def _decode_privkey(privkey): - i = 0 - curve = unpack('!H', privkey[i:i + 2])[0] - i += 2 - tmplen = unpack('!H', privkey[i:i + 2])[0] - i += 2 - privkey = privkey[i:i + tmplen] - i += tmplen - return curve, privkey, i - - def _generate(self): - try: - pub_key_x = OpenSSL.BN_new() - pub_key_y = OpenSSL.BN_new() - - key = OpenSSL.EC_KEY_new_by_curve_name(self.curve) - if key == 0: - raise Exception("[OpenSSL] EC_KEY_new_by_curve_name FAIL ...") - if (OpenSSL.EC_KEY_generate_key(key)) == 0: - raise Exception("[OpenSSL] EC_KEY_generate_key FAIL ...") - if (OpenSSL.EC_KEY_check_key(key)) == 0: - raise Exception("[OpenSSL] EC_KEY_check_key FAIL ...") - priv_key = OpenSSL.EC_KEY_get0_private_key(key) - - group = OpenSSL.EC_KEY_get0_group(key) - pub_key = OpenSSL.EC_KEY_get0_public_key(key) - - if (OpenSSL.EC_POINT_get_affine_coordinates_GFp(group, pub_key, - pub_key_x, - pub_key_y, 0 - )) == 0: - raise Exception( - "[OpenSSL] EC_POINT_get_affine_coordinates_GFp FAIL ...") - - privkey = OpenSSL.malloc(0, OpenSSL.BN_num_bytes(priv_key)) - pubkeyx = OpenSSL.malloc(0, OpenSSL.BN_num_bytes(pub_key_x)) - pubkeyy = OpenSSL.malloc(0, OpenSSL.BN_num_bytes(pub_key_y)) - OpenSSL.BN_bn2bin(priv_key, privkey) - privkey = privkey.raw - OpenSSL.BN_bn2bin(pub_key_x, pubkeyx) - pubkeyx = pubkeyx.raw - OpenSSL.BN_bn2bin(pub_key_y, pubkeyy) - pubkeyy = pubkeyy.raw - self.raw_check_key(privkey, pubkeyx, pubkeyy) - - return privkey, pubkeyx, pubkeyy - - finally: - OpenSSL.EC_KEY_free(key) - OpenSSL.BN_free(pub_key_x) - OpenSSL.BN_free(pub_key_y) - - def get_ecdh_key(self, pubkey): - """ - High level function. Compute public key with the local private key - and returns a 512bits shared key - """ - curve, pubkey_x, pubkey_y, i = ECC._decode_pubkey(pubkey) - if curve != self.curve: - raise Exception("ECC keys must be from the same curve !") - return sha512(self.raw_get_ecdh_key(pubkey_x, pubkey_y)).digest() - - def raw_get_ecdh_key(self, pubkey_x, pubkey_y): - try: - ecdh_keybuffer = OpenSSL.malloc(0, 32) - - other_key = OpenSSL.EC_KEY_new_by_curve_name(self.curve) - if other_key == 0: - raise Exception("[OpenSSL] EC_KEY_new_by_curve_name FAIL ...") - - other_pub_key_x = OpenSSL.BN_bin2bn(pubkey_x, len(pubkey_x), 0) - other_pub_key_y = OpenSSL.BN_bin2bn(pubkey_y, len(pubkey_y), 0) - - other_group = OpenSSL.EC_KEY_get0_group(other_key) - other_pub_key = OpenSSL.EC_POINT_new(other_group) - - if (OpenSSL.EC_POINT_set_affine_coordinates_GFp(other_group, - other_pub_key, - other_pub_key_x, - other_pub_key_y, - 0)) == 0: - raise Exception( - "[OpenSSL] EC_POINT_set_affine_coordinates_GFp FAIL ...") - if (OpenSSL.EC_KEY_set_public_key(other_key, other_pub_key)) == 0: - raise Exception("[OpenSSL] EC_KEY_set_public_key FAIL ...") - if (OpenSSL.EC_KEY_check_key(other_key)) == 0: - raise Exception("[OpenSSL] EC_KEY_check_key FAIL ...") - - own_key = OpenSSL.EC_KEY_new_by_curve_name(self.curve) - if own_key == 0: - raise Exception("[OpenSSL] EC_KEY_new_by_curve_name FAIL ...") - own_priv_key = OpenSSL.BN_bin2bn( - self.privkey, len(self.privkey), 0) - - if (OpenSSL.EC_KEY_set_private_key(own_key, own_priv_key)) == 0: - raise Exception("[OpenSSL] EC_KEY_set_private_key FAIL ...") - - OpenSSL.ECDH_set_method(own_key, OpenSSL.ECDH_OpenSSL()) - ecdh_keylen = OpenSSL.ECDH_compute_key( - ecdh_keybuffer, 32, other_pub_key, own_key, 0) - - if ecdh_keylen != 32: - raise Exception("[OpenSSL] ECDH keylen FAIL ...") - - return ecdh_keybuffer.raw - - finally: - OpenSSL.EC_KEY_free(other_key) - OpenSSL.BN_free(other_pub_key_x) - OpenSSL.BN_free(other_pub_key_y) - OpenSSL.EC_POINT_free(other_pub_key) - OpenSSL.EC_KEY_free(own_key) - OpenSSL.BN_free(own_priv_key) - - def check_key(self, privkey, pubkey): - """ - Check the public key and the private key. - The private key is optional (replace by None) - """ - curve, pubkey_x, pubkey_y, i = ECC._decode_pubkey(pubkey) - if privkey is None: - raw_privkey = None - curve2 = curve - else: - curve2, raw_privkey, i = ECC._decode_privkey(privkey) - if curve != curve2: - raise Exception("Bad public and private key") - return self.raw_check_key(raw_privkey, pubkey_x, pubkey_y, curve) - - def raw_check_key(self, privkey, pubkey_x, pubkey_y, curve=None): - if curve is None: - curve = self.curve - elif isinstance(curve, str): - curve = OpenSSL.get_curve(curve) - else: - curve = curve - try: - key = OpenSSL.EC_KEY_new_by_curve_name(curve) - if key == 0: - raise Exception("[OpenSSL] EC_KEY_new_by_curve_name FAIL ...") - if privkey is not None: - priv_key = OpenSSL.BN_bin2bn(privkey, len(privkey), 0) - pub_key_x = OpenSSL.BN_bin2bn(pubkey_x, len(pubkey_x), 0) - pub_key_y = OpenSSL.BN_bin2bn(pubkey_y, len(pubkey_y), 0) - - if privkey is not None: - if (OpenSSL.EC_KEY_set_private_key(key, priv_key)) == 0: - raise Exception( - "[OpenSSL] EC_KEY_set_private_key FAIL ...") - - group = OpenSSL.EC_KEY_get0_group(key) - pub_key = OpenSSL.EC_POINT_new(group) - - if (OpenSSL.EC_POINT_set_affine_coordinates_GFp(group, pub_key, - pub_key_x, - pub_key_y, - 0)) == 0: - raise Exception( - "[OpenSSL] EC_POINT_set_affine_coordinates_GFp FAIL ...") - if (OpenSSL.EC_KEY_set_public_key(key, pub_key)) == 0: - raise Exception("[OpenSSL] EC_KEY_set_public_key FAIL ...") - if (OpenSSL.EC_KEY_check_key(key)) == 0: - raise Exception("[OpenSSL] EC_KEY_check_key FAIL ...") - return 0 - - finally: - OpenSSL.EC_KEY_free(key) - OpenSSL.BN_free(pub_key_x) - OpenSSL.BN_free(pub_key_y) - OpenSSL.EC_POINT_free(pub_key) - if privkey is not None: - OpenSSL.BN_free(priv_key) - - def sign(self, inputb): - """ - Sign the input with ECDSA method and returns the signature - """ - try: - size = len(inputb) - buff = OpenSSL.malloc(inputb, size) - digest = OpenSSL.malloc(0, 64) - md_ctx = OpenSSL.EVP_MD_CTX_create() - dgst_len = OpenSSL.pointer(OpenSSL.c_int(0)) - siglen = OpenSSL.pointer(OpenSSL.c_int(0)) - sig = OpenSSL.malloc(0, 151) - - key = OpenSSL.EC_KEY_new_by_curve_name(self.curve) - if key == 0: - raise Exception("[OpenSSL] EC_KEY_new_by_curve_name FAIL ...") - - priv_key = OpenSSL.BN_bin2bn(self.privkey, len(self.privkey), 0) - pub_key_x = OpenSSL.BN_bin2bn(self.pubkey_x, len(self.pubkey_x), 0) - pub_key_y = OpenSSL.BN_bin2bn(self.pubkey_y, len(self.pubkey_y), 0) - - if (OpenSSL.EC_KEY_set_private_key(key, priv_key)) == 0: - raise Exception("[OpenSSL] EC_KEY_set_private_key FAIL ...") - - group = OpenSSL.EC_KEY_get0_group(key) - pub_key = OpenSSL.EC_POINT_new(group) - - if (OpenSSL.EC_POINT_set_affine_coordinates_GFp(group, pub_key, - pub_key_x, - pub_key_y, - 0)) == 0: - raise Exception( - "[OpenSSL] EC_POINT_set_affine_coordinates_GFp FAIL ...") - if (OpenSSL.EC_KEY_set_public_key(key, pub_key)) == 0: - raise Exception("[OpenSSL] EC_KEY_set_public_key FAIL ...") - if (OpenSSL.EC_KEY_check_key(key)) == 0: - raise Exception("[OpenSSL] EC_KEY_check_key FAIL ...") - - OpenSSL.EVP_MD_CTX_init(md_ctx) - OpenSSL.EVP_DigestInit(md_ctx, OpenSSL.EVP_ecdsa()) - - if (OpenSSL.EVP_DigestUpdate(md_ctx, buff, size)) == 0: - raise Exception("[OpenSSL] EVP_DigestUpdate FAIL ...") - OpenSSL.EVP_DigestFinal(md_ctx, digest, dgst_len) - OpenSSL.ECDSA_sign(0, digest, dgst_len.contents, sig, siglen, key) - if (OpenSSL.ECDSA_verify(0, digest, dgst_len.contents, sig, - siglen.contents, key)) != 1: - raise Exception("[OpenSSL] ECDSA_verify FAIL ...") - - return sig.raw[0:siglen.contents.value] - - finally: - OpenSSL.EC_KEY_free(key) - OpenSSL.BN_free(pub_key_x) - OpenSSL.BN_free(pub_key_y) - OpenSSL.BN_free(priv_key) - OpenSSL.EC_POINT_free(pub_key) - OpenSSL.EVP_MD_CTX_destroy(md_ctx) - - def verify(self, sig, inputb): - """ - Verify the signature with the input and the local public key. - Returns a boolean - """ - try: - bsig = OpenSSL.malloc(sig, len(sig)) - binputb = OpenSSL.malloc(inputb, len(inputb)) - digest = OpenSSL.malloc(0, 64) - dgst_len = OpenSSL.pointer(OpenSSL.c_int(0)) - md_ctx = OpenSSL.EVP_MD_CTX_create() - - key = OpenSSL.EC_KEY_new_by_curve_name(self.curve) - - if key == 0: - raise Exception("[OpenSSL] EC_KEY_new_by_curve_name FAIL ...") - - pub_key_x = OpenSSL.BN_bin2bn(self.pubkey_x, len(self.pubkey_x), 0) - pub_key_y = OpenSSL.BN_bin2bn(self.pubkey_y, len(self.pubkey_y), 0) - group = OpenSSL.EC_KEY_get0_group(key) - pub_key = OpenSSL.EC_POINT_new(group) - - if (OpenSSL.EC_POINT_set_affine_coordinates_GFp(group, pub_key, - pub_key_x, - pub_key_y, - 0)) == 0: - raise Exception( - "[OpenSSL] EC_POINT_set_affine_coordinates_GFp FAIL ...") - if (OpenSSL.EC_KEY_set_public_key(key, pub_key)) == 0: - raise Exception("[OpenSSL] EC_KEY_set_public_key FAIL ...") - if (OpenSSL.EC_KEY_check_key(key)) == 0: - raise Exception("[OpenSSL] EC_KEY_check_key FAIL ...") - - OpenSSL.EVP_MD_CTX_init(md_ctx) - OpenSSL.EVP_DigestInit(md_ctx, OpenSSL.EVP_ecdsa()) - if (OpenSSL.EVP_DigestUpdate(md_ctx, binputb, len(inputb))) == 0: - raise Exception("[OpenSSL] EVP_DigestUpdate FAIL ...") - - OpenSSL.EVP_DigestFinal(md_ctx, digest, dgst_len) - ret = OpenSSL.ECDSA_verify( - 0, digest, dgst_len.contents, bsig, len(sig), key) - - if ret == -1: - return False # Fail to Check - else: - if ret == 0: - return False # Bad signature ! - else: - return True # Good - return False - - finally: - OpenSSL.EC_KEY_free(key) - OpenSSL.BN_free(pub_key_x) - OpenSSL.BN_free(pub_key_y) - OpenSSL.EC_POINT_free(pub_key) - OpenSSL.EVP_MD_CTX_destroy(md_ctx) - - @staticmethod - def encrypt(data, pubkey, ephemcurve=None, ciphername='aes-256-cbc'): - """ - Encrypt data with ECIES method using the public key of the recipient. - """ - curve, pubkey_x, pubkey_y, i = ECC._decode_pubkey(pubkey) - return ECC.raw_encrypt(data, pubkey_x, pubkey_y, curve=curve, - ephemcurve=ephemcurve, ciphername=ciphername) - - @staticmethod - def raw_encrypt(data, pubkey_x, pubkey_y, curve='sect283r1', - ephemcurve=None, ciphername='aes-256-cbc'): - if ephemcurve is None: - ephemcurve = curve - ephem = ECC(curve=ephemcurve) - key = sha512(ephem.raw_get_ecdh_key(pubkey_x, pubkey_y)).digest() - key_e, key_m = key[:32], key[32:] - pubkey = ephem.get_pubkey() - iv = OpenSSL.rand(OpenSSL.get_cipher(ciphername).get_blocksize()) - ctx = Cipher(key_e, iv, 1, ciphername) - ciphertext = ctx.ciphering(data) - mac = hmac_sha256(key_m, ciphertext) - return iv + pubkey + ciphertext + mac - - def decrypt(self, data, ciphername='aes-256-cbc'): - """ - Decrypt data with ECIES method using the local private key - """ - blocksize = OpenSSL.get_cipher(ciphername).get_blocksize() - iv = data[:blocksize] - i = blocksize - curve, pubkey_x, pubkey_y, i2 = ECC._decode_pubkey(data[i:]) - i += i2 - ciphertext = data[i:len(data) - 32] - i += len(ciphertext) - mac = data[i:] - key = sha512(self.raw_get_ecdh_key(pubkey_x, pubkey_y)).digest() - key_e, key_m = key[:32], key[32:] - if hmac_sha256(key_m, ciphertext) != mac: - raise RuntimeError("Fail to verify data") - ctx = Cipher(key_e, iv, 0, ciphername) - return ctx.ciphering(ciphertext) - - -""" - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. -""" diff --git a/blockchain/pybc/json_coin.py b/blockchain/pybc/json_coin.py deleted file mode 100644 index c0a744eda..000000000 --- a/blockchain/pybc/json_coin.py +++ /dev/null @@ -1,1258 +0,0 @@ -#!/usr/bin/env python2.7 -# json_coin.py: a coin implemented on top of pybc.coin -# with addition json_data field for transactions - -from __future__ import absolute_import -from __future__ import print_function -import six -from six.moves import range -if __name__ == '__main__': - import os.path as _p, sys - sys.path.insert(0, _p.abspath(_p.join(_p.dirname(_p.abspath(sys.argv[0])), '..'))) - -import struct -import hashlib -import traceback -import logging -import json - -try: - import pyelliptic -except BaseException: # pyelliptic didn't load. Either it's not installed or it can't find OpenSSL - from . import emergency_crypto_munitions as pyelliptic -from . import util -from .transactions import pack_transactions, unpack_transactions -from .coin import Transaction, CoinState, CoinBlockchain, Wallet - - -class JsonTransaction(Transaction): - """ - Represents a transaction on the blockchain with Json data field. - - A transaction is: - 1. a list of inputs represented by: - (transaction_hash, output_index, amount, destination, json_data) - 2. a list of outputs represented by: - (amounts, destination public key hash, json_data) - 3. a list of authorizations signing the previous two lists: - (public key, signature, json_data) - It also has a timestamp, so that two generation - transactions to the same destination address won't have the same hash. (If - you send two block rewards to the same address, make sure to use different - timestamps!) - - Everything except public keys are hashed sha512 (64 bytes). Public keys are - hashed sha256 (32 bytes). - - A transaction is properly authorized if all of the inputs referred to have - destination hashes that match public keys that signed the transaction. - - A generation transaction (or fee collection transaction) is a transaction - with no inputs. It thus requires no authorizations. - - Any input not sent to an output is used as a transaction fee, and added to - the block reward. - - Has a to_bytes and a from_bytes like Blocks do. - - """ - - def __repr__(self): - return str(self) - - def __str__(self): - """ - Represent this transaction as a string. - - """ - - # These are the lines we will return - lines = [] - - lines.append("---JsonTransaction {}---".format(util.time2string( - self.timestamp))) - lines.append("{} inputs".format(len(self.inputs))) - for transaction, index, amount, destination, json_data in self.inputs: - # Put every input (another transaction's output) - lines.append("\t{} addressed to {} from output {} of {} with {}".format( - amount, util.bytes2string(destination), index, - util.bytes2string(transaction), json.dumps(json_data))) - lines.append("{} outputs".format(len(self.outputs))) - for amount, destination, json_data in self.outputs: - # Put every output (an amount and destination public key hash) - lines.append("\t{} to {} with {}".format(amount, util.bytes2string(destination), - json.dumps(json_data))) - lines.append("{} authorizations".format(len(self.authorizations))) - for public_key, signature, json_data in self.authorizations: - # Put every authorizing key and signature. - lines.append("\tKey: {}".format(util.bytes2string(public_key))) - lines.append("\tSignature: {}".format( - util.bytes2string(signature))) - lines.append("\tJSON data: {}".format(json.dumps(json_data))) - - # Put the hash that other transactions use to talk about this one - lines.append("Hash: {}".format(util.bytes2string( - self.transaction_hash()))) - - return "\n".join(lines) - - def add_input(self, transaction_hash, output_index, amount, destination, json_data=None): - """ - Take the coins from the given output of the given transaction as input - for this transaction. It is necessary to specify and store the amount - and destination public key hash of the output, so that the blockchain - can be efficiently read backwards. - Optional json_data field can be passed to store some data in the transaction. - """ - self.inputs.append((transaction_hash, output_index, amount, destination, json_data)) - - def add_output(self, amount, destination, json_data=None): - """ - Send the given amount of coins to the public key with the given hash. - Optional json_data field can be passed to store some data in the transaction. - """ - self.outputs.append((amount, destination, json_data)) - - def add_authorization(self, public_key, signature, json_data=None): - """ - Add an authorization to this transaction by the given public key. The - given signature is that key's signature of the transaction header data - (inputs and outputs). - Both public key and signature must be bytestrings. - Optional json_data field can be passed to store some data in the transaction. - """ - self.authorizations.append((public_key, signature, json_data)) - - def get_leftover(self): - """ - Return the sum of all inputs minus the sum of all outputs. - """ - - # This is where we store our total - leftover = 0 - - for _, _, amount, _, _ in self.inputs: - # Add all the inputs on the + side - leftover += amount - - for amount, _, _ in self.outputs: - # Add all the outputs on the - side - leftover -= amount - - return leftover - - def verify_authorizations(self): - """ - Returns True if, for every public key hash in the transaction's inputs, - there is a valid authorization signature of this transaction by a public - key with that hash. - - """ - - # Get the bytestring that verifications need to sign - message_to_sign = self.header_bytes() - - # This holds SHA256 hashes of all the pubkeys with valid signatures - valid_signers = set() - - for public_key, signature, _ in self.authorizations: - # Check if each authorization is valid. - if pyelliptic.ECC(pubkey=public_key).verify(signature, message_to_sign): - - # The signature is valid. Remember the public key hash. - valid_signers.add(hashlib.sha256(public_key).digest()) - - else: - logging.warning("Invalid signature!") - # We're going to ignore extra invalid signatures on - # transactions. What could go wrong? - - for _, _, _, destination, _ in self.inputs: - if destination not in valid_signers: - # This input was not properly unlocked. - return False - - # If we get here, all inputs were to destination pubkey hashes that has - # authorizing signatures attached. - return True - - def pack_input(self, transaction_hash, output_index, amount, destination, json_data): - """ - Return packet input as a bytestring. - """ - jdata = json.dumps(json_data) - return struct.pack( - ">64sIQ32sI", - transaction_hash, - output_index, - amount, - destination, - len(jdata)) + jdata - - def pack_inputs(self): - """ - Return the inputs as a bytestring. - - """ - - # Return the 4-byte number of inputs, followed by a 64-byte transaction - # hash, a 4-byte output index, an 8 byte amount, and a 32 byte - # destination public key hash for each input. - # also added json data for every input: - # 4-byte length and n-bytes string - return struct.pack(">I", len(self.inputs)) + "".join( - self.pack_input(*inpt) for inpt in self.inputs) - - def unpack_input(self, bytestring, offset=0): - """ - Extract single transaction input from bytestring. - """ - transaction_hash, output_index, amount, destination, json_data_len = struct.unpack( - ">64sIQ32sI", bytestring[offset:offset + 112]) - json_data = json.loads(bytestring[offset + 112:offset + 112 + json_data_len]) - return 112 + json_data_len, transaction_hash, output_index, amount, destination, json_data - - def unpack_inputs(self, bytestring): - """ - Set this transaction's inputs to those encoded by the given bytestring. - - """ - - # Start with a fresh list of inputs. - self.inputs = [] - # How many inputs are there? - (input_count,) = struct.unpack(">I", bytestring[0:4]) - - # Where are we in the string - index = 4 - - for _ in range(input_count): - # Unpack that many 108-byte records of 64-byte transaction hashes, - # 4-byte output indices, 8-byte amounts, and 32-byte destination - # public key hashes. - parts = self.unpack_input(bytestring, index) - self.inputs.append(parts[1:]) - index += parts[0] - - def pack_output(self, amount, destination, json_data): - """ - Pack single output ite as a bytestring. - """ - jdata = json.dumps(json_data) - return struct.pack(">Q32sI", amount, destination, len(jdata)) + jdata - - def pack_outputs(self): - """ - Return the outputs as a bytestring. - - """ - - # Return the 4-byte number of outputs, followed by an 8-byte amount and - # a 32-byte destination public key hash for each output - # also added json data for every input: - # 4-byte length and n-bytes string - - return struct.pack(">I", len(self.outputs)) + "".join( - self.pack_output(*outpt) for outpt in self.outputs) - - def unpack_output(self, bytestring, offset=0): - amount, destination, json_data_len = struct.unpack( - ">Q32sI", bytestring[offset:offset + 44]) - json_data = json.loads(bytestring[offset + 44:offset + 44 + json_data_len]) - return 44 + json_data_len, amount, destination, json_data - - def unpack_outputs(self, bytestring): - """ - Set this transaction's outputs to those encoded by the given bytestring. - - """ - - # Start with a fresh list of outputs. - self.outputs = [] - - # How many outputs are there? - (output_count,) = struct.unpack(">I", bytestring[0:4]) - - # Where are we in the string - index = 4 - - for _ in range(output_count): - # Unpack that many 40-byte records of 8-byte amounts and 32-byte - # destination public key hashes. - parts = self.unpack_output(bytestring, index) - self.outputs.append(parts[1:]) - index += parts[0] - - def pack_authorizations(self): - """ - Return a bytestring of all the authorizations for this transaction. - - """ - - # We have a 4-byte number of authorization records, and then pairs of 4 - # -byte-length and n-byte-data strings for each record. - - # This holds all our length-delimited bytestrings as we make them - authorization_bytestrings = [] - - for public_key, signature, json_data in self.authorizations: - # Add the public key - authorization_bytestrings.append(struct.pack(">I", - len(public_key)) + public_key) - # Add the signature - authorization_bytestrings.append(struct.pack(">I", - len(signature)) + signature) - # Add json data - jdata = json.dumps(json_data) - authorization_bytestrings.append(struct.pack(">I", - len(jdata)) + jdata) - - # Send back the number of records and all the records. - return (struct.pack(">I", len(self.authorizations)) + - "".join(authorization_bytestrings)) - - def unpack_authorizations(self, bytestring): - """ - Set this transaction's authorizations to those encoded by the given - bytestring. - - """ - - # Start with a fresh list of authorizations. - self.authorizations = [] - - # How many outputs are there? - (authorization_count,) = struct.unpack(">I", bytestring[0:4]) - - # Where are we in the string - index = 4 - - for _ in range(authorization_count): - # Get the length of the authorization's public key - (length,) = struct.unpack(">I", bytestring[index:index + 4]) - index += 4 - - # Get the public key itself - public_key = bytestring[index: index + length] - index += length - - # Get the length of the authorization's signature - (length,) = struct.unpack(">I", bytestring[index:index + 4]) - index += 4 - - # Get the signature itself - signature = bytestring[index: index + length] - index += length - - # Get the length of the json data - (length,) = struct.unpack(">I", bytestring[index:index + 4]) - index += 4 - - # Get the signature itself - jdata = bytestring[index: index + length] - index += length - json_data = json.loads(jdata) - - # Add the authorization - self.authorizations.append((public_key, signature, json_data)) - - -class JsonCoinState(CoinState): - """ - A State that keeps track of all unused outputs in blocks. Can also be used - as a generic persistent set of output tuples. - Keep track of Json data stored in the transactions. - """ - - def json_output2bytes(self, output): - """ - Turn an unused output tuple into a bytestring: - (transaction hash, output index, output amount, destination, json_data) - """ - as_list = list(output) - json_data = as_list[4] - jdata = json.dumps(json_data) - as_list[4] = len(jdata) - result = struct.pack(">64sIQ32sI", *as_list) + jdata - del as_list - return result - - def bytes2json_output(self, rawbytes): - """ - Turn a bytestring into a tuple of: - (transaction hash, output index, output amount, destination, json_data) - """ - output = list(struct.unpack(">64sIQ32s", rawbytes[:108])) - json_data = None - if len(rawbytes) == 108: - output += [json_data, ] - return tuple(output) - json_data_length = struct.unpack(">I", rawbytes[108:112]) - json_data_raw = rawbytes[112:112 + json_data_length] - json_data = json.loads(json_data_raw) - output += [json_data, ] - return tuple(output) - - def add_unused_json_output(self, output): - """ - Add the given output tuple as an unused output. - Store json data (last element of the tuple) as a value, - and exclude it from key string. - """ - key = self.output2bytes(list(output)[:4]) - value = json.dumps(output[4]) - self.unused_outputs.insert(key, value) - - def remove_unused_json_output(self, output): - """ - Remove the given output tuple as an unused output. - Last element storing json data will be excluded from key string. - """ - key = self.output2bytes(list(output)[:4]) - self.unused_outputs.remove(key) - - def get_unused_json_outputs(self): - """ - Yield each unused output, as a tuple: - (transaction hash, index, amount, destination, json_data) - """ - for key, value in six.iteritems(self.unused_outputs): - output4 = list(self.bytes2output(key)) - output = tuple(output4 + [json.loads(value), ]) - yield output - - def has_unused_json_output(self, output): - """ - Returns true if the given unised output tuple of of is in the current set of - unused outputs, and false otherwise: - (transaction hash, output index, output amount, destination, json_data) - """ - key = self.output2bytes(list(output)[:4]) - return self.unused_outputs.find(key) is not None - - def apply_transaction(self, transaction): - """ - Update this state by applying the given Transaction object. The - transaction is assumed to be valid. - - """ - # Get the hash of the transaction - transaction_hash = transaction.transaction_hash() - - logging.debug("Applying transaction {}".format(util.bytes2string(transaction_hash))) - - for spent_output in transaction.inputs: - # The inputs are in exactly the same tuple format as we use. - # Remove each of them from the unused output set. - - if not self.has_unused_json_output(spent_output): - logging.error("Trying to remove nonexistent output: {} {} {} " - "{}".format(util.bytes2string(spent_output[0]), - spent_output[1], spent_output[2], - util.bytes2string(spent_output[3]))) - - self.audit() - - # See if things just work out - else: - self.remove_unused_json_output(spent_output) - - for i, output in enumerate(transaction.outputs): - # Unpack each output tuple - if len(output) == 3: - amount, destination, json_data = output - else: - raise Exception('Wrong output: {}'.format(output)) - - # Make a tupe for the full unused output - unused_output = (transaction_hash, i, amount, destination, json_data) - - if self.has_unused_json_output(unused_output): - logging.error("Trying to create duplicate output: {} {} {} " - "{}".format(util.bytes2string(unused_output[0]), - unused_output[1], unused_output[2], - util.bytes2string(unused_output[3]))) - - self.audit() - - # See if things just work out - else: - self.add_unused_json_output(unused_output) - - def remove_transaction(self, transaction): - """ - Update this state by removing the given Transaction object. - - """ - - # Get the hash of the transaction - transaction_hash = transaction.transaction_hash() - - logging.debug("Reverting transaction {}".format(util.bytes2string( - transaction_hash))) - - for spent_output in transaction.inputs: - # The inputs are in exactly the same tuple format as we use. Add - # them back to the unused output set, pointing to our useless value - # of "". - - if self.has_unused_json_output(spent_output): - logging.error("Trying to create duplicate output: {} {} {} " - "{}".format(util.bytes2string(spent_output[0]), - spent_output[1], spent_output[2], - util.bytes2string(spent_output[3]))) - - self.audit() - - # See if things just work out - else: - self.add_unused_json_output(spent_output) - - for i, output in enumerate(transaction.outputs): - # Unpack each output tuple - amount, destination, json_data = output - - # Make a tuple for the full unused output - unused_output = (transaction_hash, i, amount, destination, json_data) - - if not self.has_unused_json_output(unused_output): - logging.error("Trying to remove nonexistent output: {} {} {} " - "{}".format(util.bytes2string(unused_output[0]), - unused_output[1], unused_output[2], - util.bytes2string(unused_output[3]))) - - # Audit ourselves - self.audit() - - # See if things just work out - else: - # Remove its record from the set of unspent outputs - self.remove_unused_json_output(unused_output) - - def step_forwards(self, block): - """ - Add any outputs of this block's transactions to the set of unused - outputs, and consume all the inputs. - - Updates the CoinState in place. - """ - - for transaction_bytes in unpack_transactions(block.payload): - - # Parse the transaction - json_transaction = JsonTransaction.from_bytes(transaction_bytes) - - self.apply_transaction(json_transaction) - - if self.get_hash() != block.state_hash: - # We've stepped forwards incorrectly - raise Exception("Stepping forward to state {} instead produced state {}.".format( - util.bytes2string(block.state_hash), - util.bytes2string(self.get_hash()))) - - def step_backwards(self, block): - """ - Add any inputs from this block to the set of unused outputs, and remove - all the outputs. - - Updates the CoinState in place. - """ - - if self.get_hash() != block.state_hash: - # We're trying to step back the wrong block - raise Exception("Stepping back block for state {} when actually in state {}.".format( - util.bytes2string(block.state_hash), - util.bytes2string(self.get_hash()))) - - logging.debug("Correctly in state {} before going back.".format( - util.bytes2string(self.get_hash()))) - - for transaction_bytes in reversed(list(unpack_transactions(block.payload))): - # Parse the transaction - json_transaction = JsonTransaction.from_bytes(transaction_bytes) - self.remove_transaction(json_transaction) - - def copy(self): - """ - Return a copy of the CoinState that can be independently operated on. - This CoinState must have had no modifications made to it since its - creation or last commit() operation. Modifications made to this - CoinState will invalidate the copy and its descendents. - - """ - - # Make a new JsonCoinState, with its AuthenticatedDictionary a child of - # ours. - return JsonCoinState(unused_outputs=self.unused_outputs.copy()) - - -class JsonCoinBlockchain(CoinBlockchain): - """ - Represents a Blockchain for a Bitcoin-like currency. - - """ - - def __init__(self, block_store, minification_time=None, state=None): - """ - Make a new CoinBlockchain that stores blocks and state in the specified - file. If a minification_time is specified, accept mini-blocks and throw - out the bodies of blocks burried deeper than the minification time. - - """ - - # Just make a new Blockchain using the default POW algorithm and a - # CoinState to track unspent outputs. Store/load the state to/from the - # "state" table of the blockstore database. Because the state and the - # other blockstore components use the same database, they can't get out - # of sync; the Blockchain promises never to sync its databases without - # committing its State. - super(JsonCoinBlockchain, self).__init__( - block_store, - minification_time=minification_time, - state=state or JsonCoinState(filename=block_store, table="state"), - ) - - # Set up the blockchain for 1 minute blocks, retargeting every 10 - # blocks - # This is in blocks - self.retarget_period = 10 - # This is in seconds - self.retarget_time = self.retarget_period * 60 - - def transaction_valid_for_relay(self, transaction_bytes): - """ - Say that normal transactions can be accepted from peers, but generation - and fee collection transactions cannot. - - """ - - if len(JsonTransaction.from_bytes(transaction_bytes).inputs) > 0: - # It has an input, so isn't a reward. - return True - - # No inputs. Shouldn't accept this, even if it's valid. It will steal - # our fees. - return False - - def get_block_reward(self, previous_block): - """ - Get the block reward for a block based on the given previous block, - which may be None. - """ - # How many coins should we generate? We could do something based on - # height, but just be easy. - coins = 5 - logging.debug("Block reward is {} coins".format(coins)) - # Return the number of coins to generate - return coins - - def verify_payload(self, next_block): - """ - Verify all the transactions in the block as a group. Each individually - gets validated the normal way with verify_transaction, but after that's - done we make sure that the total excess coin spent exactly equals the - amount that's supposed to be generated. - - """ - - if not super(CoinBlockchain, self).verify_payload(next_block): - # Some transaction failed basic per-transaction validation. - return False - - # How much left-over coin do we have in this block so far? Start with - # the block reward. - block_leftover = self.get_block_reward(next_block) - - for transaction_bytes in unpack_transactions(next_block.payload): - # Parse a Transaction object out - json_transaction = JsonTransaction.from_bytes(transaction_bytes) - - # Add (or remove) the transaction's leftover coins from the block's - # leftover coins - block_leftover += json_transaction.get_leftover() - - if block_leftover == 0: - # All the fees and rewards went to the exact right places. Only - # transactions with no inputs can take from the fees at all (as per - # verify_transaction). - return True - else: - # Reject the block if its transactions have uncollected - # fees/rewards, or if it tries to give out more in fees and rewards - # than it deserves. - logging.warning("Block disburses rewards/fees incorrectly.") - return False - - def verify_transaction(self, transaction_bytes, chain_head, state, advance=False): - """ - If the given Transaction is valid on top of the given chain head block - (which may be None), in the given State (which may be None), return - True. Otherwise, return False. If advance is True, and the transaction - is valid, advance the State in place. - - Ensures that: - - The transaction's inputs are existing unspent outputs that the other - transactions didn't use. - - The transaction's outputs are not already present as unspent outputs - (in case someone tries to put in the same generation transaction twice). - - The transaction's authorizations are sufficient to unlock its inputs. - from pybc.transactions import pack_transactions, unpack_transactions - The transaction's outputs do not excede its inputs, if it has inputs. - - """ - - try: - # This holds our parsed transaction - json_transaction = JsonTransaction.from_bytes(transaction_bytes) - except BaseException: # The transaction is uninterpretable - logging.warning("Uninterpretable transaction.") - traceback.print_exc() - return False - - for i, (amount, destination, json_data) in enumerate(json_transaction.outputs): - if amount == 0: - logging.warning("Transaction trying to send a 0 output.") - return False - - # What unused output tuple would result from this? They all need to - # be unique. - unused_output = (json_transaction.transaction_hash(), i, amount, destination, json_data) - - if state is not None: - if state.has_unused_json_output(unused_output): - # We have a duplicate transaction. Probably a generation - # transaction, since all others need unspent inputs. - logging.warning("Transaction trying to create a duplicate unused output") - logging.info("{}".format(json_transaction)) - return False - else: - # If the State is None, we have to skip verifying that these - # outputs are unused - - logging.debug("Not checking for duplicate output, since we have no State.") - - # Outputs can never be negative since they are unsigned. So we don't - # need to check that. - - if len(json_transaction.inputs) == 0: - # This is a fee-collecting/reward-collecting transaction. We can't - # verify them individually, but we can make sure the total they come - # to is not too big or too small in verify_payload. - - if advance and state is not None: - # We're supposed to advance the state since transaction is valid - state.apply_transaction(json_transaction) - - return True - - # Now we know the transaction has inputs. Check them. - - for source in json_transaction.inputs: - if state is not None: - # Make sure each input is accounted for by a previous unused - # output. - if not state.has_unused_json_output(source): - # We're trying to spend something that doesn't exist or is - # already spent. - logging.warning("Transaction trying to use spent or nonexistent input") - return False - else: - logging.debug("Not checking for re-used input, since we have no State.") - - if json_transaction.get_leftover() < 0: - # Can't spend more than is available to the transaction. - logging.warning("Transaction trying to output more than it inputs") - return False - - if not json_transaction.verify_authorizations(): - # The transaction isn't signed properly. - logging.warning("Transaction signature(s) invalid") - return False - - # If we get here, the transaction must be valid. All its inputs are - # authorized, and its outputs aren't too large. - - if advance and state is not None: - # We're supposed to advance the state since transaction is valid - state.apply_transaction(json_transaction) - - return True - - def make_block(self, destination, min_fee=1, json_data=None, with_inputs=False): - """ - Override CoinBlockchain make_block with a additional json_data parameter. - """ - # Don't let anybody mess with our transactions and such until we've made - # the block. It can still be rendered invalid after that, though. - with self.lock: - - logging.debug("Making a block with JSON: {}".format(json_data)) - - if not self.state_available: - # We can't mine without the latest State - logging.debug("Can't make block without State") - return None - - if with_inputs and not self.transactions: - logging.debug("No transactions found, skip block generation") - return None - - # This holds the list of Transaction objects to include - to_include = [] - all_inputs = [] - - # This holds the total fee available, starting with the block - # reward. - total_fee = self.get_block_reward(self.highest_block) - - # This holds the state that we will move to when we apply all these - # transactions. We already know none of the transactions in - # self.transactions conflict, and that none of them depend on each - # other, so we can safely advance the State with them. - next_state = self.state.copy() - - # How many bytes of transaction data have we used? - transaction_bytes_used = 0 - - for transaction_bytes in self.transactions.values(): - # Parse this transaction out. - json_transaction = JsonTransaction.from_bytes(transaction_bytes) - - # Get how much it pays - fee = json_transaction.get_leftover() - - if fee >= min_fee: - # This transaction pays enough. Use it. - - if self.verify_transaction(transaction_bytes, self.highest_block, next_state, advance=True): - - # The transaction is OK to go in the block. We checked - # it earlier, but we should probably check it again. - to_include.append(json_transaction) - total_fee += fee - transaction_bytes_used += len(transaction_bytes) - all_inputs.extend(json_transaction.inputs) - else: - logging.info('INVALID TRANSACTION:\n{}'.format(str(json_transaction))) - - if transaction_bytes_used >= 1024 * 1024: - # Don't make blocks bigger than 1 MB of transaction data - logging.info("Hit block size limit for generation.") - break - - # Add a transaction that gives all the generated coins and fees to - # us. - reward_transaction = JsonTransaction() - reward_json = {'j': json_data, 'f': total_fee, } - reward_transaction.add_output(total_fee, destination, reward_json) - - # This may match exactly the reward transaction in the last block if - # we just made and solved that. What unused output does the - # generation transaction produce? - if not self.verify_transaction(reward_transaction.to_bytes(), - self.highest_block, next_state, advance=True): - - # Our reward-taking transaction is invalid for some reason. We - # probably already generated a block this second or something. - - logging.info("Reward transaction would be invalid (maybe " - "already used). Skipping block creation.") - - return None - - to_include.append(reward_transaction) - - # Make a block moving to the state we have after we apply all those - # transactions, with the transaction packed into its payload - block = super(CoinBlockchain, self).make_block( - next_state, pack_transactions( - [transaction.to_bytes() for transaction in to_include])) - - if block is None: - logging.debug("Base class cound not make block") - return block - - # next_state gets discarded without being committed, which is - # perfectly fine. It won't touch the database at all, not even by - # making unsynced changes, unless we commit. - - def dump_block(self, block): - """ - Return a string version of the block, with string versions of all the - Transactions appended, for easy viewing. - - """ - - # Keep a list of all the parts to join - parts = [str(block)] - - if block.has_body: - for transaction_bytes in unpack_transactions(block.payload): - - parts.append(str(JsonTransaction.from_bytes(transaction_bytes))) - - return "\n".join(parts) - - def iterate_transactions_by_address(self, address, include_inputs=True, include_outputs=True): - for block in self.longest_chain(): - for transaction_bytes in unpack_transactions(block.payload): - tr = JsonTransaction.from_bytes(transaction_bytes) - tr.block_hash = block.block_hash() - related = False - if include_inputs: - for inpt in tr.inputs: - if inpt[3] == address: - related = True - break - if include_outputs: - for outp in tr.outputs: - if outp[1] == address: - related = True - break - if related: - yield tr - - -class JsonWallet(Wallet): - """ - """ - - def __init__(self, blockchain, filename, state=None): - """ - """ - super(JsonWallet, self).__init__( - blockchain, - filename, - state=state or JsonCoinState(filename=filename, table="spendable"), - ) - - def blockchain_event(self, event, argument): - """ - """ - - # TODO: Track balance as an int for easy getting. - - with self.lock: - if event == "forward": - logging.debug("Advancing wallet forward") - # We've advanced forward a block. Get any new spendable outputs. - for transaction_bytes in unpack_transactions(argument.payload): - - # Parse the transaction - json_transaction = JsonTransaction.from_bytes(transaction_bytes) - - transaction_hash = json_transaction.transaction_hash() - - for spent_output in json_transaction.inputs: - if self.spendable.has_unused_json_output(spent_output): - # This output we had available got spent - self.spendable.remove_unused_json_output(spent_output) - - for i, output in enumerate(json_transaction.outputs): - # Unpack each output tuple - amount, destination, json_data = output - - if destination in self.keystore: - # This output is spendable by us - self.spendable.add_unused_json_output(( - transaction_hash, i, amount, destination, json_data, - )) - - # Re-set our set of transactions we should draw upon to all the - # transactions available - self.willing_to_spend = self.spendable.copy() - - elif event == "backward": - # We've gone backward a block - - logging.debug("Advancing wallet backward") - - for transaction_bytes in unpack_transactions(argument.payload): - - # Parse the transaction - json_transaction = JsonTransaction.from_bytes(transaction_bytes) - - transaction_hash = json_transaction.transaction_hash() - - for spent_output in json_transaction.inputs: - if spent_output[3] in self.keystore: - # This output we spent got unspent - self.spendable.add_unused_json_output(spent_output) - - for i, output in enumerate(json_transaction.outputs): - # Unpack each output tuple - amount, destination, json_data = output - - # Make an output tuple in the full format - spent_output = (transaction_hash, i, amount, destination, json_data) - - if self.spendable.has_unused_json_output(spent_output): - # This output we had available to spend got un-made - self.spendable.remove_unused_json_output(spent_output) - - # Re-set our set of transactions we should draw upon to all the - # transactions available - self.willing_to_spend = self.spendable.copy() - - elif event == "reset": - # Argument is a CoinState that's not really related to our - # previous one. - - logging.info("Rebuilding wallet's index of spendable outputs.") - - # Throw out our current idea of our spendable outputs. - self.spendable.clear() - - # How many outputs are for us? - found = 0 - # And how much are they worth? - balance = 0 - - for unused_output in argument.get_unused_json_outputs(): - if unused_output[3] in self.keystore: - # This output is addressed to us. Say it's spendable - self.spendable.add_unused_json_output(unused_output) - logging.debug("\t{} to {}".format(unused_output[2], - util.bytes2string(unused_output[3]))) - found += 1 - balance += unused_output[2] - - logging.info("{} outputs available, totaling {}".format(found, balance)) - - # Re-set our set of transactions we should draw upon to all the - # transactions available - self.willing_to_spend = self.spendable.copy() - - elif event == "sync": - # Save anything to disk that depends on the blockchain. This - # doesn't guarantee that we won't get out of sync with the - # blockchain, but it helps. - - # TODO: We happen to know it's safe to look at state here, but - # in general it's not. Also, the state may be invalid or the top - # block, but if that's true we'll get a reset when we have a - # better state. - - logging.info("Saving wallet") - - # Save the blockchain state that we are up to date with. - self.wallet_metadata["blockchain_state"] = self.blockchain.state.get_hash() - - self.spendable.commit() - self.keystore.sync() - - else: - logging.warning("Unknown event {} from blockchain".format(event)) - - def get_balance(self): - """ - Return the total balance of all spendable outputs. - - """ - - # This holds the balance so far - balance = 0 - - for _, _, amount, _, _ in self.spendable.get_unused_json_outputs(): - # Sum up the amounts over all spendable outputs - balance += amount - - return balance - - def make_simple_transaction(self, amount, destination, fee=1, - json_data=None, auth_data=None, spendable_filter=None): - """ - Return a JsonTransaction object sending the given amount to the given - destination, and any remaining change back to ourselves, leaving the - specified miner's fee unspent. - - Optional json_data field can be passed as JSON to the transaction output. - Optional auth_data field can be passed as JSON to the authorization. - - If we don't have enough in outputs that we're willing to spend (i.e. - which we haven't used to make transaction already, and which aren't - change that hasn't been confirmed yet), return None. - - If the amount isn't strictly positive, also returns None, since such a - transaction would be either useless or impossible depending on the - actual value. - - Destination must be a 32-byte public key SHA256 hash. - - A negative fee can be passed, but the resulting transaction will not be - valid. - - """ - with self.lock: - - if not amount > 0: - # Transaction is unreasonable: not sending any coins anywhere. - return None - - # Make a transaction - json_transaction = JsonTransaction() - - # This holds how much we have accumulated from the spendable outputs - # we've added to the transaction's inputs. - coins_collected = 0 - - # This holds the set of public key hashes that we need to sign the - # transaction with. - key_hashes = set() - - # This holds our willing_to_spend set with updates - willing_to_spend = self.willing_to_spend.copy() - - # This holds our outputs we need to take out of - # willing_to_spend when done iterating over it. - spent = [] - - for spendable in willing_to_spend.get_unused_json_outputs(): - if spendable_filter and not spendable_filter(spendable): - continue - # Unpack the amount we get from this as an input, and the key we - # need to use to spend it. - _, _, input_amount, key_needed, _ = spendable - # Add the unspent output as an input to the transaction - json_transaction.add_input(*spendable) - - # Say we've collected that many coins - coins_collected += input_amount - - # Say we need to sign with the appropriate key - key_hashes.add(key_needed) - - # Say we shouldn't spend this again in our next transaction. - spent.append(spendable) - - if coins_collected >= amount + fee: - # We have enough coins. - break - - for output in spent: - # Mark all the outputs we just tried to spend as used until a - # block comes along either confirming or denying this. - willing_to_spend.remove_unused_json_output(output) - - if coins_collected < amount + fee: - # We couldn't find enough money for this transaction! - # Maybe wait until some change transactions get into blocks. - return None - - # We're going through with the transaction. Don't re-use these - # inputs until after the next block comes in. - self.willing_to_spend = willing_to_spend - - # We've made a transaction with enough inputs! - # Add the outputs. - # First the amount we actually wanted to send. - json_transaction.add_output(amount, destination, json_data) - - if coins_collected - amount - fee > 0: - # Then the change, if any, back to us at some address we can - # receive on. - json_transaction.add_output( - amount=coins_collected - amount - fee, - destination=self.get_address(), - json_data={'f': fee, 'c': coins_collected, }, - ) - # The fee should be left over. - - # Now do the authorizations. What do we need to sign? - to_sign = json_transaction.header_bytes() - - for key_hash in key_hashes: - # Load the keypair - keypair = self.keystore[key_hash] - - # Grab the public key - public_key = keypair.get_pubkey() - - # Make the signature - signature = keypair.sign(to_sign) - - # Add the authorization to the transaction - json_transaction.add_authorization(public_key, signature, auth_data) - - # TODO: If we have a change output, put it in the willing to spend - # set so we can immediately spend from it. - - # Now the transaction is done! - return json_transaction - - -if __name__ == "__main__": - # Do a transaction test - - def generate_block(blockchain, destination, min_fee=1, json_data=None): - """ - Given a blockchain, generate a block (synchronously!), sending the - generation reward to the given destination public key hash. - - min_fee specifies the minimum fee to charge. - - TODO: Move this into the Blockchain's get_block method. - - """ - - # Make a block with the transaction as its payload - block = blockchain.make_block(destination, min_fee=min_fee, json_data=json_data) - - # Now unpack and dump the block for debugging. - print("Block will be:\n{}".format(block)) - - for transaction in unpack_transactions(block.payload): - # Print all the transactions - print("JsonTransaction: {}".format(JsonTransaction.from_bytes(transaction))) - - # Do proof of work on the block to mine it. - block.do_work(blockchain.algorithm) - - print("Successful nonce: {}".format(block.nonce)) - - # See if the work really is enough - print("Work is acceptable: {}".format(block.verify_work(blockchain.algorithm))) - - # See if the block is good according to the blockchain - print("Block is acceptable: {}".format(blockchain.verify_block(block))) - - # Add it to the blockchain through the complicated queueing mechanism - blockchain.queue_block(block) - - # Make a blockchain - blockchain = JsonCoinBlockchain("coin.blocks") - - # Make a wallet that hits against it - wallet = JsonWallet(blockchain, "coin.wallet") - - print("Receiving address: {}".format(util.bytes2string(wallet.get_address()))) - - # Make a block that gives us coins. - import time - generate_block(blockchain, wallet.get_address(), json_data=dict( - test=123, - time=int(time.time()), - )) - - # Send some coins to ourselves - print("Sending ourselves 10 coins...") - transaction = wallet.make_simple_transaction( - 10, - wallet.get_address(), - json_data=dict( - test=456, - time=int(time.time()), - ), - auth_data=dict(k='test123'), - ) - print(transaction) - blockchain.add_transaction(transaction.to_bytes()) - - # Make a block that confirms that transaction. - generate_block(blockchain, wallet.get_address(), json_data=dict( - test=789, - time=int(time.time()), - )) diff --git a/blockchain/pybc/science.py b/blockchain/pybc/science.py deleted file mode 100644 index 71d15d4e8..000000000 --- a/blockchain/pybc/science.py +++ /dev/null @@ -1,177 +0,0 @@ -""" -science.py: Module containing functions to log performance for later statistical -analysis. - -This is different than the normal logging module because we always want to save -science information, but it's not really a log level like ERROR or WARNING. We -also want the science information to be machine-parseable. - -Science information is logged as a TSV with 4 columns: - - - Hostname - - Time - - Event - - Value (optional) - -Additionally, we support auto-magic named timers with statr_timer and end_timer, -where the elapsed times are automatically logged. - -We can also log the sizes of files and, if on Unix, current process memory -usage. - -""" - -from __future__ import absolute_import -import sys -import socket -import time -import threading -import os -from io import open - -try: - import resource -except BaseException: # We couldn't import the resource module; we are probably not running on - # Unix. Memory logging will be a no-op. - - # Remember that the module is unavailable. - resource = None - -# This holds the science logging output stream that the module uses. -log_stream = None - -# This holds the time format to use -time_format = "%Y-%m-%d %H:%M:%S" - -# This holds a dict of active timer start times by name. -timers = {} - -# This holds our global module lock -lock = threading.RLock() - - -def log_to(log_filename): - """ - Set up science logging to a file with the given name. - - """ - - with lock: - - # We need to replace the module-level variable - global log_stream - - # Open up the new log stream - log_stream = open(log_filename, "a") - - -def log_event(event, value=None): - """ - Log the given event. - - """ - - with lock: - - # What time is it right now? - now = time.gmtime() - - # This holds all the parts we are going to assemble - parts = [socket.getfqdn(), time.strftime(time_format, now), str(event)] - - if value is not None: - # We got a value and should log that too. - parts.append(str(value)) - - if log_stream is not None: - # Log to the science output stream. - log_stream.write("\t".join(parts)) - log_stream.write("\n") - log_stream.flush() - - -def start_timer(name, replace=True): - """ - Start a named timer with the given name. If replace is false, don't change - the start time if it already exists. - - """ - - # Record the time we were called at - start_time = time.clock() - - with lock: - - if replace or name not in timers: - # Start the timer by saving the current clock value (which is a - # float in seconds, but is supposed to be good to the microsecond or - # thereabouts. - timers[name] = start_time - - -def stop_timer(name): - """ - Stop a named timer and log the elapsed time in seconds, under the timer - name. Does nothing if the timer is not started. - - """ - - # Record the time we were called at - stop_time = time.clock() - - with lock: - - if name in timers: - # How long elapsed? - elapsed_time = stop_time - timers[name] - - # Log the time - log_event(name, elapsed_time) - - # Cancel the timer - del timers[name] - - -def log_filesize(name, filename): - """ - Log an event with the given name, and a value equal to the current size of - the file with the given file name, in bytes. - - Nonexistent files have size 0. - - """ - - try: - # What is the file size? - size = os.path.getsize(filename) - except os.error: - # File probably doesn't exist. - size = 0 - - # Log the size event. - log_event(name, size) - - -def log_memory(): - """ - Logs the current memory usage of the process, if available. - - Memory usage is in whatever unit the resource module uses, probably - kilobytes, and is logged as "memory_usage". - - """ - - if resource is None: - # Can't log memory without the resource module. - return - - try: - # Get the memory usage. See . - # Apparently "memory usage" is best described as "max resident set - # size". - memory_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - - log_event("memory_usage", memory_usage) - except resource.error: - # Skip errors due to resource not working properly. - pass diff --git a/blockchain/pybc/sqliteshelf.py b/blockchain/pybc/sqliteshelf.py deleted file mode 100644 index 9474b5b92..000000000 --- a/blockchain/pybc/sqliteshelf.py +++ /dev/null @@ -1,371 +0,0 @@ -""" -By default, things are stored in a "shelf" table - ->>> d = SQLiteShelf("test.sdb") - -You can put multiple shelves into a single SQLite database. Some can be lazy. - ->>> e = SQLiteShelf("test.sdb", "othertable", lazy=True) - -Both are empty to start with. - ->>> d -{} ->>> e -{} - -Adding stuff is as simple as a regular dict. ->>> d['a'] = "moo" ->>> e['a'] = "moo" - -Regular dict actions work. - ->>> d['a'] -'moo' ->>> e['a'] -'moo' ->>> 'a' in d -True ->>> len(d) -1 ->>> del d['a'] ->>> 'a' in d -False ->>> len(d) -0 ->>> del e['a'] - -Lazy shelves should be synced to disk, but accesses to non-lazy shelves in the -same database can happen just fine (and will actually do the same thing -internally). - ->>> e['thing'] = "stuff" ->>> d['stuff'] = "thing" ->>> e['otherthing'] = "more stuff" ->>> e.sync() - -Objects can be stored in shelves. - ->> class Test: -.. def __init__(self): -.. self.foo = "bar" -.. ->> t = Test() ->> d['t'] = t ->> print d['t'].foo -bar - -Errors are as normal for a dict. - ->>> d['x'] -Traceback (most recent call last): - ... -KeyError: 'x' ->>> del d['x'] -Traceback (most recent call last): - ... -KeyError: 'x' - -When you're done, you probably want to close your shelves, which will sync lazy -shelves to disk. - ->>> d.close() ->>> e.close() - -""" - -from __future__ import absolute_import -try: - from UserDict import DictMixin -except ImportError: - from collections import MutableMapping as DictMixin - -from collections import MutableSet - -try: - import six.moves.cPickle as pickle -except ImportError: - import pickle - -import sqlite3 -import threading - - -class SQLiteDict(DictMixin): - - def __init__(self, filename=':memory:', table='shelf', flags='r', mode=None, lazy=False): - """ - Make a new SQLite-backed dict that holds strings. filename specifies the - file to use; by default a special filename of ":memory:" is used to keep - all data in memory. table specifies the name of the table to use; - multiple tables can be used in the same file at the same time; they will - share the same underlying database connection. flags and mode exist to - match the shelve module's interface and are ignored. By default, data is - saved to disk after every operation; passing lazy=True will require you - to call the sync() and close() methods for changes to be guaranteed to - make it to disk. - - Note that this object is not thread safe. It is willing to be used by - multiple threads, but access should be controlled by a lock. If multiple - SqliteDicts are using different tables in the same database, they all - should be controlled by the same lock. - - """ - - # Grab the lock appropriate for the file we're accessing - self.lock = SQLiteDict.get_lock(filename) - - with self.lock: - - self.filename = filename - self.table = table - self.lazy = lazy - MAKE_SHELF = 'CREATE TABLE IF NOT EXISTS ' + self.table + ' (key TEXT, value TEXT)' - MAKE_INDEX = 'CREATE UNIQUE INDEX IF NOT EXISTS ' + self.table + '_keyndx ON ' + self.table + '(key)' - self.conn = SQLiteDict.get_connection(filename) - self.conn.text_factory = str - self.conn.execute(MAKE_SHELF) - self.conn.execute(MAKE_INDEX) - self.maybe_sync() - - # This holds all the locks by filename - lock_cache = {} - - # This lock is used to serialize access to the dict of locks. - locks_lock = threading.RLock() - - @staticmethod - def get_lock(filename): - """ - Get the lock used to control access to the shared connection to the - given database file. - - """ - - with SQLiteDict.locks_lock: - - if filename not in SQLiteDict.lock_cache: - # We need to make the lock - SQLiteDict.lock_cache[filename] = threading.RLock() - - # We don't bother with reference counting on the locks. - # TODO: reference count the locks. - - # Use the cached lock. - return SQLiteDict.lock_cache[filename] - - # This is the database connection cache - connection_cache = {} - - # This is the reference counts for connections by filename. - connection_references = {} - - @staticmethod - def get_connection(filename): - """ - Since we can't have transactions from more than one connection happening - at the same time, but we may want to have more than one SqliteDict - accessing the same database (on different tables) when some of them are - lazy. Thus, we need to share database connections, stored by file name. - - This function gets the database connection to use for a given database - filename. - - """ - - if filename not in SQLiteDict.connection_cache: - # We need to make the connection - SQLiteDict.connection_cache[filename] = sqlite3.connect(filename, check_same_thread=False) - # It starts with no references - SQLiteDict.connection_references[filename] = 0 - - # Add a reference so we can close the connection when all the shelves close. - SQLiteDict.connection_references[filename] += 1 - - # Use the cached connection. - return SQLiteDict.connection_cache[filename] - - @staticmethod - def drop_connection(filename): - """ - Drop a reference to the given database. If the database now has no - references, close its connection. - - """ - - SQLiteDict.connection_references[filename] -= 1 - - if SQLiteDict.connection_references[filename] == 0: - # We can get rid of this connection. All the SQLiteDicts are done - # with it. - SQLiteDict.connection_cache[filename].commit() - SQLiteDict.connection_cache[filename].close() - del SQLiteDict.connection_cache[filename] - - def maybe_sync(self): - """ - Sync to disk if this SqliteDict is not lazy. - - """ - - with self.lock: - if not self.lazy: - # We're not lazy. Commit now! - self.conn.commit() - - def sync(self): - """ - Sync to disk. Finishes any internal sqlite transactions. - - """ - with self.lock: - self.conn.commit() - - def discard(self): - """ - Discard any changes made since the last sync. If the shelf is not lazy, - the last sync was whenever anything was stored into the shelf. - Otherwise, it was when sync was last explicitly called. - - """ - with self.lock: - self.conn.discard() - - def clear(self): - """ - Delete all entries in the SQLiteDict. - - """ - with self.lock: - # Make a command to delete everything - DELETE_ALL = 'DELETE FROM ' + self.table - - # Actually run it - self.conn.execute(DELETE_ALL) - - def get_size(self): - """ - Return the size of the database file used to store this SQLiteDict, in - bytes. - - Includes usage by the object itself and any other tables in the - database. - - """ - with self.lock: - try: - # Grab the page size in bytes - page_size = self.conn.execute('PRAGMA page_size').fetchone()[0] - - # Grab the number of pages (both used and free) - page_count = self.conn.execute('PRAGMA ' - 'page_count').fetchone()[0] - - return page_size * page_count - except BaseException: # This isn't allowed on some sqlites - return 0 - - def __getitem__(self, key): - with self.lock: - GET_ITEM = 'SELECT value FROM ' + self.table + ' WHERE key = ?' - item = self.conn.execute(GET_ITEM, (key,)).fetchone() - if item is None: - raise KeyError(key) - return item[0] - - def __setitem__(self, key, item): - with self.lock: - ADD_ITEM = 'REPLACE INTO ' + self.table + ' (key, value) VALUES (?,?)' - self.conn.execute(ADD_ITEM, (key, item)) - self.maybe_sync() - - def __delitem__(self, key): - with self.lock: - if key not in self: - raise KeyError(key) - DEL_ITEM = 'DELETE FROM ' + self.table + ' WHERE key = ?' - self.conn.execute(DEL_ITEM, (key,)) - self.maybe_sync() - - def __iter__(self): - with self.lock: - c = self.conn.cursor() - try: - c.execute('SELECT key FROM ' + self.table + ' ORDER BY key') - for row in c: - yield row[0] - finally: - c.close() - - def keys(self): - with self.lock: - c = self.conn.cursor() - try: - c.execute('SELECT key FROM ' + self.table + ' ORDER BY key') - return [row[0] for row in c] - finally: - c.close() - - ################################################################### - # optional bits - - def __len__(self): - with self.lock: - GET_LEN = 'SELECT COUNT(*) FROM ' + self.table - return self.conn.execute(GET_LEN).fetchone()[0] - - def close(self): - with self.lock: - if self.conn is not None: - SQLiteDict.drop_connection(self.filename) - self.conn = None - - def __del__(self): - with self.lock: - self.close() - - def __repr__(self): - with self.lock: - return repr(dict(self)) - - -class SQLiteShelf(SQLiteDict): - """ - A class that extends the SQLiteShelf to storing pickleable Python objects as - values, instead of only strings. - - """ - - def __getitem__(self, key): - return pickle.loads(SQLiteDict.__getitem__(self, key)) - - def __setitem__(self, key, item): - SQLiteDict.__setitem__(self, key, pickle.dumps(item)) - - -if __name__ == "__main__": - import doctest - doctest.testmod() - -""" -The MIT License (MIT) - -Copyright (c) 2013 Shish - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" diff --git a/blockchain/pybc/token.py b/blockchain/pybc/token.py deleted file mode 100644 index 599e517ea..000000000 --- a/blockchain/pybc/token.py +++ /dev/null @@ -1,356 +0,0 @@ -#!/usr/bin/env python2.7 -# token.py: a digital token implemented on top of pybc.coin -# - -#------------------------------------------------------------------------------ - -from __future__ import absolute_import -import logging - -#------------------------------------------------------------------------------ - -from . import json_coin -from . import transactions -from . import util - -#------------------------------------------------------------------------------ - -def pack_token(token, payloads): - """ - """ - return dict( - t=token, - p=payloads, - ) - -def unpack_token(json_data): - """ - """ - try: - return json_data['t'], json_data['p'] - except (TypeError, KeyError): - return None, None - - -class TokenRecord(object): - """ - """ - - def __init__(self, block_hash=None, transaction_hash=None): - """ - """ - self.block_hash = block_hash - self.transaction_hash = transaction_hash - self.token_id = None - self.amount = None - self.address = None - self.prev_transaction_hash = None - self.prev_output_index = None - self.prev_amount = None - self.prev_address = None - self.input_payloads = [] - self.output_payloads = [] - - def __str__(self): - return "'{}' with {} addressed from {} to {}".format( - self.token_id, - self.value(), - util.bytes2string(self.prev_address or '') or '', - util.bytes2string(self.address or '') or '', - ) - - def add_output(self, output_tuple): - """ - """ - self.amount, self.address, _ = output_tuple - output_token_id, self.output_payloads = unpack_token(output_tuple[2]) - if output_token_id is None: - raise ValueError('output json data does not contain token record') - if self.token_id is None: - self.token_id = output_token_id - if output_token_id != self.token_id: - raise ValueError('output token ID does not match to input') - - def add_input(self, input_tuple): - """ - """ - input_token_id, self.input_payloads = unpack_token(input_tuple[4]) - if input_token_id is None: - raise ValueError('input json data does not contain token record') - if self.token_id is None: - self.token_id = input_token_id - if input_token_id != self.token_id: - raise ValueError('input token ID does not match to output') - self.prev_transaction_hash, self.prev_output_index, self.prev_amount, self.prev_address, _ = input_tuple - - def value(self): - """ - """ - return self.amount or self.prev_amount - - -class TokenProfile(object): - """ - """ - def __init__(self, token_records=[]): - self.token_id = None - self.records = [] - for t_record in token_records: - self.add_record(t_record) - logging.info('\n{}'.format(str(self))) - - def __str__(self): - """ - """ - lines = [] - lines.append("---TokenProfile {}---".format(self.token_id)) - lines.append("{} records".format(len(self.records))) - for i, record in enumerate(self.records): - lines.append("\t{}: {}".format(i, str(record))) - lines.append('Owner: {}'.format(util.bytes2string(self.owner().address))) - lines.append('Creator: {}'.format(util.bytes2string(self.creator().address))) - return "\n".join(lines) - - def add_record(self, token_record): - if token_record in self.records: - raise ValueError('duplicated token record') - if self.token_id is None: - self.token_id = token_record.token_id - if self.token_id != token_record.token_id: - raise ValueError('invalid token ID, not matching with first record') - if not self.records: - self.records.append(token_record) - return - if token_record.prev_address is None: - # this is "create new token" record - self.records.insert(0, token_record) - return - if token_record.address is None: - # this is "delete existing token" record - self.records.append(token_record) - return - for i, existing_record in enumerate(self.records): - if existing_record.prev_address is None: - if existing_record.address == token_record.prev_address: - # put after the first record - self.records.insert(i + 1, token_record) - return - if existing_record.address is None: - if existing_record.prev_address == token_record.address: - # put before the last record - self.records.insert(i, token_record) - return - if existing_record.address == token_record.prev_address: - # put after matched record in the middle - self.records.insert(i, token_record) - return - if existing_record.prev_address == token_record.address: - # put before matched record in the middle - self.records.insert(i, token_record) - return - # BAD CASE: put it just before the last record - self.records.insert(-1, token_record) - - def creator(self): - """ - """ - return self.records[0] - - def owner(self): - """ - """ - return self.records[-1] - - -class TokenBlockchain(json_coin.JsonCoinBlockchain): - """ - """ - - def iterate_records(self, include_inputs=True): - with self.lock: - for block in self.longest_chain(): - for transaction_bytes in transactions.unpack_transactions(block.payload): - json_transaction = json_coin.JsonTransaction.from_bytes(transaction_bytes) - token_records = dict() - if include_inputs: - for tr_input in json_transaction.inputs: - token_id, _ = unpack_token(tr_input[4]) - if not token_id: - continue - if token_id in token_records: - raise ValueError('duplicated token ID in transaction inputs') - token_records[token_id] = [tr_input, None, ] - for tr_output in json_transaction.outputs: - token_id, _ = unpack_token(tr_output[2]) - if not token_id: - continue - if token_id not in token_records: - token_records[token_id] = [None, None, ] - token_records[token_id][1] = tr_output - for token_id, input_output in token_records.items(): - tr_input, tr_output = input_output - token_record = TokenRecord( - block_hash=block.block_hash(), - transaction_hash=json_transaction.transaction_hash(), - ) - if tr_input is not None: - try: - token_record.add_input(tr_input) - except ValueError: - logging.exception('Failed to add an input to the token record: {}'.format(tr_input)) - continue - if tr_output is not None: - try: - token_record.add_output(tr_output) - except ValueError: - logging.exception('Failed to add an output to the token record: {}'.format(tr_output)) - continue - yield token_record - - def iterate_records_by_address(self, address, include_inputs=True): - """ - """ - with self.lock: - for token_record in self.iterate_records(include_inputs=include_inputs): - if token_record.address == address: - yield token_record - - def iterate_records_by_token(self, token_id, include_inputs=True): - """ - """ - with self.lock: - for token_record in self.iterate_records(include_inputs=include_inputs): - if token_record.token_id == token_id: - yield token_record - - def get_records_by_token(self, token): - """ - """ - with self.lock: - return [t for t in self.iterate_records_by_token(token, include_inputs=True)] - - def is_records_for_address(self, address): - """ - """ - with self.lock: - for _ in self.iterate_records_by_address(address, include_inputs=False): - return True - return False - - def is_records_for_token(self, token): - """ - """ - with self.lock: - for _ in self.iterate_records_by_token(token, include_inputs=False): - return True - return False - - def get_token_profile(self, token): - """ - """ - with self.lock: - try: - return TokenProfile(self.get_records_by_token(token)) - except ValueError: - return None - - def get_token_profiles_by_owner(self, address): - """ - """ - with self.lock: - result = [] - related_token_ids = set() - token_records_by_id = dict() - for token_record in self.iterate_records(include_inputs=True): - if token_record.token_id not in token_records_by_id: - token_records_by_id[token_record.token_id] = [] - token_records_by_id[token_record.token_id].append(token_record) - if token_record.address == address: - related_token_ids.add(token_record.token_id) - for token_id in related_token_ids: - result.append(TokenProfile(token_records_by_id[token_id][:])) - logging.info('{} tokens was found'.format(len(result))) - return result - - -class TokenWallet(json_coin.JsonWallet): - """ - """ - - def tokens_list(self): - return self.blockchain.get_token_profiles_by_owner(self.get_address()) - - def token_create(self, token, value, address=None, fee=1, payload=None, auth_data=None): - """ - """ - with self.lock: - if self.blockchain.is_records_for_token(token): - raise Exception('found existing token, but all tokens must be unique') - return self.make_simple_transaction( - value, - address or self.get_address(), - fee=fee, - json_data=pack_token(token, [payload, ]), - auth_data=auth_data, - spendable_filter=self._skip_all_tokens, - ) - - def token_delete(self, token, address=None, fee=1, auth_data=None): - """ - """ - with self.lock: - token_profile = self.blockchain.get_token_profile(token) - if not token_profile: - raise Exception('this token is not exist') - if token_profile.owner().address != self.get_address(): - raise Exception('this token is not belong to you') - return self.make_simple_transaction( - token_profile.owner().amount, - address or self.get_address(), - fee=fee, - json_data=None, - auth_data=auth_data, - spendable_filter=lambda tr_input: self._skip_tokens_except_one(token, tr_input), - ) - - def token_transfer(self, token, new_address, new_value=None, fee=1, payload=None, payload_history=True, auth_data=None): - """ - """ - with self.lock: - token_profile = self.blockchain.get_token_profile(token) - if not token_profile: - raise Exception('this token is not exist') - if token_profile.owner().address != self.get_address(): - raise Exception('this token is not belong to you') - payloads = token_profile.owner().output_payloads - if payload: - if payload_history: - payloads += [payload, ] - else: - payloads = [payload, ] - new_value = new_value or token_profile.owner().amount - return self.make_simple_transaction( - new_value, - new_address, - fee=fee, - json_data=pack_token(token, payloads), - auth_data=auth_data, - spendable_filter=lambda tr_input: self._skip_tokens_except_one(token, tr_input), - ) - - def _skip_all_tokens(self, tr_input): - """ - Filter input tuple and return bool result. - If input does not contain a token, then those coins are spendable. - """ - token_id, _ = unpack_token(tr_input[4]) - return token_id is None - - def _skip_tokens_except_one(self, spendable_token, tr_input): - """ - Filter input tuple and return bool result. - If input does not contain a token or we want to "delete/sell" this token, - then those coins are spendable. - """ - token_id, _ = unpack_token(tr_input[4]) - return token_id is None or token_id == spendable_token diff --git a/blockchain/pybc/transactions.py b/blockchain/pybc/transactions.py deleted file mode 100644 index a12d29c04..000000000 --- a/blockchain/pybc/transactions.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -transactions.py: Code for dealing with transactions, which at the Blockchainq -level are not real objects. Transactions are represented as bytestrings. - -""" - -from __future__ import absolute_import -import traceback -import struct -import logging -from six.moves import range - - -class InvalidPayloadError(Exception): - """ - Thrown when a transaction list can't be decoded. - - """ - - -def unpack_transactions(block_payload): - """ - Given a block payload, parse it out into its constituent transactions. - Yield each of them. - - The encoding we use is - <4 byte transaction count> - [<4 byte transaction byte length> - ]* - - May throw an InvalidPayloadError if the payload is not a properly - encoded list. - - """ - try: - # How many transactions do we need? - (transaction_count,) = struct.unpack(">I", block_payload[0:4]) - # Where should we start the next record from? - location = 4 - - for _ in range(transaction_count): - # How many bytes is the transaction? - (transaction_length,) = struct.unpack(">I", - block_payload[location:location + 4]) - location += 4 - - # Grab the transaction bytes - transaction_bytes = block_payload[location:location + - transaction_length] - location += transaction_length - - yield transaction_bytes - except GeneratorExit: - # This is fine - pass - except BaseException: # We broke while decoding a transaction. - logging.error("Exception while unpacking transactions") - logging.error(traceback.format_exc()) - raise InvalidPayloadError - - -def pack_transactions(transactions): - """ - Encode the given list of transactions into a payload. - - The encoding we use is - <4 byte transaction count> - [<4 byte transaction byte length> - ]* - - """ - - # Make a list of transaction records with their lengths at the - # front. - transactions_with_lengths = [struct.pack(">I", len(transaction)) + - transaction for transaction in transactions] - - # Return the number of transaction records followed by the records. - return (struct.pack(">I", len(transactions_with_lengths)) + - "".join(transactions_with_lengths)) diff --git a/blockchain/pybc/util.py b/blockchain/pybc/util.py deleted file mode 100644 index 747f2bd24..000000000 --- a/blockchain/pybc/util.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -util.py: PyBC utility functions. - -Contains functions for converting among bytestrings (strings that are just -arrays of bytes), strings (strings that are actaul character data) and other -basic types. - -Also contains a useful function for setting up logging. - -""" - -from __future__ import absolute_import -import base64 -import binascii -import time -import logging - - -def bytes2string(bytestring, limit=None): - """ - Encode the given byte string as a text string. Returns the encoded string. - - """ - if not bytestring: - return bytestring - if limit: - return base64.b64encode(bytestring)[:limit] - return base64.b64encode(bytestring) - - -def string2bytes(string): - """ - Decode the given printable string into a byte string. Returns the decoded - bytes. - - """ - if not string: - return string - return base64.b64decode(string) - - -def bytes2hex(bytestring): - """ - Encode the given bytestring as hex. - - """ - if not bytestring: - return bytestring - return "".join("{:02x}".format(ord(char)) for char in bytestring) - - -def bytes2long(s): - """ - Decode a Python long integer (bigint) from a string of bytes. - - See http://bugs.python.org/issue923643 - - """ - return int(binascii.hexlify(s), 16) - - -def long2bytes(n): - """ - Encode a Python long integer (bigint) as a string of bytes. - - See http://bugs.python.org/issue923643 - - """ - - hexstring = "%x" % n - - if len(hexstring) % 2 != 0: - # Pad to even length, big-endian. - hexstring = "0" + hexstring - - return binascii.unhexlify(hexstring) - - -def time2string(seconds): - """ - Given a seconds since epock int, return a UTC time string. - - """ - - return time.strftime("%d %b %Y %H:%M:%S", time.gmtime(seconds)) - - -def set_loglevel(loglevel, logformat="%(levelname)s:%(message)s", logdatefmt="%H:%M:%S", logfilename=None): - """ - Given a string log level name, set the logging module's log level to that - level. Raise an exception if the log level is invalid. - - """ - - # Borrows heavily from (actually is a straight copy of) the recipe at - # - - # What's the log level number from the module (or None if not found)? - numeric_level = getattr(logging, loglevel.upper(), None) - - if not isinstance(numeric_level, int): - # Turns out not to be a valid log level - raise ValueError("Invalid log level: {}".format(loglevel)) - - # Set the log level to the correct numeric value - logging.basicConfig(format=logformat, level=numeric_level, datefmt=logdatefmt, filename=logfilename, filemode='w') diff --git a/blockchain/pybc/wallet.py b/blockchain/pybc/wallet.py deleted file mode 100644 index f7cc74f78..000000000 --- a/blockchain/pybc/wallet.py +++ /dev/null @@ -1,430 +0,0 @@ -from __future__ import absolute_import -import time - -from twisted.web import server, resource -from twisted.internet import reactor # @UnresolvedImport - -from . import json_coin -from . import transactions -from . import util -import traceback -import json - -#------------------------------------------------------------------------------ - -def css_styles(css_dict): - return """ -body {margin: 0 auto; padding: 0;} -.panel-big {display: block; padding: 5px; margin-bottom: 5px; border-radius: 10px; overflow: auto;} -.panel-medium {display: block; padding: 2px; margin-bottom: 2px; border-radius: 5px; overflow: auto;} -.panel-small {display: block; padding: 1px; margin-bottom: 1px; border-radius: 3px; overflow: auto;} -#content {margin: 0 auto; padding: 0; text-align: justify; line-height: 16px; - width: 90%%; font-size: 14px; text-decoration: none; - font-family: "Century Gothic", Futura, Arial, sans-serif;} -#form_block {background-color: %(form_block_bg)s; border: 1px solid %(form_block_border)s;} -#form_block form {margin: 0px;} -#form_transaction {background-color: %(form_transaction_bg)s; border: 1px solid %(form_transaction_border)s;} -#form_transaction form {margin: 0px;} -#form_token {background-color: %(form_token_bg)s; border: 1px solid %(form_token_border)s;} -#form_token_transfer {background-color: %(form_token_transfer_bg)s; border: 1px solid %(form_token_transfer_border)s;} -#form_token_delete {background-color: %(form_token_transfer_bg)s; border: 1px solid %(form_token_transfer_border)s;} -.transaction {background-color: %(transaction_bg)s; border: 1px solid %(transaction_border)s;} -.tr_header {} -.tr_body {} -.tr_inputs {} -.tr_outputs {} -.tr_authorizations {} -.tr_input {background-color: %(tr_input_bg)s; border: 1px solid %(tr_input_border)s;} -.tr_output {background-color: %(tr_output_bg)s; border: 1px solid %(tr_output_border)s;} -.tr_authorization {background-color: %(tr_authorization_bg)s; border: 1px solid %(tr_authorization_border)s;} -.token {background-color: %(token_bg)s; border: 1px solid %(token_border)s;} -.to_payload {float: left; overflow: auto; margin: 2px;} -.to_record {background-color: %(to_record_bg)s; border: 1px solid %(to_record_border)s;} -.field {margin: 0 auto; text-align: left; float: left; padding: 0px 10px; display:inline-block;} -.field-right {margin: 0 auto; text-align: left; float: right; padding: 0px 10px; display:inline-block;} -.field code {font-size: 12px;} -.field input {line-height: 20px; padding:0px 5px; margin: 0px; border: 1px solid #888; border-radius: 3px;} -.field input[type=submit] {line-height: 20px; padding:0px 5px; margin: 0px; border: 1px solid #888; border-radius: 3px;} -.f_json {background-color: %(f_json_bg)s; border: 1px solid %(f_json_border)s; border-radius: 3px;} -""" % css_dict - -def colors(): - return dict( - form_block_bg='#E6E6FA', - form_block_border='#000080', - form_transaction_bg='#E6E6FA', - form_transaction_border='#000080', - form_token_bg='#E6E6FA', - form_token_border='#000080', - form_token_transfer_bg='#E6E6FA', - form_token_transfer_border='#000080', - form_token_delete_bg='#E6E6FA', - form_token_delete_border='#000080', - transaction_bg='#f8f8f8', - transaction_border='#d0d0d0', - tr_input_bg='#e0e0ff', - tr_input_border='#c0c0df', - tr_output_bg='#e0ffe0', - tr_output_border='#c0dfc0', - tr_authorization_bg='#ffe0e0', - tr_authorization_border='#dfc0c0', - f_json_bg='#FAFAD2', - f_json_border='#dfc0c0', - token_bg='#fff099', - token_border='#d0d0d0', - to_record_bg='#e0ffe0', - to_record_border='#c0dfc0', - to_payload_bg='#e0e0ff', - to_payload_border='#c0dfc0', - ) - -#------------------------------------------------------------------------------ - -class MainPage(resource.Resource): - isLeaf = True - peer = None - - def __init__(self, peer, wallet, *args, **kwargs): - resource.Resource.__init__(self) - self.peer = peer - self.wallet = wallet - - def _solve_block(self, json_data=None): - new_block = self.peer.blockchain.make_block( - self.wallet.get_address(), - json_data=json_data, - with_inputs=False, - ) - if not new_block: - return None - if not new_block.do_some_work(self.peer.blockchain.algorithm, - iterations=10000000): - return None - self.peer.send_block(new_block) - return new_block - - def render_POST(self, request): - command = request.args.get('command', ['', ])[0] - #--- solve block --- - if command == 'solve block': - json_data = request.args.get('json', ['{}', ])[0] - try: - json_data = json.loads(json_data or '{}') - new_block = self._solve_block(json_data) - if new_block: - result = self.peer.blockchain.dump_block(new_block) - code = 'OK' - else: - result = 'Block was not solved' - code = 'OK' - except: - result = traceback.format_exc() - code = 'ERROR' - - #--- create output --- - elif command == 'create output': - destination = request.args.get('destination', ['', ])[0] - amount = request.args.get('amount', ['0', ])[0] - json_data = request.args.get('json', ['{}', ])[0] - fee = request.args.get('fee', ['1', ])[0] - try: - amount = int(amount or 0) - fee = int(fee or 1) - json_data = json.loads(json_data or '{}') - new_transaction = self.wallet.make_simple_transaction( - amount, - util.string2bytes(destination), - fee=fee, - json_data=json_data, - ) - if new_transaction: - self.peer.send_transaction(new_transaction.to_bytes()) - result = str(new_transaction) - code = 'OK' - else: - result = 'Invalid transaction' - code = 'FAILED' - except: - result = traceback.format_exc() - code = 'ERROR' - - #--- create token --- - elif command == 'create token': - token_id = request.args.get('token_id', ['', ])[0] - amount = request.args.get('amount', ['0', ])[0] - json_data = request.args.get('json', ['{}', ])[0] - fee = request.args.get('fee', ['1', ])[0] - try: - amount = int(amount or 0) - fee = int(fee or 1) - json_data = json.loads(json_data or '{}') - new_transaction = self.wallet.token_create( - token_id, - amount, - fee=fee, - payload=json_data, - ) - if new_transaction: - self.peer.send_transaction(new_transaction.to_bytes()) - result = str(new_transaction) - code = 'OK' - else: - result = 'Invalid transaction' - code = 'FAILED' - except: - result = traceback.format_exc() - code = 'ERROR' - - #--- transfer token --- - elif command == 'transfer token': - token_id = request.args.get('token_id', ['', ])[0] - amount = request.args.get('amount', ['0', ])[0] - destination = request.args.get('destination', ['', ])[0] - fee = request.args.get('fee', ['1', ])[0] - json_data = request.args.get('json', ['{}', ])[0] - json_history = bool(request.args.get('json_history', ['', ])[0]) - try: - amount = int(amount or 0) - fee = int(fee or 1) - json_data = json.loads(json_data or '{}') - new_transaction = self.wallet.token_transfer( - token_id, - util.string2bytes(destination), - new_value=amount, - fee=fee, - payload=json_data, - payload_history=json_history, - ) - if new_transaction: - self.peer.send_transaction(new_transaction.to_bytes()) - result = str(new_transaction) - code = 'OK' - else: - result = 'Invalid transaction' - code = 'FAILED' - except: - result = traceback.format_exc() - code = 'ERROR' - - #--- delete token --- - elif command == 'delete token': - token_id = request.args.get('token_id', ['', ])[0] - destination = request.args.get('destination', ['', ])[0] - fee = request.args.get('fee', ['1', ])[0] - try: - fee = int(fee or 1) - new_transaction = self.wallet.token_delete( - token_id, - address=destination, - fee=fee, - ) - if new_transaction: - self.peer.send_transaction(new_transaction.to_bytes()) - result = str(new_transaction) - code = 'OK' - else: - result = 'Invalid transaction' - code = 'FAILED' - except: - result = traceback.format_exc() - code = 'ERROR' - - else: - result = 'Wrong command' - code = 'ERROR' - - return ''' - -PyBC Wallet on %(hostname)s:%(hostinfo)s - - - -
-
%(result)s
-result: %(code)s -
- -
-
- -''' % dict( - hostname=self.peer.external_address, - hostinfo=self.peer.port, - css='', - result=result, - code=code, - ) - - def render_GET(self, request): - src = ''' - -PyBC Wallet on %(hostname)s:%(hostinfo)s - - -''' % dict( - hostname=self.peer.external_address, - hostinfo=self.peer.port, - css=css_styles(colors()), - ) - src += '
\n' - src += '

PyBC Wallet on %(hostname)s:%(hostinfo)s

\n' % dict( - hostname=self.peer.external_address, - hostinfo=self.peer.port, - ) - src += '

address: {}

\n'.format( - util.bytes2string(self.wallet.get_address())) - src += '

current balance: {}

\n'.format(self.wallet.get_balance()) - src += 'network name: {}
\n'.format(self.peer.network) - src += 'software version: {}
\n'.format(self.peer.version) - src += 'number of blocks: {}
\n'.format(len(self.peer.blockchain.blockstore)) - src += 'local disk usage: {} bytes

\n'.format(self.peer.blockchain.get_disk_usage()) - #--- form_block - src += '
\n' - src += '
\n' - src += '
json:
\n' - src += '
\n' - src += '
\n' - src += '
\n' - #--- form_transaction - src += '
\n' - src += '
\n' - src += '
destination:
\n' - src += '
amount:
\n' - src += '
fee:
\n' - src += '
json:
\n' - src += '
\n' - src += '
\n' - src += '
\n' - #--- form_token - src += '
\n' - src += '
\n' - src += '
token:
\n' - src += '
value:
\n' - src += '
fee:
\n' - src += '
json payload:
\n' - src += '
\n' - src += '
\n' - src += '
\n' - #--- form_token_transfer - src += '
\n' - src += '
\n' - src += '
token:
\n' - src += '
destination:
\n' - src += '
new value:
\n' - src += '
fee:
\n' - src += '
json:
\n' - src += '
history:
\n' - src += '
\n' - src += '
\n' - src += '
\n' - #--- form_token_delete - src += '
\n' - src += '
\n' - src += '
token:
\n' - src += '
destination:
\n' - src += '
fee:
\n' - src += '
\n' - src += '
\n' - src += '
\n' - src += '

tokens:

\n' - for token_profile in self.wallet.tokens_list(): - src += '
\n' - src += '
\n' - src += '{}\n'.format(token_profile.owner().token_id) - if token_profile.owner().address is None: - src += 'disbanded\n' - elif token_profile.owner().address != self.wallet.get_address(): - src += 'belongs to {}\n'.format( - util.bytes2string(token_profile.owner().address, limit=8)) - elif token_profile.creator().address != self.wallet.get_address(): - src += 'created by {}\n'.format( - util.bytes2string(token_profile.creator().address, limit=8)) - src += '
\n' # to_header - src += '
\n' - for token_record in token_profile.records: - src += '
\n' - src += '
{} coins
\n'.format(token_record.value()) - src += '
from {}
\n'.format( - util.bytes2string(token_record.prev_address, limit=8) or '$$$') - src += '
to {}
\n'.format( - util.bytes2string(token_record.address, limit=8) or '$$$') - for payload in token_record.output_payloads: - src += '
\n' - src += '
{}
\n'.format(payload) - src += '
\n' # to_payload - src += '
\n' # to_record - src += '
\n' # to_body - src += '

\n' # token - src += '

related transactions:

\n' - for tr in self.peer.blockchain.iterate_transactions_by_address(self.wallet.get_address()): - src += '
\n' - src += '
\n' - src += 'transaction from {}\n'.format(time.ctime(tr.timestamp)) - src += 'hash: {},\n'.format( - util.bytes2string(tr.transaction_hash(), limit=8)) - src += 'block hash: {},\n'.format( - util.bytes2string(tr.block_hash, limit=8)) - src += 'with {} inputs, {} outputs and {} authorizations'.format( - len(tr.inputs), len(tr.outputs), len(tr.authorizations)) - src += '
\n' # tr_header - src += '
\n' - src += '
\n' - if not tr.inputs: - src += '
\n' - src += '
no inputs
\n' - src += '
\n' # tr_input - else: - for inpt in tr.inputs: - src += '
\n' - src += '
{}
\n'.format(inpt[2]) - src += '
{}
\n'.format( - util.bytes2string(inpt[3], limit=8)) - src += '
#{}
\n'.format(inpt[1]) - src += '
{}
\n'.format( - util.bytes2string(inpt[0], limit=8)) - if inpt[4] is not None: - src += '
{}
\n'.format(inpt[4]) - src += '
\n' # tr_input - src += '
\n' # tr_inputs - src += '
\n' - if not tr.outputs: - src += '
\n' - src += '
no outputs
\n' - src += '
\n' # tr_output - else: - for outpt in tr.outputs: - src += '
\n' - src += '
{}
\n'.format(outpt[0]) - src += '
{}
\n'.format( - util.bytes2string(outpt[1], limit=8)) - if outpt[2] is not None: - src += '
{}
\n'.format(outpt[2]) - src += '
\n' # tr_input - src += '
\n' # tr_outputs - src += '
\n' - if not tr.authorizations: - src += '
\n' - src += '
no authorizations
\n' - src += '
\n' # tr_authorization - else: - for author in tr.authorizations: - src += '
\n' - src += '
{}
\n'.format( - util.bytes2string(author[0], limit=10)) - src += '
{}
\n'.format( - util.bytes2string(author[1], limit=10)) - if author[2] is not None: - src += '
{}
\n'.format(author[2]) - src += '
\n' # tr_authorization - src += '
\n' # tr_authorizations - src += '
\n' # tr_body - src += '
\n' # transaction - src += '
\n' # content - src += '\n' - src += '' - return src - - -def start(port, peer_instance, wallet_instance): - site = server.Site(MainPage(peer_instance, wallet_instance)) - return reactor.listenTCP(port, site) diff --git a/blockchain/pybc_service.py b/blockchain/pybc_service.py deleted file mode 100644 index 6f3f212d8..000000000 --- a/blockchain/pybc_service.py +++ /dev/null @@ -1,525 +0,0 @@ -#!/usr/bin/python -#pybc_service.py -# -# Copyright (C) 2008-2016 Veselin Penev, http://bitdust.io -# -# This file (pybc_service.py) is part of BitDust Software. -# -# BitDust is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# BitDust Software is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with BitDust Software. If not, see . -# -# Please contact us if you have any questions at bitdust.io@gmail.com - - -""" -.. module:: pybc_service - -I am running this locally from command line. -Started several instances on same local machine, configuration files are split and separated for every process/node. -Say you have local folders: _1, _2, _3 and _4 for each node respectively. -Then you can test it that way, run every command in a separate terminal window: - -First node: - - python pybc_service.py --blockstore=_1/blocks --keystore=_1/keys --peerstore=_1/peers --port=8001 --loglevel=DEBUG --seeds="127.0.0.1:8001,127.0.0.1:8002,127.0.0.1:8003,127.0.0.1:8004" - - -Second node, this will also run block explorer on port 9002: - - python pybc_service.py --block_explorer=9002 --blockstore=_2/blocks --keystore=_2/keys --peerstore=_2/peers --port=8002 --loglevel=DEBUG --seeds="127.0.0.1:8001,127.0.0.1:8002,127.0.0.1:8003,127.0.0.1:8004" - - -Third node, this will also run a wallet UI on port 9003: - - python pybc_service.py --wallet=9003 --blockstore=_3/blocks --keystore=_3/keys --peerstore=_3/peers --port=8003 --loglevel=DEBUG --seeds="127.0.0.1:8001,127.0.0.1:8002,127.0.0.1:8003,127.0.0.1:8004" - - -Fourth node, this will start a minining process, will wait for incoming transactions, will try to solve a block and write into blockchain: - - python pybc_service.py --mine --blockstore=_4/blocks --keystore=_4/keys --peerstore=_4/peers --port=8004 --loglevel=DEBUG --seeds="127.0.0.1:8001,127.0.0.1:8002,127.0.0.1:8003,127.0.0.1:8004" - - -Fifth node, this will also start a minining process, but will only solve one block containing usefull info: - - python pybc_service.py --generate --json='{"this":"is",1:["cool", "!"]}' --blockstore=_5/blocks --keystore=_5/keys --peerstore=_5/peers --port=8005 --loglevel=DEBUG --seeds="127.0.0.1:8001,127.0.0.1:8002,127.0.0.1:8003,127.0.0.1:8004" - - -""" - -#------------------------------------------------------------------------------ - -from __future__ import absolute_import - -#------------------------------------------------------------------------------ - -_Debug = True -_DebugLevel = 4 - -#------------------------------------------------------------------------------ - -import os -import sys -import json -import logging -import time - -from twisted.internet import reactor # @UnresolvedImport - -#------------------------------------------------------------------------------ - -# This is a simple hack to be able to execute this module directly from command line for testing purposes -if __name__ == '__main__': - import os.path as _p - sys.path.insert(0, _p.abspath(_p.join(_p.dirname(_p.abspath(sys.argv[0])), '..'))) - -#------------------------------------------------------------------------------ - -from logs import lg - -from main import events - -#------------------------------------------------------------------------------ - -from blockchain import pybc - -#------------------------------------------------------------------------------ - -_SeenAddresses = set() # Keep a global set of addreses we have seen in blocks. -_PeerNode = None -_PeerListener = None -_BlockExplorerListener = None -_WalletListener = None -_Wallet = None -_BlockInProgress = None - -#------------------------------------------------------------------------------ - -def init(host='127.0.0.1', - port=8008, - seed_nodes=[], - blockstore_filename='./blockstore', - keystore_filename='./keystore', - peerstore_filename='./peerstore', - minify=None, - loglevel='INFO', - logfilepath='/tmp/pybc.log', - stats_filename=None, - ): - global _PeerNode - global _Wallet - if _Debug: - lg.out(_DebugLevel, 'pybc_service.init') - # Set the log level - pybc.util.set_loglevel(loglevel, logformat="%(asctime)s [%(module)s] %(message)s", logfilename=logfilepath) - if stats_filename is not None: - # Start the science - pybc.science.log_to(stats_filename) - if not os.path.exists(os.path.dirname(blockstore_filename)): - os.makedirs(os.path.dirname(blockstore_filename)) - if not os.path.exists(os.path.dirname(keystore_filename)): - os.makedirs(os.path.dirname(keystore_filename)) - if not os.path.exists(os.path.dirname(peerstore_filename)): - os.makedirs(os.path.dirname(peerstore_filename)) - logging.info("Starting server on {}:{}".format(host, port)) - pybc.science.log_event("startup") - # Make a CoinBlockchain, using the specified blockchain file - logging.info("Loading blockchain from {}".format(blockstore_filename)) - blockchain = pybc.token.TokenBlockchain(blockstore_filename, minification_time=minify) - # Listen to it so we can see incoming new blocks going forward and read - # their addresses. - blockchain.subscribe(on_event) - # Make a Wallet that uses the blockchain and our keystore - logging.info("Loading wallet keystore from {}".format(keystore_filename)) - _Wallet = pybc.token.TokenWallet(blockchain, keystore_filename) - logging.info("Wallet address: {}".format(pybc.util.bytes2string(_Wallet.get_address()))) - logging.info("Current balance: {}".format(_Wallet.get_balance())) - # Now make a Peer. - logging.info("Loading peers from {}".format(peerstore_filename)) - _PeerNode = pybc.Peer( - "PyBC-Coin", 2, blockchain, - peer_file=peerstore_filename, - external_address=host, - port=port, - ) - - def _on_listener_started(l): - global _PeerListener - _PeerListener = l - - _PeerNode.listener.addCallback(_on_listener_started) - # Start connecting to seed nodes - for peer_host, peer_port in seed_nodes: - if peer_host == host and peer_port == port: - # Skip our own node - continue - # logging.info('Connecting via TCP to peer {}:{}'.format(peer_host, peer_port)) - _PeerNode.connect(peer_host, peer_port) - _PeerNode.peer_seen(peer_host, peer_port, None) - logging.info("Number of blocks: {}".format(len(_PeerNode.blockchain.blockstore))) - logging.info("INIT DONE") - - -def shutdown(): - global _PeerListener - global _PeerNode - if _Debug: - lg.out(_DebugLevel, 'pybc_service.shutdown') - stop_block_explorer() - stop_wallet() - if _PeerListener: - """ TODO: """ - _PeerListener.loseConnection() - if _Debug: - lg.out(_DebugLevel, ' peer listener stopped') - else: - if _Debug: - lg.out(_DebugLevel, ' peer listener not initialized') - _PeerListener = None - _PeerNode = None - -#------------------------------------------------------------------------------ - -def node(): - global _PeerNode - return _PeerNode - - -def wallet(): - global _Wallet - return _Wallet - -#------------------------------------------------------------------------------ - -def seed_nodes(): - known_nodes = [ - # ('208.78.96.185', 9100), # datahaven.net - # ('67.207.147.183', 9100), # identity.datahaven.net - # ('185.5.250.123', 9100), # p2p-id.ru - # ('86.110.117.159', 9100), # veselin-p2p.ru - # ('185.65.200.231', 9100), # bitdust.io - # ('45.32.246.95', 9100), # bitdust.ai - - ('datahaven.net', 9100), # datahaven.net - ('identity.datahaven.net', 9100), # identity.datahaven.net - ('p2p-id.ru', 9100), # p2p-id.ru - ('veselin-p2p.ru', 9100), # veselin-p2p.ru - ('bitdust.io', 9100), # bitdust.io - ('work.offshore.ai', 9100), # bitdust.ai - ] - if _Debug: - # add 5 local nodes for testing - known_nodes.extend([ - ('127.0.0.1', 9100), - ('127.0.0.1', 9100), - ('127.0.0.1', 9100), - ('127.0.0.1', 9100), - ('127.0.0.1', 9100), - ]) - return known_nodes - -#------------------------------------------------------------------------------ - -def on_event(event, argument): - """ - A function that listens to the blockchain and collects addresses from - incoming blocks. - """ - logging.info("EVENT: [{}] with {} bytes".format(event, len(str(argument)))) - # reactor.callFromThread(events.send, 'blockchain-{}'.format(event), ) # data=dict(argument=argument)) - events.send('blockchain-{}'.format(event), ) - -# global _SeenAddresses -# if event == "forward": -# # We're moving forward a block, and the argument is a block. -# if argument.has_body: -# # Only look at blocks that actually have transactions in them. -# for transaction_bytes in pybc.transactions.unpack_transactions(argument.payload): -# # Turn each transaction into a Transaction object. -# transaction = pybc.json_coin.JsonTransaction.from_bytes(transaction_bytes) -# for _, _, _, source, json_data in transaction.inputs: -# # Nothe that we saw a transaction from this source -# _SeenAddresses.add(source) -# logging.info('{} >>>>>>>> {}'.format( -# pybc.util.bytes2string(source), json_data)) -# for _, destination, json_data in transaction.outputs: -# # Note that we saw a transaction to this destination -# _SeenAddresses.add(destination) -# logging.info('{} <<<<<<<< {}'.format( -# pybc.util.bytes2string(destination), json_data)) - -#------------------------------------------------------------------------------ - -def start_block_explorer(port_number, peer_instance): - """ - """ - global _BlockExplorerListener - if _BlockExplorerListener: - logging.info('Block Explorer already started') - return False - logging.info('Starting Block Explorer on port %d, peer network versions is: %s/%s' % (port_number, peer_instance.network, peer_instance.version)) - # from . import pybc.block_explorer - _BlockExplorerListener = pybc.block_explorer.start(port_number, peer_instance) - return True - - -def stop_block_explorer(): - """ - """ - global _BlockExplorerListener - if not _BlockExplorerListener: - logging.info('Block Explorer not started') - return False - _BlockExplorerListener.loseConnection() - logging.info("Block Explorer stopped") - return True - -#------------------------------------------------------------------------------ - -def start_wallet(port_number, peer_instance, wallet_instance): - """ TODO: listener.loseConnection() """ - global _WalletListener - if _WalletListener: - logging.info('Wallet already started') - return False - logging.info('Starting Wallet on port %d, peer network versions is: %s/%s' % (port_number, peer_instance.network, peer_instance.version)) - # from . import pybc.wallet - _WalletListener = pybc.wallet.start(port_number, peer_instance, wallet_instance) - return True - - -def stop_wallet(): - global _WalletListener - if not _WalletListener: - logging.info('Wallet not started') - return False - _WalletListener.loseConnection() - logging.info("Wallet stopped") - return True - -#------------------------------------------------------------------------------ - -def generate_block(json_data=None, with_inputs=True, repeat=False, threaded=True): - """ - Keep on generating blocks in the background. - Put the blocks in the given peer's blockchain, and send the proceeds to the - given wallet. - Don't loop indefinitely, so that the Twisted main thread dying will stop - us. - """ - global _PeerNode - global _Wallet - global _BlockInProgress - success = False - - if not _PeerNode: - logging.info("Peer node is not exist, stop") - return None - - if _BlockInProgress is None: - if with_inputs and not _PeerNode.blockchain.transactions: - logging.info("Blockchain is empty, skip block generation and wait for incoming transactions, retry after 10 seconds...") - if repeat: - reactor.callLater(10, generate_block, json_data, with_inputs, repeat,) - return None - # We need to start a new block - _BlockInProgress = _PeerNode.blockchain.make_block( - _Wallet.get_address(), - json_data=json_data, # timestamped_json_data, - with_inputs=with_inputs, - ) - if _BlockInProgress is not None: - lg.info('started block generation with %d bytes json data, receiving address is %s, my ballance is %s' % ( - len(json.dumps(json_data)), pybc.util.bytes2string(_Wallet.get_address()), _Wallet.get_balance())) - logging.info("Starting a block!") - # Might as well dump balance here too - logging.info("Receiving address: {}".format(pybc.util.bytes2string(_Wallet.get_address()))) - logging.info("Current balance: {}".format(_Wallet.get_balance())) - else: - if repeat: - logging.info('Not able to start a new block, retry after 10 seconds...') - reactor.callLater(10, generate_block, json_data, with_inputs, repeat, ) - else: - logging.info('Failed to start a new block') - return None - - else: - # Keep working on the block we were working on - success = _BlockInProgress.do_some_work(_PeerNode.blockchain.algorithm) - if success: - # We found a block! - logging.info("Generated a block !!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - # Dump the block - logging.info("{}".format(_BlockInProgress)) - for transaction_bytes in pybc.unpack_transactions(_BlockInProgress.payload): - logging.info("{}".format(pybc.json_coin.JsonTransaction.from_bytes(transaction_bytes))) - _PeerNode.send_block(_BlockInProgress) - # Start again - _BlockInProgress = None - else: - if int(time.time()) > _BlockInProgress.timestamp + 60: - # This block is too old. Try a new one. - logging.info("Current generating block is {} seconds old. Restart mining!".format( - int(time.time()) - _BlockInProgress.timestamp)) - _BlockInProgress = None - elif (_PeerNode.blockchain.highest_block is not None and - _PeerNode.blockchain.highest_block.block_hash() != _BlockInProgress.previous_hash): - # This block is no longer based on the top of the chain - logging.info("New block from elsewhere! Restart generation!") - _BlockInProgress = None - - if threaded: - # Tell the main thread to make us another thread. - reactor.callFromThread(reactor.callInThread, generate_block, - json_data=json_data, with_inputs=with_inputs, repeat=repeat, threaded=threaded) - return None - - if not _BlockInProgress: - return success - - # now _BlockInProgress must exist and we will start mining directly in that thread - return generate_block(json_data=json_data, with_inputs=with_inputs, repeat=repeat, threaded=threaded) - -#------------------------------------------------------------------------------ - -def new_transaction(destination, amount, json_data, auth_data=None): - global _Wallet - global _PeerNode - if amount <= 0: - raise Exception('negative amount') - # How much fee should we pay? TODO: make this dynamic or configurable - fee = 1 - # How much do we need to send this transaction with its fee? - total_input = amount + fee - current_balance = _Wallet.get_balance() - if total_input > current_balance: - raise Exception("Insufficient funds! Current balance is {}, but {} needed".format( - current_balance, total_input)) - # If we get here this is an actually sane transaction. - # Make the transaction - transaction = _Wallet.make_simple_transaction( - amount, - pybc.util.string2bytes(destination), - fee=fee, - json_data=json_data, - auth_data=auth_data, - ) - # The user wants to make the transaction. Send it. - if not transaction: - logging.warning('Failed to create a new transaction: {} to {}'.format(amount, destination)) - else: - logging.info('Starting a new transaction:\n{}'.format(str(transaction))) - _PeerNode.send_transaction(transaction.to_bytes()) - return transaction - -#------------------------------------------------------------------------------ - -def _default_location(filename): - return os.path.join(os.path.expanduser('~'), '.pybc', filename) - - -def _parse_args(args): - import argparse - args = args[1:] - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--blockstore", default=_default_location('blocks'), - help="the name of a file to store blocks in") - parser.add_argument("--keystore", default=_default_location('keys'), - help="the name of a file to store keys in") - parser.add_argument("--peerstore", default=_default_location('peers'), - help="the name of a file to store peer addresses in") - parser.add_argument("--host", default='127.0.0.1', - help="the host or IP to advertise to other nodes") - parser.add_argument("--port", type=int, default=58585, - help="the port to listen on") - parser.add_argument("--seeds", default='127.0.0.1:58501,127.0.0.1:58502,127.0.0.1:58503,127.0.0.1:58504', - help="default set of seed nodes") - parser.add_argument("--minify", type=int, default=None, - help="minify blocks burried deeper than this") - parser.add_argument("--transaction", action="store_true", - help="create a new transaction") - parser.add_argument("--destination", - help="destination address for new transaction") - parser.add_argument("--amount", type=int, - help="amount of coins for new transaction") - parser.add_argument("--json", default='{}', - help="json data to store in the new transaction") - parser.add_argument("--generate", action="store_true", - help="generate a block with inputs every so often") - parser.add_argument("--mine", action="store_true", - help="generate any blocks, just to get some rewards and mine coins") - parser.add_argument("--wallet", type=int, default=None, - help="start wallet http listener, set port to listen on") - parser.add_argument("--block_explorer", type=int, default=None, - help="start block explorer http listener, set port to listen on") - parser.add_argument("--stats", - help="filename to log statistics to, for doing science") - parser.add_argument("--loglevel", default="INFO", - choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", ], - help="logging level to use") - parser.add_argument("--pdbshell", action="store_true", - help="import and run pdb set_trace() shell") - return parser.parse_args(args) - -#------------------------------------------------------------------------------ - -def main(): - global _PeerNode - global _Wallet - lg.set_debug_level(16) - lg.life_begins() - options = _parse_args(sys.argv) - init( - host=options.host, - port=options.port, - seed_nodes=[(i.split(':')[0], int(i.split(':')[1]), ) for i in options.seeds.split(',')], - blockstore_filename=options.blockstore, - keystore_filename=options.keystore, - peerstore_filename=options.peerstore, - loglevel='DEBUG' if _Debug else 'INFO', - ) - if options.block_explorer: - reactor.callLater(1, start_block_explorer, - options.block_explorer, _PeerNode) - if options.wallet: - reactor.callLater(1, start_wallet, - options.wallet, _PeerNode, _Wallet) - if options.generate: - reactor.callFromThread(generate_block, - json_data=dict(started=time.time(), data=json.loads(options.json)), - with_inputs=False, - repeat=False, ) - if options.mine: - reactor.callFromThread(generate_block, - json_data=dict(started=time.time(), data=json.loads(options.json)), - with_inputs=True, - repeat=True, ) - if options.transaction: - reactor.callLater(5, new_transaction, - options.destination, - options.amount, - json.loads(options.json), ) - if options.pdbshell: - import pdb - pdb.set_trace() - return - - reactor.addSystemEventTrigger('before', 'shutdown', shutdown) - reactor.callLater(5, shutdown) - reactor.run() - - -if __name__ == "__main__": - main() diff --git a/blockchain/trinity/deploy_linux.sh b/blockchain/trinity/deploy_linux.sh deleted file mode 100755 index c609ce7ef..000000000 --- a/blockchain/trinity/deploy_linux.sh +++ /dev/null @@ -1,114 +0,0 @@ -#/bin/sh - -ROOT_DIR="$HOME/.bitdust" -BLOCKCHAIN_DIR="${ROOT_DIR}/blockchain" -BLOCKCHAIN_SRC="${BLOCKCHAIN_DIR}/src" -BLOCKCHAIN_DATA="${BLOCKCHAIN_DIR}/data" -BLOCKCHAIN_VENV="${BLOCKCHAIN_DIR}/venv" -BLOCKCHAIN_PYTHON="${BLOCKCHAIN_VENV}/bin/python3.6" -BLOCKCHAIN_VIRTUALENV="${BLOCKCHAIN_DIR}/virtualenv" -BLOCKCHAIN_VIRTUALENV_BIN="${BLOCKCHAIN_VIRTUALENV}/bin/virtualenv" - - -if ! [ -x "$(command -v python3.6)" ]; then - echo '' - echo '##### DEPLOYMENT FAILED! Python 3.6 development version is required but not installed!' - echo '' - echo 'You can install it this way:' - echo ' sudo add-apt-repository ppa:deadsnakes/ppa' - echo ' sudo apt-get update' - echo ' sudo apt-get install python3.6-dev' - echo '' - exit 1 -else - echo '' - echo '##### Python 3.6 is already installed' -fi - - -if ! [ -x "$(command -v pip3)" ]; then - echo '' - echo '##### DEPLOYMENT FAILED! Pip3 is required but not installed!' - echo '' - echo 'You can install it this way:' - echo ' sudo apt-get install python3-pip' - echo '' - exit 1 -else - echo '' - echo '##### Pip3 is already installed' -fi - - -if [ ! -d $BLOCKCHAIN_DIR ]; then - mkdir -p "${BLOCKCHAIN_DIR}" - echo '' - echo "##### Created parent folder for Py-EVM Blockchain in ${BLOCKCHAIN_DIR}" -fi - - -if [ ! -d $BLOCKCHAIN_DATA ]; then - mkdir -p "${BLOCKCHAIN_DATA}" - mkdir -p "${BLOCKCHAIN_DATA}/logs" - echo '' > $BLOCKCHAIN_DATA/logs/trinity.log - echo '' - echo "##### Created data folder for Py-EVM Blockchain in ${BLOCKCHAIN_DATA}" -fi - - -if [ ! -f $BLOCKCHAIN_VIRTUALENV/bin/virtualenv ]; then - echo '' - echo '##### Installing isolated virtualenv' - mkdir -p "${BLOCKCHAIN_VIRTUALENV}" - PYTHONUSERBASE=$BLOCKCHAIN_VIRTUALENV pip3 install --ignore-installed --user virtualenv -else - echo '' - echo '##### Found isolated virtualenv binaries' -fi - - -if [ ! -f $BLOCKCHAIN_SRC/setup.py ]; then - echo '' - echo '##### Cloning Py-EVM repository' - git clone --depth=1 https://github.com/vesellov/py-evm.git $BLOCKCHAIN_SRC -else - echo '' - echo '##### Updating the source code of Py-EVM' - cd $BLOCKCHAIN_SRC - git fetch - git reset --hard origin/master - cd $OLDPWD -fi - - -if [ ! -d $BLOCKCHAIN_VENV ]; then - echo '' - echo '##### Building Py-EVM virtual environment' - PYTHONUSERBASE=$BLOCKCHAIN_VIRTUALENV $BLOCKCHAIN_VIRTUALENV_BIN -p python3.6 $BLOCKCHAIN_VENV -fi - - -if [ ! -f $BLOCKCHAIN_VENV/bin/pip ]; then - echo '' - echo '##### Pip is not found inside virtual environment, rebuilding' - rm -rf $BLOCKCHAIN_VENV - PYTHONUSERBASE=$BLOCKCHAIN_VIRTUALENV $BLOCKCHAIN_VIRTUALENV_BIN -p python3.6 $BLOCKCHAIN_VENV -fi - - -echo '' -echo '##### Installing/Updating trinity with pip' -cd $BLOCKCHAIN_SRC -$BLOCKCHAIN_PYTHON setup.py install - - -echo '' -echo 'To start trinity process you can run via command line:' -echo '' -echo "${BLOCKCHAIN_VENV}/bin/trinity --data-dir=${BLOCKCHAIN_DATA} --port=30345" - - -echo '' -echo '' -echo 'DONE!' -echo '' From 632b9742d32922e2544d88f3beced94822bffcd8 Mon Sep 17 00:00:00 2001 From: Veselin Penev Date: Fri, 8 Mar 2019 16:09:08 +0100 Subject: [PATCH 4/8] copied and adopted parallelp 1.5.7 sources in the code base, thanks to Vitalii Vanovschi --- bpworker.py | 59 ++ parallelp/README | 34 ++ parallelp/__init__.py | 29 + parallelp/pp/AUTHORS | 1 + parallelp/pp/CHANGELOG | 28 + parallelp/pp/COPYING | 25 + parallelp/pp/PKG-INFO | 23 + parallelp/pp/README | 7 + parallelp/pp/__init__.py | 1054 +++++++++++++++++++++++++++++++++++ parallelp/pp/ppauto.py | 121 ++++ parallelp/pp/ppserver.py | 356 ++++++++++++ parallelp/pp/pptransport.py | 220 ++++++++ parallelp/pp/ppworker.py | 138 +++++ 13 files changed, 2095 insertions(+) create mode 100644 bpworker.py create mode 100644 parallelp/README create mode 100644 parallelp/__init__.py create mode 100644 parallelp/pp/AUTHORS create mode 100644 parallelp/pp/CHANGELOG create mode 100644 parallelp/pp/COPYING create mode 100644 parallelp/pp/PKG-INFO create mode 100644 parallelp/pp/README create mode 100644 parallelp/pp/__init__.py create mode 100644 parallelp/pp/ppauto.py create mode 100644 parallelp/pp/ppserver.py create mode 100644 parallelp/pp/pptransport.py create mode 100644 parallelp/pp/ppworker.py diff --git a/bpworker.py b/bpworker.py new file mode 100644 index 000000000..d3396b4f0 --- /dev/null +++ b/bpworker.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# bpworker.py +# +# Copyright (C) 2008-2018 Veselin Penev, https://bitdust.io +# +# This file (bpworker.py) is part of BitDust Software. +# +# BitDust is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# BitDust Software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with BitDust Software. If not, see . +# +# Please contact us if you have any questions at bitdust.io@gmail.com +# +# +# +# + +from __future__ import absolute_import + +import sys +import os + +import six +import platform +ostype = platform.uname()[0] +if ostype == 'Windows': + if six.PY3: + sys.stdout = sys.stdout.buffer + else: + try: + import msvcrt + msvcrt.setmode(1, os.O_BINARY) # @UndefinedVariable + msvcrt.setmode(2, os.O_BINARY) # @UndefinedVariable + except: + pass +else: + if six.PY3: + sys.stdout = sys.stdout.buffer + + +if __name__ == "__main__": + try: + sys.path.append(os.path.abspath(os.path.join('.', 'parallelp', 'pp'))) + from parallelp.pp.ppworker import _WorkerProcess + wp = _WorkerProcess() + wp.run() + except Exception as exc: + pass + # import traceback + # open('/tmp/raid.log', 'a').write(traceback.format_exc()) diff --git a/parallelp/README b/parallelp/README new file mode 100644 index 000000000..89a911c29 --- /dev/null +++ b/parallelp/README @@ -0,0 +1,34 @@ +Parallel Python and Py2EXE +-------------------------- + +This project is demostration how to bundle a program which is using Parallel Python (parallelpython.com) into the .exe via py2exe. + +* Requirement (what I have used): + +Windows XP, Python 2.5 from python.org, py2exe module + +* Introduction: + +Parallel Python starts new processes with subprocess.popen() function (on a local machine). The input and output is encoded with "pickle" and transfered via pipe between the processes. Traditionaly the new workers are started as: "python -u ppworker.py" with complete path. +Location of python interpreter is detected as sys.executable, location of ppworker.py is derived from __file__. + +Pickle must be able to read the source code of the function, so .pyc is not enough. A simple proxy function with available source code is enough. +Details: http://www.parallelpython.com/component/option,com_smf/Itemid,29/topic,206.0 + +Py2exe exectuble is a stripped version of python interpreter, renamed according your script name (so script.exe instead of python.exe). Your script is embeded inside of the .exe file resources and executed during the start of the exe. All dependent python modules are compiled into .pyc/.pyo and zipped into library.zip (which can have different name or can be bundled in the .exe resources as well). Details: http://www.py2exe.org/index.cgi/FAQ + +* Usage: + +python setup.py + +cd dist +sum_primes.exe + +* Notes: + +In the setup.py is an extra "zip" step to include the source code of function necessary for pickle functionality used in parallel python. + +We must distribute the python.exe binary as well, because py2exe does not correctly implement the "unbuffered" option. +Once the "python -u" equivalent is available via py2exe, we have more options: + - Distribute special ppworker.exe (compiled from ppworker.py) + - Implement something like http://docs.python.org/dev/library/multiprocessing.html#multiprocessing.freeze%5Fsupport for pp. diff --git a/parallelp/__init__.py b/parallelp/__init__.py new file mode 100644 index 000000000..3b7ca0e07 --- /dev/null +++ b/parallelp/__init__.py @@ -0,0 +1,29 @@ +# +# Copyright (C) 2008-2018 Veselin Penev, https://bitdust.io +# +# This file (__init__.py) is part of BitDust Software. +# +# BitDust is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# BitDust Software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with BitDust Software. If not, see . +# +# Please contact us if you have any questions at bitdust.io@gmail.com +# +# +# +# + +""" +.. + +module:: parallelp +""" diff --git a/parallelp/pp/AUTHORS b/parallelp/pp/AUTHORS new file mode 100644 index 000000000..2185d9ab3 --- /dev/null +++ b/parallelp/pp/AUTHORS @@ -0,0 +1 @@ +Vitalii Vanovschi - support@parallelpython.com diff --git a/parallelp/pp/CHANGELOG b/parallelp/pp/CHANGELOG new file mode 100644 index 000000000..df6a3d0b0 --- /dev/null +++ b/parallelp/pp/CHANGELOG @@ -0,0 +1,28 @@ +pp-1.5.7: + 1) Added ppworker restart after task completion functionality + 2) Added pickle protocol option + 3) Merges patch for Python 2.6 compatibility (contributed by mrtss) + 4) Merged patch for config file support (contributed by stevemilner) + 5) Documentation has been moved to doc folder. + +pp-1.5.6 + 1) Fixed problem with autodiscovery service on Winsows XP and Vista + 2) Merged new code quality improvement patches (contributed by stevemilner) + +pp-1.5.5 + 1) Fixed bug which caused segmentation fault when calling destroy() method. + 2) Merged performance and quality improvement patches (contributed by stevemilner) + +pp-1.5.4 + 1) Fixed bug with unindented comments + 2) easy_intall functionality repaired + 3) Code quality improved (many small changes) + +pp-1.5.3 + 1) Added support for methods of new-style classes. + 2) Added ability to read secret key from pp_secret variable of .pythonrc.py + 3) ppdoc.html and ppserver.1 are included in the distribution + 4) examples bundled with the distribution +CHANGELOG started + +* - nicknames of the contributors refer to the PP forum profile login names. diff --git a/parallelp/pp/COPYING b/parallelp/pp/COPYING new file mode 100644 index 000000000..0e3fe23b8 --- /dev/null +++ b/parallelp/pp/COPYING @@ -0,0 +1,25 @@ +Parallel Python Software: http://www.parallelpython.com +Copyright (c) 2005-2008, Vitalii Vanovschi +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/parallelp/pp/PKG-INFO b/parallelp/pp/PKG-INFO new file mode 100644 index 000000000..8c90ab13d --- /dev/null +++ b/parallelp/pp/PKG-INFO @@ -0,0 +1,23 @@ +Metadata-Version: 1.0 +Name: pp +Version: 1.5.7 +Summary: Parallel and distributed programming for Python +Home-page: http://www.parallelpython.com +Author: Vitalii Vanovschi +Author-email: support@parallelpython.com +License: BSD-like +Download-URL: http://www.parallelpython.com/downloads/pp/pp-1.5.7.zip +Description: + Parallel Python module (PP) provides an easy and efficient way to create parallel-enabled applications for SMP computers and clusters. PP module features cross-platform portability and dynamic load balancing. Thus application written with PP will parallelize efficiently even on heterogeneous and multi-platform clusters (including clusters running other application with variable CPU loads). Visit http://www.parallelpython.com for further information. + +Platform: Windows +Platform: Linux +Platform: Unix +Classifier: Topic :: Software Development +Classifier: Topic :: System :: Distributed Computing +Classifier: Programming Language :: Python +Classifier: Operating System :: OS Independent +Classifier: License :: OSI Approved :: BSD License +Classifier: Natural Language :: English +Classifier: Intended Audience :: Developers +Classifier: Development Status :: 5 - Production/Stable diff --git a/parallelp/pp/README b/parallelp/pp/README new file mode 100644 index 000000000..6bc595851 --- /dev/null +++ b/parallelp/pp/README @@ -0,0 +1,7 @@ +Visit http://www.parallelpython.com for up-to-date documentation, examples and support forums + +INSTALATION: + python setup.py install + +LOCAL DOCUMENTATION: + pydoc.html diff --git a/parallelp/pp/__init__.py b/parallelp/pp/__init__.py new file mode 100644 index 000000000..bb35d85cf --- /dev/null +++ b/parallelp/pp/__init__.py @@ -0,0 +1,1054 @@ +#!/usr/bin/env python +#__init__.py +# +# Parallel Python Software: http://www.parallelpython.com +# Copyright (c) 2005-2009, Vitalii Vanovschi +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. + +""" +Parallel Python Software, Execution Server. + +http://www.parallelpython.com - updates, documentation, examples and support +forums +""" + +from __future__ import absolute_import +from __future__ import print_function +# from six import text_type as str +from six.moves import range +# import six.moves.cPickle as pickle +import six.moves._thread +from io import TextIOWrapper + +import os +import json +import logging +import inspect +import sys +import types +import time +import atexit +import traceback + +# import user +from . import pptransport +from . import ppauto + +copyright = "Copyright (c) 2005-2009 Vitalii Vanovschi. All rights reserved" +version = "1.5.7" + +# reconnect persistent rworkers in 5 sec +_RECONNECT_WAIT_TIME = 5 + +# we need to have set even in Python 2.3 +try: + set +except NameError: + from sets import Set as set # @UnresolvedImport + +_USE_SUBPROCESS = False +try: + import subprocess + _USE_SUBPROCESS = True +except ImportError: + import popen2 + + +class _Task(object): + """ + Class describing single task (job) + """ + + def __init__(self, server, tid, callback=None, + callbackargs=(), group='default'): + """ + Initializes the task. + """ + self.lock = six.moves._thread.allocate_lock() + self.lock.acquire() + self.tid = tid + self.server = server + self.callback = callback + self.callbackargs = callbackargs + self.group = group + self.finished = False + self.unpickled = False + self.worker_pid = -1 + self.cancelled = False + + def finalize(self, sresult): + """ + Finalizes the task. + + For internal use only + """ + if self.cancelled: + self.sresult = None + else: + self.sresult = sresult + if self.callback: + self.__unpickle() + self.lock.release() + self.finished = True + + def __call__(self, raw_result=False): + """ + Retrieves result of the task. + """ + self.wait() + + if not self.unpickled and not raw_result: + self.__unpickle() + + if raw_result: + return self.sresult + else: + return self.result + + def wait(self): + """ + Waits for the task. + """ + if not self.finished: + self.lock.acquire() + self.lock.release() + + def __unpickle(self): + """ + Unpickles the result of the task. + """ + if self.sresult is not None: + raw = self.sresult + # if isinstance(raw, six.text_type): + # raw = raw.encode('latin1') + try: + self.result, sout = json.loads(raw)['v'] + except: + traceback.print_exc() + print(raw) + # self.result, sout = pickle.loads(raw) + if len(sout) > 0: + print(sout, end=' ') + else: + self.result = None + self.unpickled = True + if self.callback: + args = self.callbackargs + (self.result, ) + self.callback(*args) + + +class _Worker(object): + """ + Local worker class. + """ + command = [sys.executable, '-u', 'bpworker.py'] + + def __init__(self, restart_on_free, pickle_proto): + """ + Initializes local worker. + """ + self.restart_on_free = restart_on_free + self.pickle_proto = pickle_proto + # print '_Worker', self.command, os.path.abspath('.') + self.start() + + def start(self): + """ + Starts local worker. + """ + if _USE_SUBPROCESS: + if sys.platform.startswith("win"): + import win32process # @UnresolvedImport + my_env = os.environ + my_env['PYTHONIOENCODING'] = 'latin1' + proc = subprocess.Popen( + self.command, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=False, + creationflags=win32process.CREATE_NO_WINDOW, + env=my_env, + # encoding='latin1', + ) + self.t = pptransport.PipeTransport(proc.stdout, proc.stdin) + else: + if six.PY2: + my_env = os.environ + my_env['PYTHONIOENCODING'] = 'latin1' + proc = subprocess.Popen( + self.command, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=False, + env=my_env, + ) + self.t = pptransport.PipeTransport(proc.stdout, proc.stdin) + else: + my_env = os.environ + my_env['PYTHONIOENCODING'] = 'latin1' + proc = subprocess.Popen( + self.command, + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=False, + env=my_env, + # encoding='latin1', + ) + self.t = pptransport.PipeTransport(proc.stdout, proc.stdin) + else: + self.t = pptransport.PipeTransport(*popen2.popen3(self.command)[:2]) + self.pid = int(self.t.receive()) + # self.t.send(str(self.pickle_proto)) + # self.t.send(str(self.pickle_proto).encode('latin1')) + self.is_free = True + logging.info('Started new worker: %s with %s', self.pid, self.t) + + def stop(self): + """ + Stops local worker. + """ + logging.info('Stop worker: %s with %s', self.pid, self.t) + self.is_free = False + self.t.close() + + def restart(self): + """ + Restarts local worker. + """ + self.stop() + self.start() + + def free(self): + """ + Frees local worker. + """ + if self.restart_on_free: + self.restart() + else: + self.is_free = True + + +class _RWorker(pptransport.CSocketTransport): + """ + Remote worker class. + """ + + def __init__(self, host, port, secret, message=None, persistent=True): + """ + Initializes remote worker. + """ + self.persistent = persistent + self.host = host + self.port = port + self.secret = secret + self.address = (host, port) + self.id = host + ":" + six.text_type(port) + # self.id = host + ":" + str(port) + logging.info("Creating Rworker id=%s persistent=%s" + % (self.id, persistent)) + self.connect(message) + self.is_free = True + + def __del__(self): + """ + Closes connection with remote server. + """ + self.close() + + def connect(self, message=None): + """ + Connects to a remote server. + """ + while True: + try: + pptransport.SocketTransport.__init__(self) + self._connect(self.host, self.port) + if not self.authenticate(self.secret): + logging.error("Authentication failed for host=%s, port=%s" + % (self.host, self.port)) + return False + if message: + self.send(message) + self.is_free = True + return True + except: + if not self.persistent: + logging.info("Deleting from queue Rworker %s" + % (self.id, )) + return False +# print sys.excepthook(*sys.exc_info()) + logging.info("Failed to reconnect with " + "(host=%s, port=%i), will try again in %i s" + % (self.host, self.port, _RECONNECT_WAIT_TIME)) + time.sleep(_RECONNECT_WAIT_TIME) + + +class _Statistics(object): + """ + Class to hold execution statisitcs for a single node. + """ + + def __init__(self, ncpus, rworker=None): + """ + Initializes statistics for a node. + """ + self.ncpus = ncpus + self.time = 0.0 + self.njobs = 0 + self.rworker = rworker + + +class Template(object): + """ + Template class. + """ + + def __init__(self, job_server, func, depfuncs=(), modules=(), + callback=None, callbackargs=(), group='default', globals=None): + """ + Creates Template instance. + + jobs_server - pp server for submitting jobs + func - function to be executed + depfuncs - tuple with functions which might be called from 'func' + modules - tuple with module names to import + callback - callback function which will be called with argument + list equal to callbackargs+(result,) + as soon as calculation is done + callbackargs - additional arguments for callback function + group - job group, is used when wait(group) is called to wait for + jobs in a given group to finish + globals - dictionary from which all modules, functions and classes + will be imported, for instance: globals=globals() + """ + self.job_server = job_server + self.func = func + self.depfuncs = depfuncs + self.modules = modules + self.callback = callback + self.callbackargs = callbackargs + self.group = group + self.globals = globals + + def submit(self, *args): + """ + Submits function with *arg arguments to the execution queue. + """ + return self.job_server.submit(self.func, args, self.depfuncs, + self.modules, self.callback, self.callbackargs, + self.group, self.globals) + + +class Server(object): + """ + Parallel Python SMP execution server class. + """ + + default_port = 60000 + default_secret = "epo20pdosl;dksldkmm" + + def __init__(self, ncpus="autodetect", ppservers=(), secret=None, + loglevel=logging.WARNING, logstream=sys.stderr, logfile=None, + restart=False, proto=0): + """ + Creates Server instance. + + ncpus - the number of worker processes to start on the local + computer, if parameter is omitted it will be set to + the number of processors in the system + ppservers - list of active parallel python execution servers + to connect with + secret - passphrase for network connections, if omitted a default + passphrase will be used. It's highly recommended to use a + custom passphrase for all network connections. + loglevel - logging level + logstream - log stream destination + logfile - if not None, will log to file instead of logstream + restart - wheather to restart worker process after each task completion + proto - protocol number for pickle module + + With ncpus = 1 all tasks are executed consequently + For the best performance either use the default "autodetect" value + or set ncpus to the total number of processors in the system + """ + + if not isinstance(ppservers, tuple): + raise TypeError("ppservers argument must be a tuple") + + self.__initlog(loglevel, logstream=logstream, logfile=logfile) + logging.info("Creating server instance (pp-" + version + ")") + self.__tid = 0 + self.__active_tasks = 0 + self.__active_tasks_lock = six.moves._thread.allocate_lock() + self.__queue = [] + self.__queue_lock = six.moves._thread.allocate_lock() + self.__workers = [] + self.__rworkers = [] + self.__rworkers_reserved = [] + self.__rworkers_reserved4 = [] + self.__sourcesHM = {} + self.__sfuncHM = {} + self.__waittasks = [] + self.__waittasks_lock = six.moves._thread.allocate_lock() + self.__exiting = False + self.__accurate_stats = True + self.autopp_list = {} + self.__active_rworkers_list_lock = six.moves._thread.allocate_lock() + self.__restart_on_free = restart + self.__pickle_proto = proto + + # add local directory and sys.path to PYTHONPATH + pythondirs = [os.getcwd()] + sys.path + + if "PYTHONPATH" in os.environ and os.environ["PYTHONPATH"]: + pythondirs += os.environ["PYTHONPATH"].split(os.pathsep) + os.environ["PYTHONPATH"] = os.pathsep.join(set(pythondirs)) + + atexit.register(self.destroy) + self.__stats = {"local": _Statistics(0)} + self.set_ncpus(ncpus) + + self.ppservers = [] + self.auto_ppservers = [] + + for ppserver in ppservers: + ppserver = ppserver.split(":") + host = ppserver[0] + if len(ppserver) > 1: + port = int(ppserver[1]) + else: + port = Server.default_port + if host.find("*") == -1: + self.ppservers.append((host, port)) + else: + if host == "*": + host = "*.*.*.*" + interface = host.replace("*", "0") + broadcast = host.replace("*", "255") + self.auto_ppservers.append(((interface, port), + (broadcast, port))) + self.__stats_lock = six.moves._thread.allocate_lock() + if secret is not None: + # if not isinstance(secret, bytes): + # raise TypeError("secret must be of a string type") + # self.secret = str(secret) + self.secret = six.text_type(secret) +# elif hasattr(user, "pp_secret"): +# secret = user["pp_secret"] +# if not isinstance(secret, bytes): +# raise TypeError("secret must be of a string type") +# self.secret = str(secret) + else: + self.secret = Server.default_secret + self.__connect() + self.__creation_time = time.time() + logging.info("pp local server started with %d workers" + % (self.__ncpus, )) + + def submit(self, func, args=(), depfuncs=(), modules=(), + callback=None, callbackargs=(), group='default', globals=None): + """ + Submits function to the execution queue. + + func - function to be executed + args - tuple with arguments of the 'func' + depfuncs - tuple with functions which might be called from 'func' + modules - tuple with module names to import + callback - callback function which will be called with argument + list equal to callbackargs+(result,) + as soon as calculation is done + callbackargs - additional arguments for callback function + group - job group, is used when wait(group) is called to wait for + jobs in a given group to finish + globals - dictionary from which all modules, functions and classes + will be imported, for instance: globals=globals() + """ + # perform some checks for frequent mistakes + if self.__exiting: + raise RuntimeError("Cannot submit jobs: server" + " instance has been destroyed") + + if not isinstance(args, tuple): + raise TypeError("args argument must be a tuple") + + if not isinstance(depfuncs, tuple): + raise TypeError("depfuncs argument must be a tuple") + + if not isinstance(modules, tuple): + raise TypeError("modules argument must be a tuple") + + if not isinstance(callbackargs, tuple): + raise TypeError("callbackargs argument must be a tuple") + + for module in modules: + if not isinstance(module, six.string_types): + raise TypeError("modules argument must be a list of strings") + + tid = self.__gentid() + + if globals: + modules += tuple(self.__find_modules("", globals)) + modules = tuple(set(modules)) + self.__logger.info("Task %i will autoimport next modules: %s" % + (tid, six.text_type(modules))) + for object1 in globals.values(): + if isinstance(object1, types.FunctionType) \ + or isinstance(object1, type): + depfuncs += (object1, ) + + task = _Task(self, tid, callback, callbackargs, group) + + self.__waittasks_lock.acquire() + self.__waittasks.append(task) + self.__waittasks_lock.release() + + # if the function is a method of a class add self to the arguments list + if isinstance(func, types.MethodType) and func.__self__ is not None: + args = (func.__self__, ) + args + + # if there is an instance of a user defined class in the arguments add + # whole class to dependancies + # for arg in args: + # # Checks for both classic or new class instances + # if isinstance(arg, object) or str(type(arg))[:6] == " 0") + if ncpus > len(self.__workers): + self.__workers.extend([_Worker(self.__restart_on_free, self.__pickle_proto) for x in range(ncpus - len(self.__workers))]) + self.__stats["local"].ncpus = ncpus + self.__ncpus = ncpus + + def get_active_nodes(self): + """ + Returns active nodes as a dictionary. + + [keys - nodes, values - ncpus] + """ + active_nodes = {} + for node, stat in self.__stats.items(): + if node == "local" or node in self.autopp_list \ + and self.autopp_list[node]: + active_nodes[node] = stat.ncpus + return active_nodes + + def get_stats(self): + """ + Returns job execution statistics as a dictionary. + """ + for node, stat in self.__stats.items(): + if stat.rworker: + try: + stat.rworker.send("TIME") + stat.time = float(stat.rworker.receive()) + except: + self.__accurate_stats = False + stat.time = 0.0 + return self.__stats + + def print_stats(self): + """ + Prints job execution statistics. + + Useful for benchmarking on clusters + """ + + print("Job execution statistics:") + walltime = time.time() - self.__creation_time + statistics = list(self.get_stats().items()) + totaljobs = 0.0 + for ppserver, stat in statistics: + totaljobs += stat.njobs + print(" job count | % of all jobs | job time sum | " \ + "time per job | job server") + for ppserver, stat in statistics: + if stat.njobs: + print(" %6i | %6.2f | %8.4f | %11.6f | %s" \ + % (stat.njobs, 100.0 * stat.njobs / totaljobs, stat.time, + stat.time / stat.njobs, ppserver, )) + print("Time elapsed since server creation", walltime) + + if not self.__accurate_stats: + print("WARNING: statistics provided above is not accurate" \ + " due to job rescheduling") + print() + + # all methods below are for internal use only + + def insert(self, sfunc, sargs, task=None): + """ + Inserts function into the execution queue. + + It's intended for internal use only (ppserver.py). + """ + if not task: + tid = self.__gentid() + task = _Task(self, tid) + self.__queue_lock.acquire() + self.__queue.append((task, sfunc, sargs)) + self.__queue_lock.release() + + self.__logger.info("Task %i inserted" % (task.tid, )) + self.__scheduler() + return task + + def connect1(self, host, port, persistent=True): + """ + Conects to a remote ppserver specified by host and port. + """ + try: + rworker = _RWorker(host, port, self.secret, "STAT", persistent) + ncpus = int(rworker.receive()) + hostid = host + ":" + six.text_type(port) + self.__stats[hostid] = _Statistics(ncpus, rworker) + + for x in range(ncpus): + rworker = _RWorker(host, port, self.secret, "EXEC", persistent) + self.__update_active_rworkers(rworker.id, 1) + # append is atomic - no need to lock self.__rworkers + self.__rworkers.append(rworker) + # creating reserved rworkers + for x in range(ncpus): + rworker = _RWorker(host, port, self.secret, "EXEC", persistent) + self.__update_active_rworkers(rworker.id, 1) + self.__rworkers_reserved.append(rworker) + # creating reserved4 rworkers + for x in range(ncpus * 0): + rworker = _RWorker(host, port, self.secret, "EXEC", persistent) +# self.__update_active_rworkers(rworker.id, 1) + self.__rworkers_reserved4.append(rworker) + logging.info("Connected to ppserver (host=%s, port=%i) \ + with %i workers" % (host, port, ncpus)) + self.__scheduler() + except: + pass +# sys.excepthook(*sys.exc_info()) + + def __connect(self): + """ + Connects to all remote ppservers. + """ + for ppserver in self.ppservers: + six.moves._thread.start_new_thread(self.connect1, ppserver) + + discover = ppauto.Discover(self, True) + for ppserver in self.auto_ppservers: + six.moves._thread.start_new_thread(discover.run, ppserver) + + def __detect_ncpus(self): + """ + Detects the number of effective CPUs in the system. + """ + # for Linux, Unix and MacOS + if hasattr(os, "sysconf"): + if "SC_NPROCESSORS_ONLN" in os.sysconf_names: + #Linux and Unix + ncpus = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(ncpus, int) and ncpus > 0: + return ncpus + else: + # MacOS X + return int(os.popen2("sysctl -n hw.ncpu")[1].read()) # @UndefinedVariable + # for Windows + if "NUMBER_OF_PROCESSORS" in os.environ: + ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) + if ncpus > 0: + return ncpus + # return the default value + return 1 + + def __initlog(self, loglevel, logstream=None, logfile=None): + """ + Initializes logging facility. + """ + if logfile: + log_handler = logging.FileHandler(logfile) + else: + log_handler = logging.StreamHandler(logstream) + log_handler.setLevel(loglevel) + LOG_FORMAT = ' %(levelname)s %(message)s' + log_handler.setFormatter(logging.Formatter(LOG_FORMAT)) + self.__logger = logging.getLogger('') + self.__logger.addHandler(log_handler) + self.__logger.setLevel(loglevel) + + def __dumpsfunc(self, funcs, modules): + """ + Serializes functions and modules. + """ + hashs = hash(funcs + modules) + if hashs not in self.__sfuncHM: + sources = [self.__get_source(func) for func in funcs] + # self.__sfuncHM[hashs] = pickle.dumps( + # (funcs[0].__name__, sources, modules), + # self.__pickle_proto) + self.__sfuncHM[hashs] = json.dumps({ + 'v': (funcs[0].__name__, sources, modules), + }) + return self.__sfuncHM[hashs] + + def __find_modules(self, prefix, dict): + """ + recursively finds all the modules in dict. + """ + modules = [] + for name, object in dict.items(): + if isinstance(object, types.ModuleType) \ + and name not in ("__builtins__", "pp"): + if object.__name__ == prefix + name or prefix == "": + modules.append(object.__name__) + modules.extend(self.__find_modules( + object.__name__ + ".", object.__dict__)) + return modules + + def __scheduler(self): + """ + Schedules jobs for execution. + """ + self.__queue_lock.acquire() + while self.__queue: + if self.__active_tasks < self.__ncpus: + # TODO: select a job number on the basis of heuristic + task = self.__queue.pop(0) + for worker in self.__workers: + if worker.is_free: + worker.is_free = False + break + else: + self.__logger.error("There are no free workers left") + raise RuntimeError("Error: No free workers") + self.__add_to_active_tasks(1) + try: + self.__stats["local"].njobs += 1 + six.moves._thread.start_new_thread(self.__run, task + (worker, )) + except: + pass + for i in range(len(self.__waittasks)): + if self.__waittasks[i].tid == task[0].tid: + self.__waittasks[i].worker_pid = worker.pid + break + else: + for rworker in self.__rworkers: + if rworker.is_free: + rworker.is_free = False + task = self.__queue.pop(0) + self.__stats[rworker.id].njobs += 1 + six.moves._thread.start_new_thread(self.__rrun, task + (rworker, )) + break + else: + if len(self.__queue) > self.__ncpus: + for rworker in self.__rworkers_reserved: + if rworker.is_free: + rworker.is_free = False + task = self.__queue.pop(0) + self.__stats[rworker.id].njobs += 1 + six.moves._thread.start_new_thread(self.__rrun, + task + (rworker, )) + break + else: + break + # this code will not be executed + # and is left for further releases + if len(self.__queue) > self.__ncpus * 0: + for rworker in self.__rworkers_reserved4: + if rworker.is_free: + rworker.is_free = False + task = self.__queue.pop(0) + self.__stats[rworker.id].njobs += 1 + six.moves._thread.start_new_thread(self.__rrun, + task + (rworker, )) + break + else: + break + + self.__queue_lock.release() + + def __get_source(self, func): + """ + Fetches source of the function. + """ + hashf = hash(func) + if hashf not in self.__sourcesHM: + # get lines of the source and adjust indent + sourcelines = inspect.getsourcelines(func)[0] + # remove indentation from the first line + sourcelines[0] = sourcelines[0].lstrip() + self.__sourcesHM[hashf] = "".join(sourcelines) + return self.__sourcesHM[hashf] + + def __rrun(self, job, sfunc, sargs, rworker): + """ + Runs a job remotelly. + """ + self.__logger.info("Task (remote) %i started" % (job.tid, )) + + try: + rworker.send(sfunc) + rworker.send(sargs) + sresult = rworker.receive() + rworker.is_free = True + except: + self.__logger.info("Task %i failed due to broken network " + "connection - rescheduling" % (job.tid, )) + self.insert(sfunc, sargs, job) + self.__scheduler() + self.__update_active_rworkers(rworker.id, -1) + if rworker.connect("EXEC"): + self.__update_active_rworkers(rworker.id, 1) + self.__scheduler() + return + + job.finalize(sresult) + + # remove the job from the waiting list + if self.__waittasks: + self.__waittasks_lock.acquire() + self.__waittasks.remove(job) + self.__waittasks_lock.release() + + self.__logger.info("Task (remote) %i ended" % (job.tid, )) + self.__scheduler() + + def __run(self, job, sfunc, sargs, worker): + """ + Runs a job locally. + """ + + if self.__exiting: + return + self.__logger.info("Task %i started" % (job.tid, )) + + start_time = time.time() + + sresult = None + try: + worker.t.send(sfunc) + worker.t.send(sargs) + sresult = worker.t.receive() + except: + if self.__exiting: + return + else: + if not job.cancelled: + sys.excepthook(*sys.exc_info()) + + worker.free() + + job.finalize(sresult) + + # remove the job from the waiting list + if self.__waittasks: + self.__waittasks_lock.acquire() + self.__waittasks.remove(job) + self.__waittasks_lock.release() + + self.__add_to_active_tasks(-1) + if not self.__exiting: + self.__stat_add_time("local", time.time() - start_time) + self.__logger.info("Task %i ended" % (job.tid, )) + self.__scheduler() + + def __add_to_active_tasks(self, num): + """ + Updates the number of active tasks. + """ + self.__active_tasks_lock.acquire() + self.__active_tasks += num + self.__active_tasks_lock.release() + + def __stat_add_time(self, node, time_add): + """ + Updates total runtime on the node. + """ + self.__stats_lock.acquire() + self.__stats[node].time += time_add + self.__stats_lock.release() + + def __stat_add_job(self, node): + """ + Increments job count on the node. + """ + self.__stats_lock.acquire() + self.__stats[node].njobs += 1 + self.__stats_lock.release() + + def __update_active_rworkers(self, id, count): + """ + Updates list of active rworkers. + """ + self.__active_rworkers_list_lock.acquire() + + if id not in self.autopp_list: + self.autopp_list[id] = 0 + self.autopp_list[id] += count + + self.__active_rworkers_list_lock.release() + + def __gentid(self): + """ + Generates a unique job ID number. + """ + self.__tid += 1 + return self.__tid - 1 + + def destroy(self): + """ + Kills ppworkers and closes open files. + """ + self.__exiting = True + self.__queue_lock.acquire() + self.__queue = [] + self.__queue_lock.release() + + for worker in self.__workers: + worker.t.exiting = True + if sys.platform.startswith("win"): + os.popen('TASKKILL /PID ' + six.text_type(worker.pid) + ' /F') + else: + try: + os.kill(worker.pid, 9) + os.waitpid(worker.pid, 0) + except: + pass + +# Parallel Python Software: http://www.parallelpython.com diff --git a/parallelp/pp/ppauto.py b/parallelp/pp/ppauto.py new file mode 100644 index 000000000..b6f21f27a --- /dev/null +++ b/parallelp/pp/ppauto.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# ppauto.py +# +# Parallel Python Software: http://www.parallelpython.com +# Copyright (c) 2005-2009, Vitalii Vanovschi +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +""" +Parallel Python Software, Auto-Discovery Service. + +http://www.parallelpython.com - updates, documentation, examples and support +forums +""" + +from __future__ import absolute_import +import six.moves._thread + +copyright = "Copyright (c) 2005-2009 Vitalii Vanovschi. All rights reserved" +version = "1.5.7" + +import socket +import sys +import time +import logging + +# broadcast every 10 sec +BROADCAST_INTERVAL = 10 + + +class Discover(object): + """ + Auto-discovery service class. + """ + + def __init__(self, base, isclient=False): + self.base = base + self.hosts = [] + self.isclient = isclient + + def run(self, interface_addr, broadcast_addr): + """ + Starts auto-discovery. + """ + self.interface_addr = interface_addr + self.broadcast_addr = broadcast_addr + self.bsocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.bsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.bsocket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + + try: + self.listen() + except: + sys.excepthook(*sys.exc_info()) + + def broadcast(self): + """ + Sends a broadcast. + """ + if self.isclient: + logging.debug("Client sends initial broadcast to (%s, %i)" + % self.broadcast_addr) + self.bsocket.sendto("C", self.broadcast_addr) + else: + while True: + logging.debug("Server sends broadcast to (%s, %i)" + % self.broadcast_addr) + self.bsocket.sendto("S", self.broadcast_addr) + time.sleep(BROADCAST_INTERVAL) + + def listen(self): + """ + Listens for broadcasts from other clients/servers. + """ + logging.debug("Listening (%s, %i)" % self.interface_addr) + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + s.bind(self.interface_addr) + + six.moves._thread.start_new_thread(self.broadcast, ()) + + while True: + try: + message, (host, port) = s.recvfrom(1024) + remote_address = (host, self.broadcast_addr[1]) + hostid = host + ":" + six.text_type(self.broadcast_addr[1]) + logging.debug("Discovered host (%s, %i) message=%c" + % (remote_address + (message[0], ))) + if not self.base.autopp_list.get(hostid, 0) and self.isclient \ + and message[0] == 'S': + logging.debug("Connecting to host %s" % (hostid, )) + six.moves._thread.start_new_thread(self.base.connect1, + remote_address + (False, )) + if not self.isclient and message[0] == 'C': + logging.debug("Replying to host %s" % (hostid, )) + self.bsocket.sendto("S", self.broadcast_addr) + except: + logging.error("An error has occured during execution of " + "Discover.listen") + sys.excepthook(*sys.exc_info()) diff --git a/parallelp/pp/ppserver.py b/parallelp/pp/ppserver.py new file mode 100644 index 000000000..4e072b846 --- /dev/null +++ b/parallelp/pp/ppserver.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python +# ppserver.py +# +# Parallel Python Software: http://www.parallelpython.com +# Copyright (c) 2005-2009, Vitalii Vanovschi +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +""" +Parallel Python Software, Network Server. + +http://www.parallelpython.com - updates, documentation, examples and support +forums +""" + +from __future__ import absolute_import +from __future__ import print_function +from six import text_type as str +from six.moves import range +import six.moves._thread + +copyright = "Copyright (c) 2005-2009 Vitalii Vanovschi. All rights reserved" +version = "1.5.7" + +import logging +import getopt +import sys +import socket +import random +import string +import time +import os + +from . import pptransport +from . import ppauto + +from pp import Server # @UnresolvedImport + +# compartibility with Python 2.6 +try: + import hashlib + sha_new = hashlib.sha1 +except ImportError: + import sha + sha_new = sha.new + + +class _NetworkServer(Server): + """ + Network Server Class. + """ + + def __init__(self, ncpus="autodetect", interface="0.0.0.0", + broadcast="255.255.255.255", port=None, secret=None, + timeout=None, loglevel=logging.WARNING, restart=False, + proto=0): + Server.__init__(self, ncpus, secret=secret, loglevel=loglevel, + restart=restart, proto=proto) + self.host = interface + self.bcast = broadcast + if port is not None: + self.port = port + else: + self.port = self.default_port + self.timeout = timeout + self.ncon = 0 + self.last_con_time = time.time() + self.ncon_lock = six.moves._thread.allocate_lock() + + logging.debug("Strarting network server interface=%s port=%i" + % (self.host, self.port)) + if self.timeout is not None: + logging.debug("ppserver will exit in %i seconds if no " + "connections with clients exist" % (self.timeout)) + six.moves._thread.start_new_thread(self.check_timeout, ()) + + def ncon_add(self, val): + """ + Keeps track of the number of connections and time of the last one. + """ + self.ncon_lock.acquire() + self.ncon += val + self.last_con_time = time.time() + self.ncon_lock.release() + + def check_timeout(self): + """ + Checks if timeout happened and shutdowns server if it did. + """ + while True: + if self.ncon == 0: + idle_time = time.time() - self.last_con_time + if idle_time < self.timeout: + time.sleep(self.timeout - idle_time) + else: + logging.debug("exiting ppserver due to timeout (no client" + " connections in last %i sec)", self.timeout) + os._exit(0) + else: + time.sleep(self.timeout) + + def listen(self): + """ + Initiates listenting to incoming connections. + """ + try: + ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # following allows ppserver to restart faster on the same port + ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + ssocket.bind((self.host, self.port)) + ssocket.listen(5) + except socket.error: + logging.error("Cannot create socket with port " + str(self.port) + + " (port is already in use)") + + try: + while True: + # accept connections from outside + (csocket, address) = ssocket.accept() + # now do something with the clientsocket + # in this case, we'll pretend this is a threaded server + six.moves._thread.start_new_thread(self.crun, (csocket, )) + except: + logging.debug("Closing server socket") + ssocket.close() + + def crun(self, csocket): + """ + Authenticates client and handles its jobs. + """ + mysocket = pptransport.CSocketTransport(csocket) + # send PP version + mysocket.send(version) + # generate a random string + srandom = "".join([random.choice(string.ascii_letters) + for i in range(16)]) + mysocket.send(srandom) + answer = sha_new(srandom + self.secret).hexdigest() + cleintanswer = mysocket.receive() + if answer != cleintanswer: + logging.warning("Authentification failed, client host=%s, port=%i" + % csocket.getpeername()) + mysocket.send("FAILED") + csocket.close() + return + else: + mysocket.send("OK") + + ctype = mysocket.receive() + logging.debug("Control message received: " + ctype) + self.ncon_add(1) + try: + if ctype == "STAT": + # reset time at each new connection + self.get_stats()["local"].time = 0.0 + mysocket.send(str(self.get_ncpus())) + while True: + mysocket.receive() + mysocket.send(str(self.get_stats()["local"].time)) + elif ctype == "EXEC": + while True: + sfunc = mysocket.receive() + sargs = mysocket.receive() + fun = self.insert(sfunc, sargs) + sresult = fun(True) + mysocket.send(sresult) + except: + logging.debug("Closing client socket") + csocket.close() + self.ncon_add(-1) + + def broadcast(self): + """ + Initiaates auto-discovery mechanism. + """ + discover = ppauto.Discover(self) + six.moves._thread.start_new_thread(discover.run, + ((self.host, self.port), + (self.bcast, self.port)), + ) + + +def parse_config(file_loc): + """ + Parses a config file in a very forgiving way. + """ + # If we don't have configobj installed then let the user know and exit + try: + from configobj import ConfigObj + except ImportError as ie: + print("ERROR: You must have configobj installed to use \ +configuration files. You can still use command line switches.", file=sys.stderr) + sys.exit(1) + + if not os.access(file_loc, os.F_OK): + print("ERROR: Can not access %s." % arg, file=sys.stderr) + sys.exit(1) + + # Load the configuration file + config = ConfigObj(file_loc) + # try each config item and use the result if it exists. If it doesn't + # then simply pass and move along + try: + args['secret'] = config['general'].get('secret') + except: + pass + + try: + autodiscovery = config['network'].as_bool('autodiscovery') + except: + pass + + try: + args['interface'] = config['network'].get('interface', + default="0.0.0.0") + except: + pass + + try: + args['broadcast'] = config['network'].get('broadcast') + except: + pass + + try: + args['port'] = config['network'].as_int('port') + except: + pass + + try: + args['loglevel'] = config['general'].as_bool('debug') + except: + pass + + try: + args['ncpus'] = config['general'].as_int('workers') + except: + pass + + try: + args['proto'] = config['general'].as_int('proto') + except: + pass + + try: + args['restart'] = config['general'].as_bool('restart') + except: + pass + + try: + args['timeout'] = config['network'].as_int('timeout') + except: + pass + # Return a tuple of the args dict and autodiscovery variable + return args, autodiscovery + + +def print_usage(): + """ + Prints help. + """ + print("Parallel Python Network Server (pp-" + version + ")") + print("Usage: ppserver.py [-hdar] [-n proto] [-c config_path]"\ + " [-i interface] [-b broadcast] [-p port] [-w nworkers]"\ + " [-s secret] [-t seconds]") + print() + print("Options: ") + print("-h : this help message") + print("-d : debug") + print("-a : enable auto-discovery service") + print("-r : restart worker process after each"\ + " task completion") + print("-n proto : protocol number for pickle module") + print("-c path : path to config file") + print("-i interface : interface to listen") + print("-b broadcast : broadcast address for auto-discovery service") + print("-p port : port to listen") + print("-w nworkers : number of workers to start") + print("-s secret : secret for authentication") + print("-t seconds : timeout to exit if no connections with "\ + "clients exist") + print() + print("Due to the security concerns always use a non-trivial secret key.") + print("Secret key set by -s switch will override secret key assigned by") + print("pp_secret variable in .pythonrc.py") + print() + print("Please visit http://www.parallelpython.com for extended up-to-date") + print("documentation, examples and support forums") + + +if __name__ == "__main__": + try: + opts, args = getopt.getopt(sys.argv[1:], + "hdarn:c:b:i:p:w:s:t:", ["help"]) + except getopt.GetoptError: + print_usage() + sys.exit(1) + + args = {} + autodiscovery = False + + for opt, arg in opts: + if opt in ("-h", "--help"): + print_usage() + sys.exit() + elif opt == "-c": + args, autodiscovery = parse_config(arg) + elif opt == "-d": + args["loglevel"] = logging.DEBUG + elif opt == "-i": + args["interface"] = arg + elif opt == "-s": + args["secret"] = arg + elif opt == "-p": + args["port"] = int(arg) + elif opt == "-w": + args["ncpus"] = int(arg) + elif opt == "-a": + autodiscovery = True + elif opt == "-r": + args["restart"] = True + elif opt == "-b": + args["broadcast"] = arg + elif opt == "-n": + args["proto"] = int(arg) + elif opt == "-t": + args["timeout"] = int(arg) + + server = _NetworkServer(**args) + if autodiscovery: + server.broadcast() + server.listen() + # have to destroy it here explicitelly otherwise an exception + # comes out in Python 2.4 + del server + +# Parallel Python Software: http://www.parallelpython.com diff --git a/parallelp/pp/pptransport.py b/parallelp/pp/pptransport.py new file mode 100644 index 000000000..849cd5f08 --- /dev/null +++ b/parallelp/pp/pptransport.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python +# pptransport.py +# +# Parallel Python Software: http://www.parallelpython.com +# Copyright (c) 2005-2009, Vitalii Vanovschi +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +""" +Parallel Python Software, PP Transport. + +http://www.parallelpython.com - updates, documentation, examples and support +forums +""" +from __future__ import absolute_import +import six + +copyright = "Copyright (c) 2005-2009 Vitalii Vanovschi. All rights reserved" +version = "1.5.7" + +import struct +import socket +import logging + +# compartibility with Python 2.6 +try: + import hashlib + sha_new = hashlib.sha1 + md5_new = hashlib.md5 +except ImportError: + import sha + import md5 + sha_new = sha.new + md5_new = md5.new + + +class Transport(object): + + def send(self, msg): + raise NotImplemented("abstact function 'send' must be implemented " + "in a subclass") + + def receive(self, preprocess=None): + raise NotImplemented("abstact function 'receive' must be implemented " + "in a subclass") + + def authenticate(self, secret): + remote_version = self.receive() + if version != remote_version: + logging.error("PP version mismatch (local: pp-%s, remote: pp-%s)" + % (version, remote_version)) + logging.error("Please install the same version of PP on all nodes") + return False + srandom = self.receive() + answer = sha_new(srandom + secret).hexdigest() + self.send(answer) + response = self.receive() + if response == "OK": + return True + else: + return False + + def close(self): + pass + + def _connect(self, host, port): + pass + + +class CTransport(Transport): + """ + Cached transport. + """ + rcache = {} + + def hash(self, msg): + return md5_new(msg).hexdigest() + + def csend(self, msg): + hash1 = self.hash(msg) + if hash1 in self.scache: + self.send(b"H" + hash1) + else: + self.send(b"N" + msg) + self.scache[hash1] = True + + def creceive(self, preprocess=None): + msg = self.receive() + if msg[0] == b'H': + hash1 = msg[1:] + else: + msg = msg[1:] + hash1 = self.hash(msg) + self.rcache[hash1] = [r for r in map(preprocess or (lambda m: m), (msg, ))][0] + return self.rcache[hash1] + + +class PipeTransport(Transport): + + def __init__(self, r, w): + self.scache = {} + self.exiting = False + if hasattr(r, 'read') and hasattr(w, 'write'): + self.r = r + self.w = w + else: + raise TypeError("Both arguments of PipeTransport constructor " + "must be file objects") + + def send(self, msg): + if not isinstance(msg, six.binary_type): + msg = msg.encode('latin1') + msg_size = struct.pack("!Q", len(msg)) + if not isinstance(msg_size, six.binary_type): + msg_size = msg_size.encode('latin1') + self.w.write(msg_size) + self.w.flush() + self.w.write(msg) + self.w.flush() + + def receive(self, preprocess=None): + first_bytes = struct.calcsize("!Q") + size_packed = self.r.read(first_bytes) + if not isinstance(size_packed, six.binary_type): + size_packed = size_packed.encode('latin1') + msg_len = struct.unpack("!Q", size_packed)[0] + msg = self.r.read(msg_len) + if isinstance(msg, six.binary_type): + msg = msg.decode('latin1') + return [r for r in map(preprocess or (lambda m: m), (msg, ))][0] + + def close(self): + self.w.close() + self.r.close() + + +class SocketTransport(Transport): + + def __init__(self, socket1=None): + if socket1: + self.socket = socket1 + else: + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.scache = {} + + def send(self, data): + size = struct.pack("!Q", len(data)) + t_size = struct.calcsize("!Q") + s_size = 0 + while s_size < t_size: + p_size = self.socket.send(size[s_size:]) + if p_size == 0: + raise RuntimeError("Socket connection is broken") + s_size += p_size + + t_size = len(data) + s_size = 0 + while s_size < t_size: + p_size = self.socket.send(data[s_size:]) + if p_size == 0: + raise RuntimeError("Socket connection is broken") + s_size += p_size + + def receive(self, preprocess=None): + e_size = struct.calcsize("!Q") + r_size = 0 + data = b"" + while r_size < e_size: + msg = self.socket.recv(e_size - r_size) + if not msg: + raise RuntimeError("Socket connection is broken") + r_size += len(msg) + data += msg + e_size = struct.unpack("!Q", data)[0] + + r_size = 0 + data = b"" + while r_size < e_size: + msg = self.socket.recv(e_size - r_size) + if not msg: + raise RuntimeError("Socket connection is broken") + r_size += len(msg) + data += msg + return data + + def close(self): + self.socket.close() + + def _connect(self, host, port): + self.socket.connect((host, port)) + + +class CPipeTransport(PipeTransport, CTransport): + pass + + +class CSocketTransport(SocketTransport, CTransport): + pass + +# Parallel Python Software: http://www.parallelpython.com diff --git a/parallelp/pp/ppworker.py b/parallelp/pp/ppworker.py new file mode 100644 index 000000000..20f711f12 --- /dev/null +++ b/parallelp/pp/ppworker.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# ppworker.py +# +# Parallel Python Software: http://www.parallelpython.com +# Copyright (c) 2005-2009, Vitalii Vanovschi +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +""" +Parallel Python Software, PP Worker. + +http://www.parallelpython.com - updates, documentation, examples and support +forums +""" + +from __future__ import absolute_import +from __future__ import print_function +import six +from io import BytesIO + +copyright = "Copyright (c) 2005-2009 Vitalii Vanovschi. All rights reserved" +version = "1.5.7" + +import sys +import os +import json +import traceback + +from . import pptransport + + +def import_module(name): + mod = __import__(name) + components = name.split('.') + for comp in components[1:]: + mod = getattr(mod, comp) + return mod + + +def preprocess(msg): + try: + fname, fsources, imports = json.loads(msg)['v'] + fobjs = [compile(fsource, '', 'exec') for fsource in fsources] + for module in imports: + try: + globals()[module.split('.')[0]] = __import__(module) + except: + # open('/tmp/raid.log', 'a').write(traceback.format_exc()) + # print("An error has occured during the module import") + sys.excepthook(*sys.exc_info()) + return fname, fobjs + except Exception as exc: + pass + # open('/tmp/raid.log', 'a').write(traceback.format_exc() + '\n\n%s' % msg) + + +class _WorkerProcess(object): + + def __init__(self): + self.hashmap = {} + self.e = sys.__stderr__ + self.sout = BytesIO() + origsout = sys.stdout + sys.stdout = self.sout + sys.stderr = self.sout + sin = sys.stdin + self.t = pptransport.PipeTransport(sin, origsout) + self.t.send(six.text_type(os.getpid())) + + def run(self): + try: + # execution cycle + while True: + + __fname, __fobjs = self.t.receive(preprocess) + + __sargs = self.t.receive() + + for __fobj in __fobjs: + try: + eval(__fobj) + globals().update(locals()) + except: + # print("An error has occured during the " + \ + # "function import") + sys.excepthook(*sys.exc_info()) + + __args = json.loads(__sargs)['v'] + + __f = locals()[__fname] + try: + __result = __f(*__args) + except: + # print("An error has occured during the function execution") + sys.excepthook(*sys.exc_info()) + __result = None + + __sresult = json.dumps({'v': (__result, self.sout.getvalue().decode('latin1')), }) + + self.t.send(__sresult) + self.sout.truncate(0) + except: + # print("Fatal error has occured during the function execution") + # import traceback + # open('/tmp/raid.log', 'a').write(traceback.format_exc()) + sys.excepthook(*sys.exc_info()) + __result = None + __sresult = json.dumps({'v': (__result, self.sout.getvalue().decode('latin1')), }) + + self.t.send(__sresult) + + +if __name__ == "__main__": + sys.path.append(os.path.dirname(__file__)) + wp = _WorkerProcess() + wp.run() + +# Parallel Python Software: http://www.parallelpython.com From b38154c280f8f94c8d3c90b6105a37a07c4fbdae Mon Sep 17 00:00:00 2001 From: Veselin Penev Date: Fri, 8 Mar 2019 16:12:31 +0100 Subject: [PATCH 5/8] rebuilt raid_worker() to use parallelp instead of multiprocessing --- Makefile | 3 +- main/settings.py | 7 + raid/eccmap.py | 40 ++-- raid/make.py | 37 +--- raid/raid_worker.py | 119 +++++++----- raid/read.py | 36 +--- raid/rebuild.py | 34 ++-- storage/backup_rebuilder.py | 8 +- .../{test_raid.py => test_raid_make_read.py} | 2 +- tests/test_raid_manager.py | 60 ++++++ tests/test_raid_worker.py | 176 ++++++++++++------ 11 files changed, 323 insertions(+), 199 deletions(-) rename tests/{test_raid.py => test_raid_make_read.py} (98%) create mode 100644 tests/test_raid_manager.py diff --git a/Makefile b/Makefile index 5dd1f3ba9..96c63a4ad 100644 --- a/Makefile +++ b/Makefile @@ -107,8 +107,7 @@ test_unit: $(VENV_TEST) $(PYTHON_NEW) -m unittest discover -s tests/ -v test_raid: $(VENV_TEST) - $(PYTHON_NEW) -m unittest discover -p "test_raid.py" -v - $(PYTHON_NEW) -m unittest discover -p "test_raid_worker.py" -v + $(PYTHON_NEW) -m unittest tests.test_raid_worker test_regression: PYTHON_VERSION=$(REGRESSION_PY_VER) make -C regression/ test diff --git a/main/settings.py b/main/settings.py index 0ca742946..75ceef842 100644 --- a/main/settings.py +++ b/main/settings.py @@ -236,6 +236,13 @@ def TransportLog(): return os.path.join(LogsDir(), 'transport.log') +def ParallelPLogFilename(): + """ + Log from parallelp workers goes here, raid code is executed inside child processes. + """ + return os.path.join(LogsDir(), 'parallelp.log') + + def LocalTesterLogFilename(): """ A file name path where bptester.py will write its logs. diff --git a/raid/eccmap.py b/raid/eccmap.py index f4b606702..b11d665ec 100644 --- a/raid/eccmap.py +++ b/raid/eccmap.py @@ -53,13 +53,15 @@ from __future__ import absolute_import from __future__ import print_function +from six.moves import range +from io import open + import os import re -import threading +# import threading +import traceback -from logs import lg -from six.moves import range -from io import open +# from logs import lg #------------------------------------------------------------------------------ @@ -308,7 +310,8 @@ def ReadTextFile(filename): # Windows/Linux trouble with text files return data.replace('\r\n', '\n') except: - lg.exc() + # lg.exc() + traceback.print_exc() return '' #------------------------------------------------------------------------------ @@ -334,13 +337,13 @@ def __init__(self, filename='', suppliers_number=2): self.type = 0 # 0 is data+parity on same nodes, 1 is different self.from_memory(self.name) self.convert() - lg.out(8, 'eccmap.init %s id=%d thread=%s' % (self.name, id(self), threading.currentThread().getName())) + # lg.out(8, 'eccmap.init %s id=%d thread=%s' % (self.name, id(self), threading.currentThread().getName())) - def __del__(self): - try: - lg.out(8, 'eccmap.del %s id=%d thread=%s' % (self.name, id(self), threading.currentThread().getName())) - except: - pass + # def __del__(self): + # try: + # # lg.out(8, 'eccmap.del %s id=%d thread=%s' % (self.name, id(self), threading.currentThread().getName())) + # except: + # pass def __repr__(self): return '%s' % self.name @@ -398,8 +401,9 @@ def from_memory(self, name): if datanum > maxdata: maxdata = datanum except (TypeError, ValueError): - lg.out(1, 'eccmap.from_memory ERROR') - lg.exc() + # lg.out(1, 'eccmap.from_memory ERROR') + # lg.exc() + traceback.print_exc() if oneset: self.ParityToData.append(oneset) maxparity += 1 @@ -432,7 +436,8 @@ def loadfromfile(self, fname): if datanum > maxdata: maxdata = datanum except (TypeError, ValueError): - lg.exc() + # lg.exc() + traceback.print_exc() if oneset: self.ParityToData.append(oneset) maxparity += 1 @@ -539,7 +544,8 @@ def CanMakeProgress(self, DataSegs, ParitySegs): if DataSegs[DataNum] != 1: # if this data is missing missing += 1 # increase count of those missing in this parity except: - lg.exc() + # lg.exc() + traceback.print_exc() return False # keep track of the last missing in case only one is missing if missing == 1: # if missing exactly 1 of datas in parity, we can fix the data, have work to do @@ -552,7 +558,8 @@ def CanMakeProgress(self, DataSegs, ParitySegs): if DataSegs[DataNum] != 1: # if this data is missing missing += 1 # increase count of those missing in this parity except: - lg.exc() + # lg.exc() + traceback.print_exc() return False # keep track of the last missing in case only one is missing if missing == 0: # if missing none of the data for this parity, we have work to do @@ -587,6 +594,7 @@ def GetDataFixPath(self, DataSegs, ParitySegs, DataSegNum): if (len(bestParityMap) == 0) or (len(Parity) < len(bestParityMap)): bestParityNum = paritynum bestParityMap = Parity + # open('/tmp/raid.log', 'a').write('bestParityNum=%r bestParityMap=%r\n' % (bestParityNum, bestParityMap)) return bestParityNum, bestParityMap #------------------------------------------------------------------------------ diff --git a/raid/make.py b/raid/make.py index ee3c4de15..3baf93891 100644 --- a/raid/make.py +++ b/raid/make.py @@ -50,7 +50,6 @@ from __future__ import absolute_import from io import open -from io import StringIO #------------------------------------------------------------------------------ @@ -68,32 +67,11 @@ #------------------------------------------------------------------------------ -from logs import lg - -from raid import eccmap -from raid import raidutils - -#------------------------------------------------------------------------------ - -_ECCMAP = {} +import raid.eccmap +import raid.raidutils #------------------------------------------------------------------------------ -def geteccmap(name): - global _ECCMAP - if name not in _ECCMAP: - _ECCMAP[name] = eccmap.eccmap(name) - return _ECCMAP[name] - -#------------------------------------------------------------------------------ - -def shutdown(): - global _ECCMAP - _ECCMAP.clear() - -#------------------------------------------------------------------------------ - - def RoundupFile(filename, stepsize): """ For some things we need to have files which are round sizes, for example @@ -162,7 +140,7 @@ def ReadBinaryFileAsArray(filename): def do_in_memory(filename, eccmapname, version, blockNumber, targetDir): try: INTSIZE = 4 - myeccmap = eccmap.eccmap(eccmapname) + myeccmap = raid.eccmap.eccmap(eccmapname) # any padding at end and block.Length fixes RoundupFile(filename, myeccmap.datasegments * INTSIZE) wholefile = ReadBinaryFileAsArray(filename) @@ -172,7 +150,7 @@ def do_in_memory(filename, eccmapname, version, blockNumber, targetDir): #: dict of data segments sds = {} - for seg_num, chunk in enumerate(raidutils.chunks(wholefile, int(seglength / 4))): + for seg_num, chunk in enumerate(raid.raidutils.chunks(wholefile, int(seglength / 4))): FileName = targetDir + '/' + str(blockNumber) + '-' + str(seg_num) + '-Data' with open(FileName, "wb") as f: chunk_to_write = copy.copy(chunk) @@ -180,7 +158,7 @@ def do_in_memory(filename, eccmapname, version, blockNumber, targetDir): sds[seg_num] = iter(chunk) f.write(chunk_to_write) - psds_list = raidutils.build_parity( + psds_list = raid.raidutils.build_parity( sds, int(seglength / INTSIZE), myeccmap.datasegments, myeccmap, myeccmap.paritysegments) dataNum = len(sds) @@ -194,13 +172,12 @@ def do_in_memory(filename, eccmapname, version, blockNumber, targetDir): return dataNum, parityNum except: - lg.exc() + import traceback + traceback.print_exc() return -1, -1 def main(): - from logs import lg - lg.set_debug_level(18) do_in_memory( filename=sys.argv[1], eccmapname=sys.argv[2], diff --git a/raid/raid_worker.py b/raid/raid_worker.py index ba78b1a0c..0e6a10360 100644 --- a/raid/raid_worker.py +++ b/raid/raid_worker.py @@ -61,6 +61,7 @@ import os import sys +import threading from six.moves import range @@ -71,13 +72,20 @@ #------------------------------------------------------------------------------ +if __name__ == '__main__': + import os.path as _p + sys.path.insert(0, _p.abspath(_p.join(_p.dirname(_p.abspath(sys.argv[0])), '..'))) + +#------------------------------------------------------------------------------ + from logs import lg +from automats import automat + from system import bpio -from automats import automat +from main import settings -from raid import worker from raid import read from raid import make from raid import rebuild @@ -85,31 +93,24 @@ #------------------------------------------------------------------------------ _MODULES = ( - 'os', - 'sys', - 'StringIO', - 'struct', - 'logs.lg', 'raid.read', 'raid.make', 'raid.rebuild', 'raid.eccmap', - 'raid.utils', - 'main.settings', - 'system.bpio', - 'lib.misc', - 'lib.packetid', + 'raid.raidutils', + 'os', + 'sys', 'copy', 'array', + 'traceback', + 'six', + 'io', ) _VALID_TASKS = { - 'make': (make.do_in_memory, - (make.RoundupFile, make.ReadBinaryFile, make.WriteFile, make.ReadBinaryFileAsArray)), - 'read': (read.raidread, - (read.RebuildOne, read.ReadBinaryFile,)), - 'rebuild': (rebuild.rebuild, - ()), + 'make': (make.do_in_memory, (make.RoundupFile, make.ReadBinaryFile, make.WriteFile, make.ReadBinaryFileAsArray, )), + 'read': (read.raidread, (read.RebuildOne, read.ReadBinaryFile, )), + 'rebuild': (rebuild.rebuild, ()), } #------------------------------------------------------------------------------ @@ -303,7 +304,17 @@ def doStartProcess(self, *args, **kwargs): # TODO: make an option in the software settings ncpus = int(ncpus / 2.0) - self.processor = worker.Manager(ncpus=ncpus) + if True: + from parallelp import pp + self.processor = pp.Server( + secret='bitdust', + ncpus=ncpus, + loglevel=lg.get_loging_level(_DebugLevel), + logfile=settings.ParallelPLogFilename(), + ) + else: + from raid import worker + self.processor = worker.Manager(ncpus=ncpus) self.automat('process-started') @@ -331,9 +342,9 @@ def doStartTask(self, *args, **kwargs): global _VALID_TASKS global _MODULES - if len(self.activetasks) >= self.processor.ncpus: + if len(self.activetasks) >= self.processor.get_ncpus(): lg.warn('SKIP active=%d cpus=%d' % ( - len(self.activetasks), self.processor.ncpus)) + len(self.activetasks), self.processor.get_ncpus())) return try: @@ -345,35 +356,35 @@ def doStartTask(self, *args, **kwargs): proc = self.processor.submit( func, - params, + args=params, + depfuncs=depfuncs, + modules=_MODULES, callback=lambda result: self._job_done(task_id, cmd, params, result), - error_callback=lambda err: self._job_failed(task_id, cmd, params, err), + # error_callback=lambda err: self._job_failed(task_id, cmd, params, err), ) self.activetasks[task_id] = (proc, cmd, params) if _Debug: - lg.out(_DebugLevel, 'raid_worker.doStartTask %r active=%d cpus=%d' % ( - task_id, len(self.activetasks), self.processor.ncpus)) + lg.out(_DebugLevel, 'raid_worker.doStartTask %r active=%d cpus=%d %s' % ( + task_id, len(self.activetasks), self.processor.get_ncpus(), threading.currentThread().getName())) + reactor.callLater(0.01, self.automat, 'task-started', task_id) # @UndefinedVariable def doReportTaskDone(self, *args, **kwargs): """ Action method. """ - try: - task_id, cmd, params, result = args[0] - cb = self.callbacks.pop(task_id) - reactor.callLater(0, cb, cmd, params, result) # @UndefinedVariable - if result is not None: - if _Debug: - lg.out(_DebugLevel, 'raid_worker.doReportTaskDone callbacks: %d tasks: %d active: %d' % ( - len(self.callbacks), len(self.tasks), len(self.activetasks))) - else: - if _Debug: - lg.out(_DebugLevel, 'raid_worker.doReportTaskDone result=None !!!!! callbacks: %d tasks: %d active: %d' % ( - len(self.callbacks), len(self.tasks), len(self.activetasks))) - except: - lg.exc() + task_id, cmd, params, result = args[0] + cb = self.callbacks.pop(task_id) + reactor.callLater(0, cb, cmd, params, result) # @UndefinedVariable + if result is not None: + if _Debug: + lg.out(_DebugLevel, 'raid_worker.doReportTaskDone callbacks: %d tasks: %d active: %d' % ( + len(self.callbacks), len(self.tasks), len(self.activetasks))) + else: + if _Debug: + lg.out(_DebugLevel, 'raid_worker.doReportTaskDone result=None !!!!! callbacks: %d tasks: %d active: %d' % ( + len(self.callbacks), len(self.tasks), len(self.activetasks))) def doReportTasksFailed(self, *args, **kwargs): """ @@ -399,9 +410,9 @@ def doDestroyMe(self, *args, **kwargs): def _job_done(self, task_id, cmd, params, result): if _Debug: - lg.out(_DebugLevel, 'raid_worker._job_done %r : %r active:%r cmd=%r params=%r' % ( - task_id, result, list(self.activetasks.keys()), cmd, params)) - self.automat('task-done', (task_id, cmd, params, result)) + lg.out(_DebugLevel, 'raid_worker._job_done %r : %r active:%r cmd=%r params=%r %s' % ( + task_id, result, list(self.activetasks.keys()), cmd, params, threading.currentThread().getName())) + reactor.callFromThread(self.automat, 'task-done', (task_id, cmd, params, result)) # @UndefinedVariable def _job_failed(self, task_id, cmd, params, err): lg.err('task %r FAILED : %r active:%r cmd=%r params=%r' % ( @@ -410,21 +421,37 @@ def _job_failed(self, task_id, cmd, params, err): def _kill_processor(self): if self.processor: - self.processor.terminate() + self.processor.destroy() if _Debug: lg.out(_DebugLevel, 'raid_worker._kill_processor processor was destroyed') #------------------------------------------------------------------------------ +def _read_done(cmd, taskdata, result): + lg.out(0, '_read_done %r %r %r' % (cmd, taskdata, result)) + A('shutdown') + reactor.stop() # @UndefinedVariable + + +def _make_done(cmd, taskdata, result): + lg.out(0, '_make_done %r %r %r' % (cmd, taskdata, result)) + reactor.callLater(0.5, add_task, 'read', ('/tmp/destination.txt', 'ecc/18x18', 'F12345678', '5', '/tmp/raidtest'), _read_done) # @UndefinedVariable + def main(): - def _cb(cmd, taskdata, result): - print(cmd, taskdata, result) + import base64 + bpio.init() lg.set_debug_level(20) + + os.system('rm -rf /tmp/raidtest') + os.system('mkdir -p /tmp/raidtest/F12345678') + open('/tmp/source.txt', 'w').write(base64.b64encode(os.urandom(1000)).decode()) + reactor.callWhenRunning(A, 'init') # @UndefinedVariable - reactor.callLater(0.5, A, 'new-task', ('make', _cb, ('sdfsdf', '45', '324', '45'))) # @UndefinedVariable + reactor.callLater(0.5, add_task, 'make', ('/tmp/source.txt', 'ecc/18x18', 'F12345678', '5', '/tmp/raidtest/F12345678'), _make_done) # @UndefinedVariable + reactor.run() # @UndefinedVariable if __name__ == "__main__": diff --git a/raid/read.py b/raid/read.py index 27a412130..dc69fce98 100644 --- a/raid/read.py +++ b/raid/read.py @@ -64,6 +64,7 @@ import os import sys +import traceback #------------------------------------------------------------------------------ @@ -74,31 +75,10 @@ #------------------------------------------------------------------------------ -from logs import lg - -from raid import eccmap - -#------------------------------------------------------------------------------ - -_ECCMAP = {} +import raid.eccmap #------------------------------------------------------------------------------ -def geteccmap(name): - global _ECCMAP - if name not in _ECCMAP: - _ECCMAP[name] = eccmap.eccmap(name) - return _ECCMAP[name] - -#------------------------------------------------------------------------------ - -def shutdown(): - global _ECCMAP - _ECCMAP.clear() - -#------------------------------------------------------------------------------ - - def ReadBinaryFile(filename): """ """ @@ -114,9 +94,8 @@ def ReadBinaryFile(filename): def RebuildOne(inlist, listlen, outfilename): readsize = 1 # vary from 1 byte to 4 bytes - # range(listlen) # just need a list of this size - raidfiles = [''] * listlen - raidreads = [''] * listlen # range(listlen) + raidfiles = [''] * listlen # just need a list of this size + raidreads = [''] * listlen for filenum in range(listlen): try: raidfiles[filenum] = open(inlist[filenum], "rb") @@ -128,6 +107,7 @@ def RebuildOne(inlist, listlen, outfilename): pass return False rebuildfile = open(outfilename, "wb") + progress = 0 while True: for k in range(listlen): raidreads[k] = raidfiles[k].read(2048) @@ -145,9 +125,11 @@ def RebuildOne(inlist, listlen, outfilename): out_byte = chr(xor) rebuildfile.write(out_byte) i += readsize + progress += 1 for filenum in range(listlen): raidfiles[filenum].close() rebuildfile.close() + # open('/tmp/raid.log', 'a').write('RebuildOne inlist=%s progress=%d\n' % (repr(inlist), progress)) return True @@ -167,7 +149,7 @@ def raidread( blockNumber, data_parity_dir): try: - myeccmap = eccmap.eccmap(eccmapname) + myeccmap = raid.eccmap.eccmap(eccmapname) GoodFiles = list(range(0, 200)) MakingProgress = 1 while MakingProgress == 1: @@ -222,7 +204,7 @@ def raidread( return GoodDSegs except: - lg.exc() + traceback.print_exc() return None diff --git a/raid/rebuild.py b/raid/rebuild.py index 57e750cf4..df9d37c64 100644 --- a/raid/rebuild.py +++ b/raid/rebuild.py @@ -29,21 +29,21 @@ #------------------------------------------------------------------------------ import os +import traceback #------------------------------------------------------------------------------ -from logs import lg - -from lib import packetid - -from main import settings - -from raid import read +import raid.read +import raid.eccmap #------------------------------------------------------------------------------ -def rebuild(backupID, blockNum, eccMap, availableSuppliers, remoteMatrix, localMatrix): +def rebuild(backupID, blockNum, eccMap, availableSuppliers, remoteMatrix, localMatrix, localBackupsDir): try: + customer, _, localPath = backupID.rpartition(':') + if '$' not in customer: + customer = 'master$' + customer + myeccmap = raid.eccmap.eccmap(eccMap) supplierCount = len(availableSuppliers) missingData = [0] * supplierCount missingParity = [0] * supplierCount @@ -53,11 +53,10 @@ def rebuild(backupID, blockNum, eccMap, availableSuppliers, remoteMatrix, localM remoteParity = list(remoteMatrix['P']) localData = list(localMatrix['D']) localParity = list(localMatrix['P']) - customer, localPath = packetid.SplitPacketID(backupID) def _build_raid_file_name(supplierNumber, dataOrParity): return os.path.join( - settings.getLocalBackupsDir(), + localBackupsDir, customer, localPath, str(blockNum) + '-' + str(supplierNumber) + '-' + dataOrParity) @@ -78,6 +77,10 @@ def _build_raid_file_name(supplierNumber, dataOrParity): # same for Parity file if remoteParity[supplierNum] != 1: missingParity[supplierNum] = 1 + + # open('/tmp/raid.log', 'a').write('missingData=%r missingParity=%r\n' % (missingData, missingParity)) + # open('/tmp/raid.log', 'a').write('localData=%r localParity=%r\n' % (localData, localParity)) + # This made an attempt to rebuild the missing pieces # from pieces we have on hands. # lg.out(14, 'block_rebuilder.AttemptRebuild %s %d BEGIN' % (self.backupID, self.blockNum)) @@ -90,7 +93,7 @@ def _build_raid_file_name(supplierNumber, dataOrParity): dataFileName = _build_raid_file_name(supplierNum, 'Data') # if we do not have this item on hands - we will reconstruct it from other items if localData[supplierNum] == 0: - parityNum, parityMap = eccMap.GetDataFixPath(localData, localParity, supplierNum) + parityNum, parityMap = myeccmap.GetDataFixPath(localData, localParity, supplierNum) if parityNum != -1: rebuildFileList = [] rebuildFileList.append(_build_raid_file_name(parityNum, 'Parity')) @@ -100,7 +103,7 @@ def _build_raid_file_name(supplierNumber, dataOrParity): if os.path.isfile(filename): rebuildFileList.append(filename) # lg.out(10, ' rebuilding file %s from %d files' % (os.path.basename(dataFileName), len(rebuildFileList))) - read.RebuildOne(rebuildFileList, len(rebuildFileList), dataFileName) + raid.read.RebuildOne(rebuildFileList, len(rebuildFileList), dataFileName) if os.path.exists(dataFileName): localData[supplierNum] = 1 madeProgress = True @@ -118,7 +121,7 @@ def _build_raid_file_name(supplierNumber, dataOrParity): for supplierNum in range(supplierCount): parityFileName = _build_raid_file_name(supplierNum, 'Parity') if localParity[supplierNum] == 0: - parityMap = eccMap.ParityToData[supplierNum] + parityMap = myeccmap.ParityToData[supplierNum] HaveAllData = True for segment in parityMap: if localData[segment] == 0: @@ -131,7 +134,7 @@ def _build_raid_file_name(supplierNumber, dataOrParity): if os.path.isfile(filename): rebuildFileList.append(filename) # lg.out(10, ' rebuilding file %s from %d files' % (os.path.basename(parityFileName), len(rebuildFileList))) - read.RebuildOne(rebuildFileList, len(rebuildFileList), parityFileName) + raid.read.RebuildOne(rebuildFileList, len(rebuildFileList), parityFileName) if os.path.exists(parityFileName): # lg.out(10, ' Parity file %s found after rebuilding for supplier %d' % (os.path.basename(parityFileName), supplierNum)) localParity[supplierNum] = 1 @@ -146,5 +149,6 @@ def _build_raid_file_name(supplierNumber, dataOrParity): return (newData, localData, localParity, reconstructedData, reconstructedParity, ) except: - lg.exc() + # lg.exc() + traceback.print_exc() return None diff --git a/storage/backup_rebuilder.py b/storage/backup_rebuilder.py index 2a116bf64..3aa382ec5 100644 --- a/storage/backup_rebuilder.py +++ b/storage/backup_rebuilder.py @@ -591,10 +591,14 @@ def _start_one_block(self): lg.out(10, 'backup_rebuilder._start_one_block %d to rebuild, blockIndex=%d, other blocks: %s' % ( (BlockNumber, self.blockIndex, str(self.workingBlocksQueue)))) task_params = ( - self.currentBackupID, BlockNumber, eccmap.Current(), + self.currentBackupID, + BlockNumber, + eccmap.Current().name, backup_matrix.GetActiveArray(), backup_matrix.GetRemoteMatrix(self.currentBackupID, BlockNumber), - backup_matrix.GetLocalMatrix(self.currentBackupID, BlockNumber),) + backup_matrix.GetLocalMatrix(self.currentBackupID, BlockNumber), + settings.getLocalBackupsDir(), + ) raid_worker.add_task( 'rebuild', task_params, diff --git a/tests/test_raid.py b/tests/test_raid_make_read.py similarity index 98% rename from tests/test_raid.py rename to tests/test_raid_make_read.py index f951e6649..b9edfa5c1 100644 --- a/tests/test_raid.py +++ b/tests/test_raid_make_read.py @@ -4,7 +4,7 @@ import subprocess -class Test(TestCase): +class TestMakeRead(TestCase): def setUp(self): t = time.time() diff --git a/tests/test_raid_manager.py b/tests/test_raid_manager.py new file mode 100644 index 000000000..34b9dde3f --- /dev/null +++ b/tests/test_raid_manager.py @@ -0,0 +1,60 @@ +from unittest import TestCase +import time +import mock + +from raid.worker import Manager + + +def heavy_task(id): + time.sleep(2) + return id + + +def light_task(): + return None + + +class TestManager(TestCase): + + def setUp(self): + self.manager = Manager(2) + + def tearDown(self): + self.manager.terminate() + + def test_submit(self): + callback = mock.Mock() + + self.manager.submit(func=light_task, params=(), callback=callback) + self.manager.submit(func=light_task, params=(), callback=callback) + self.manager.submit(func=light_task, params=(), callback=callback) + self.manager.submit(func=light_task, params=(), callback=callback) + self.manager.submit(func=light_task, params=(), callback=callback) + self.manager.submit(func=light_task, params=(), callback=callback) + self.manager.submit(func=light_task, params=(), callback=callback) + self.manager.submit(func=light_task, params=(), callback=callback) + self.manager.submit(func=light_task, params=(), callback=callback) + + time.sleep(5) + assert callback.call_count == 9, callback.call_count + + def test_callback(self): + heavy_task_1_callback = mock.Mock() + self.manager.submit(func=heavy_task, params=(1,), callback=heavy_task_1_callback) + time.sleep(5) + heavy_task_1_callback.assert_called_once_with(1) + + def test_cancellation(self): + heavy_task_1_callback = mock.Mock() + heavy_task_2_callback = mock.Mock() + heavy_task_3_callback = mock.Mock() + self.manager.submit(func=heavy_task, params=(1,), callback=heavy_task_1_callback) + self.manager.submit(func=heavy_task, params=(2,), callback=heavy_task_2_callback) + r3 = self.manager.submit(func=heavy_task, params=(3,), callback=heavy_task_3_callback) + + self.manager.cancel(r3.tid) + time.sleep(5) + + heavy_task_1_callback.assert_called_once() + heavy_task_2_callback.assert_called_once() + heavy_task_3_callback.assert_not_called() diff --git a/tests/test_raid_worker.py b/tests/test_raid_worker.py index 4756181de..8d2aa5640 100644 --- a/tests/test_raid_worker.py +++ b/tests/test_raid_worker.py @@ -1,60 +1,116 @@ -from unittest import TestCase -import time -import mock - -from raid.worker import Manager - - -def heavy_task(id): - time.sleep(2) - return id - - -def light_task(): - return None - - -class Test(TestCase): - - def setUp(self): - self.manager = Manager(2) - - def tearDown(self): - self.manager.terminate() - - def test_submit(self): - callback = mock.Mock() - - self.manager.submit(func=light_task, params=(), callback=callback) - self.manager.submit(func=light_task, params=(), callback=callback) - self.manager.submit(func=light_task, params=(), callback=callback) - self.manager.submit(func=light_task, params=(), callback=callback) - self.manager.submit(func=light_task, params=(), callback=callback) - self.manager.submit(func=light_task, params=(), callback=callback) - self.manager.submit(func=light_task, params=(), callback=callback) - self.manager.submit(func=light_task, params=(), callback=callback) - self.manager.submit(func=light_task, params=(), callback=callback) - - time.sleep(5) - assert callback.call_count == 9, callback.call_count - - def test_callback(self): - heavy_task_1_callback = mock.Mock() - self.manager.submit(func=heavy_task, params=(1,), callback=heavy_task_1_callback) - time.sleep(5) - heavy_task_1_callback.assert_called_once_with(1) - - def test_cancellation(self): - heavy_task_1_callback = mock.Mock() - heavy_task_2_callback = mock.Mock() - heavy_task_3_callback = mock.Mock() - self.manager.submit(func=heavy_task, params=(1,), callback=heavy_task_1_callback) - self.manager.submit(func=heavy_task, params=(2,), callback=heavy_task_2_callback) - r3 = self.manager.submit(func=heavy_task, params=(3,), callback=heavy_task_3_callback) - - self.manager.cancel(r3.tid) - time.sleep(5) - - heavy_task_1_callback.assert_called_once() - heavy_task_2_callback.assert_called_once() - heavy_task_3_callback.assert_not_called() +import os +import base64 + +from twisted.trial.unittest import TestCase +from twisted.internet import reactor # @UnresolvedImport +from twisted.internet.defer import Deferred +from twisted.internet.base import DelayedCall +DelayedCall.debug = True + + +class TestRaidWorker(TestCase): + + def _test_make_rebuild_read(self, target_ecc_map, num_suppliers, dead_suppliers, read_success, rebuild_one_success, filesize): + test_result = Deferred() + + curdir = os.getcwd() + if curdir.count('_trial_temp'): + os.chdir(os.path.abspath(os.path.join(curdir, '..'))) + + from raid import raid_worker + + from logs import lg + lg.set_debug_level(20) + + def _read_done(cmd, taskdata, result): + # self.assertEqual(result, num_suppliers) + if read_success: + self.assertEqual(open('/tmp/source.txt', 'r').read(), open('/tmp/destination.txt', 'r').read()) + else: + self.assertNotEqual(open('/tmp/source.txt', 'r').read(), open('/tmp/destination.txt', 'r').read()) + # os.system('rm -rf /tmp/source.txt') + # os.system('rm -rf /tmp/destination.txt') + # os.system('rm -rf /tmp/raidtest') + reactor.callLater(0, raid_worker.A, 'shutdown') # @UndefinedVariable + reactor.callLater(0.1, test_result.callback, True) # @UndefinedVariable + return True + + def _rebuild_done(cmd, taskdata, result): + newData, localData, localParity, reconstructedData, reconstructedParity = result + if rebuild_one_success: + self.assertEqual(newData, True) + else: + self.assertEqual(newData, False) + # try to read all fragments now + reactor.callLater(0.5, raid_worker.add_task, 'read', ( # @UndefinedVariable + '/tmp/destination.txt', target_ecc_map, 'F12345678', '5', '/tmp/raidtest/master$alice@somehost.com/0'), _read_done) + return True + + def _make_done(cmd, taskdata, result): + self.assertEqual(result, [num_suppliers, num_suppliers]) + # remove few fragments and try to rebuild the whole block + for supplier_position in range(dead_suppliers): + os.system("rm -rf '/tmp/raidtest/master$alice@somehost.com/0/F12345678/5-%d-Data'" % supplier_position) + os.system("rm -rf '/tmp/raidtest/master$alice@somehost.com/0/F12345678/5-%d-Parity'" % supplier_position) + alive_suppliers = num_suppliers - dead_suppliers + remote_fragments = { + 'D': [0, ] * dead_suppliers + [1, ] * alive_suppliers, + 'P': [0, ] * dead_suppliers + [1, ] * alive_suppliers, + } + local_fragments = { + 'D': [0, ] * dead_suppliers + [1, ] * alive_suppliers, + 'P': [0, ] * dead_suppliers + [1, ] * alive_suppliers, + } + reactor.callLater(0.5, raid_worker.add_task, 'rebuild', ( # @UndefinedVariable + 'master$alice@somehost.com:0/F12345678', '5', target_ecc_map, [1, ] * num_suppliers, remote_fragments, local_fragments, '/tmp/raidtest'), _rebuild_done) + return True + + os.system('rm -rf /tmp/source.txt') + os.system('rm -rf /tmp/destination.txt') + os.system('rm -rf /tmp/raidtest') + os.system("mkdir -p '/tmp/raidtest/master$alice@somehost.com/0/F12345678'") + open('/tmp/source.txt', 'w').write(base64.b64encode(os.urandom(filesize)).decode()) + reactor.callWhenRunning(raid_worker.A, 'init') # @UndefinedVariable + reactor.callLater(0.5, raid_worker.add_task, 'make', ( # @UndefinedVariable + '/tmp/source.txt', target_ecc_map, 'F12345678', '5', '/tmp/raidtest/master$alice@somehost.com/0/F12345678'), _make_done) + return test_result + + def test_ecc18x18_with_5_dead_suppliers_success(self): + return self._test_make_rebuild_read( + target_ecc_map='ecc/18x18', + num_suppliers=18, + dead_suppliers=5, # for 18 suppliers max 5 "correctable" errors are possible, see raid/eccmap.py + rebuild_one_success=True, + read_success=True, + filesize=10000, + ) + + def test_ecc18x18_with_10_dead_suppliers_failed(self): + return self._test_make_rebuild_read( + target_ecc_map='ecc/18x18', + num_suppliers=18, + dead_suppliers=10, + rebuild_one_success=True, + read_success=False, + filesize=10000, + ) + + def test_ecc18x18_with_14_dead_suppliers_failed(self): + return self._test_make_rebuild_read( + target_ecc_map='ecc/18x18', + num_suppliers=18, + dead_suppliers=14, + rebuild_one_success=False, + read_success=False, + filesize=10000, + ) + + def test_ecc64x64_with_10_dead_suppliers_success(self): + return self._test_make_rebuild_read( + target_ecc_map='ecc/64x64', + num_suppliers=64, + dead_suppliers=10, # for 64 suppliers max 10 "correctable" errors are possible, see raid/eccmap.py + rebuild_one_success=True, + read_success=True, + filesize=10000, + ) From 4b55ee8270aaa9afbf5f1fba45b6c1813a0fb10d Mon Sep 17 00:00:00 2001 From: Veselin Penev Date: Fri, 8 Mar 2019 17:17:35 +0100 Subject: [PATCH 6/8] able to set python version for unit tests on travis --- .travis.yml | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 18c4af7ad..61dbb9eac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,11 @@ matrix: - name: "2.7" python: "2.7" env: REGRESSION_PY_VER=2.7.15 + env: VENV_PYTHON_VERSION=python2.7 - name: "3.6" python: "3.6" env: REGRESSION_PY_VER=3.6 + env: VENV_PYTHON_VERSION=python3.6 before_install: - set -e diff --git a/Makefile b/Makefile index 96c63a4ad..265f9fa26 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ VENV=${HOME}/.bitdust/venv PIP=${VENV}/bin/pip PIP_NEW=venv/bin/pip # REGRESSION_PY_VER=3.6 -VENV_PYTHON_VERSION=python2.7 +# VENV_PYTHON_VERSION=python3.6 CMD_FROM_VENV:=". ${VENV}/bin/activate; which" TOX=$(shell "$(CMD_FROM_VENV)" "tox") PYTHON=$(shell "$(CMD_FROM_VENV)" "python") From c0ce06d5aef2dc36848ff8b6beaa83ce4cea812a Mon Sep 17 00:00:00 2001 From: Veselin Penev Date: Sat, 15 Jun 2024 16:28:00 +0200 Subject: [PATCH 7/8] updated CHANGELOG, prepare next release --- CHANGELOG.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index a504da15a..478a58554 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,12 @@ Change Log ========== +2024-06-15 Veselin Penev [penev.veselin@gmail.com](mailto:penev.veselin@gmail.com) + +* switched from ed25519 to PyNaCl + + + 2024-06-01 Veselin Penev [penev.veselin@gmail.com](mailto:penev.veselin@gmail.com) * introduce new group streaming solution From b457459ca148b50c9e905c6b89a5eaa81c2c156f Mon Sep 17 00:00:00 2001 From: Veselin Penev Date: Sun, 16 Jun 2024 22:30:55 +0200 Subject: [PATCH 8/8] new file: .github/workflows/macos_venv.yml --- .github/workflows/macos_venv.yml | 22 ++++++++++++++++++++ requirements.txt | 35 ++++++++++++++++---------------- 2 files changed, 39 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/macos_venv.yml diff --git a/.github/workflows/macos_venv.yml b/.github/workflows/macos_venv.yml new file mode 100644 index 000000000..e01f525e6 --- /dev/null +++ b/.github/workflows/macos_venv.yml @@ -0,0 +1,22 @@ +name: MacOS build +on: [push] + +jobs: + macos_build_venv: + name: macos build + runs-on: macos-11 + + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v4 + + - name: set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: make clean venv + run: make clean venv diff --git a/requirements.txt b/requirements.txt index 23a0449d5..d2bd1c426 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,31 +9,30 @@ # pre_commit appdirs==1.4.4 -attrs==22.2.0 +attrs==23.2.0 Automat==22.10.0 base58==2.1.1 cffi==1.15.1 -coincurve==16.0.0 -constantly==15.1.0 -coverage==6.2 -cryptography==39.0.0 +coincurve==20.0.0 +constantly==23.10.4 +# coverage==6.2 +cryptography==42.0.8 hyperlink==21.0.0 -idna==3.4 +idna==3.7 incremental==22.10.0 -mock==5.0.1 -psutil==5.9.4 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pycparser==2.21 -pycryptodomex==3.16.0 -PyNaCl==1.5 -pynacl==1.5.0 -pyparsing==3.0.9 +# mock==5.0.1 +psutil==5.9.8 +pyasn1==0.6.0 +pyasn1-modules==0.4.0 +pycparser==2.22 +pycryptodomex==3.20.0 +PyNaCl==1.5.0 +pyparsing==3.1.2 PySocks==1.7.1 -requests==2.27.1 -service-identity==21.1.0 +requests==2.32.3 +service-identity==24.1.0 six==1.16.0 Twisted==22.4.0 -typing_extensions==4.1.1 +typing_extensions==4.12.2 zope.interface==5.5.2