Merge pull request p2pool#42 from jtoomim/stratum-autodiff

Stratum autodiff

p2pool/bitcoin/data.py

@@ -137,6 +137,15 @@ def is_segwit_tx(tx):
     ('lock_time', pack.IntType(32))
 ])
 
+def get_stripped_size(tx):
+    if not 'stripped_size' in tx:
+        tx['stripped_size'] = tx_id_type.packed_size(tx)
+    return tx['stripped_size']
+def get_size(tx):
+    if not 'size' in tx:
+        tx['size'] = tx_id_type.packed_size(tx)
+    return tx['size']
+
 class TransactionType(pack.Type):
     _int_type = pack.IntType(32)
     _varint_type = pack.VarIntType()

p2pool/bitcoin/networks/bitcoin_testnet.py

@@ -15,7 +15,7 @@
 RPC_PORT = 18332
 RPC_CHECK = defer.inlineCallbacks(lambda bitcoind: defer.returnValue(
             'getreceivedbyaddress' in (yield bitcoind.rpc_help()) and
-            (yield bitcoind.rpc_getinfo())['testnet']
+            (yield bitcoind.rpc_getblockchaininfo())['chain'] == 'test'
         ))
 SUBSIDY_FUNC = lambda height: 50*100000000 >> (height + 1)//210000
 POW_FUNC = data.hash256
@@ -25,6 +25,6 @@
 BLOCK_EXPLORER_URL_PREFIX = 'http://blockexplorer.com/testnet/block/'
 ADDRESS_EXPLORER_URL_PREFIX = 'http://blockexplorer.com/testnet/address/'
 TX_EXPLORER_URL_PREFIX = 'http://blockexplorer.com/testnet/tx/'
-SANE_TARGET_RANGE = (2**256//2**32//1000 - 1, 2**256//2**32 - 1)
+SANE_TARGET_RANGE = (2**256//2**32//100000000 - 1, 2**256//2**32 - 1)
 DUMB_SCRYPT_DIFF = 1
 DUST_THRESHOLD = 1e8

p2pool/bitcoin/networks/bitcoincash_testnet.py

@@ -26,7 +26,7 @@
 BLOCK_EXPLORER_URL_PREFIX = 'https://explorer.bitcoin.com/tbch/block/'
 ADDRESS_EXPLORER_URL_PREFIX = 'https://explorer.bitcoin.com/tbch/address/'
 TX_EXPLORER_URL_PREFIX = 'https://www.blocktrail.com/tBCC/tx/'
-SANE_TARGET_RANGE = (2**256//2**32//1000 - 1, 2**256//2**32 - 1)
+SANE_TARGET_RANGE = (2**256//2**32//100000000 - 1, 2**256//2**32 - 1)
 BLOCK_EXPLORER_URL_PREFIX = 'https://explorer.bitcoin.com/tbch/block/'
 ADDRESS_EXPLORER_URL_PREFIX = 'https://explorer.bitcoin.com/tbch/address/'
 TX_EXPLORER_URL_PREFIX = 'https://explorer.bitcoin.com/tbch/tx/'

p2pool/bitcoin/networks/bitcoinsv_testnet.py

@@ -26,6 +26,6 @@
 BLOCK_EXPLORER_URL_PREFIX = 'https://test.whatsonchain.com/block-height/'
 ADDRESS_EXPLORER_URL_PREFIX = 'https://test.whatsonchain.com/tx/'
 TX_EXPLORER_URL_PREFIX = 'https://test.whatsonchain.com/address/'
-SANE_TARGET_RANGE = (2**256//2**32//1000 - 1, 2**256//2**32 - 1)
+SANE_TARGET_RANGE = (2**256//2**32//100000000 - 1, 2**256//2**32 - 1)
 DUMB_SCRYPT_DIFF = 1
 DUST_THRESHOLD = 1e8

p2pool/bitcoin/stratum.py

@@ -1,12 +1,15 @@
 import random
 import sys
+import time
 
 from twisted.internet import protocol, reactor
 from twisted.python import log
 
 from p2pool.bitcoin import data as bitcoin_data, getwork
 from p2pool.util import expiring_dict, jsonrpc, pack
 
+def clip(num, bot, top):
+    return min(top, max(bot, num))
 
 class StratumRPCMiningProvider(object):
     def __init__(self, wb, other, transport):
@@ -19,6 +22,13 @@ def __init__(self, wb, other, transport):
         self.handler_map = expiring_dict.ExpiringDict(300)
 
         self.watch_id = self.wb.new_work_event.watch(self._send_work)
+
+        self.recent_shares = []
+        self.target = None
+        self.share_rate = wb.share_rate
+        self.fixed_target = False
+        self.desired_pseudoshare_target = None
+
 
     def rpc_subscribe(self, miner_version=None, session_id=None):
         reactor.callLater(0, self._send_work)
@@ -35,6 +45,7 @@ def rpc_authorize(self, username, password):
             self.authorized = username
         self.username = username.strip()
 
+        self.user, self.address, self.desired_share_target, self.desired_pseudoshare_target = self.wb.get_user_details(username)
         reactor.callLater(0, self._send_work)
         return True
 
@@ -62,8 +73,15 @@ def _send_work(self):
             log.err()
             self.transport.loseConnection()
             return
+        if self.desired_pseudoshare_target:
+            self.fixed_target = True
+            self.target = self.desired_pseudoshare_target
+            self.target = max(self.target, int(x['bits'].target))
+        else:
+            self.fixed_target = False
+            self.target = x['share_target'] if self.target == None else max(x['min_share_target'], self.target)
         jobid = str(random.randrange(2**128))
-        self.other.svc_mining.rpc_set_difficulty(bitcoin_data.target_to_difficulty(x['share_target'])*self.wb.net.DUMB_SCRYPT_DIFF).addErrback(lambda err: None)
+        self.other.svc_mining.rpc_set_difficulty(bitcoin_data.target_to_difficulty(self.target)*self.wb.net.DUMB_SCRYPT_DIFF).addErrback(lambda err: None)
         self.other.svc_mining.rpc_notify(
             jobid, # jobid
             getwork._swap4(pack.IntType(256).pack(x['previous_block'])).encode('hex'), # prevhash
@@ -92,7 +110,6 @@ def rpc_submit(self, worker_name, job_id, extranonce2, ntime, nonce, version_bit
         job_version = x['version']
         nversion = job_version
         #check if miner changed bits that they were not supposed to change
-        #print version_bits
         if version_bits:
             if ((~self.pool_version_mask) & int(version_bits,16)) != 0:
                 #todo: how to raise error back to miner?
@@ -110,7 +127,26 @@ def rpc_submit(self, worker_name, job_id, extranonce2, ntime, nonce, version_bit
             bits=x['bits'],
             nonce=pack.IntType(32).unpack(getwork._swap4(nonce.decode('hex'))),
         )
-        return got_response(header, worker_name, coinb_nonce)
+        result = got_response(header, worker_name, coinb_nonce, self.target)
+
+        # adjust difficulty on this stratum to target ~10sec/pseudoshare
+        if not self.fixed_target:
+            self.recent_shares.append(time.time())
+            if len(self.recent_shares) > 12 or (time.time() - self.recent_shares[0]) > 10*len(self.recent_shares)*self.share_rate:
+                old_time = self.recent_shares[0]
+                del self.recent_shares[0]
+                olddiff = bitcoin_data.target_to_difficulty(self.target)
+                self.target = int(self.target * clip((time.time() - old_time)/(len(self.recent_shares)*self.share_rate), 0.5, 2.) + 0.5)
+                newtarget = clip(self.target, self.wb.net.SANE_TARGET_RANGE[0], self.wb.net.SANE_TARGET_RANGE[1])
+                if newtarget != self.target:
+                    print "Clipping target from %064x to %064x" % (self.target, newtarget)
+                    self.target = newtarget
+                self.target = max(x['min_share_target'], self.target)
+                self.recent_shares = [time.time()]
+                self._send_work()
+
+        return result
+
 
     def close(self):
         self.wb.new_work_event.unwatch(self.watch_id)

p2pool/bitcoin/worker_interface.py

@@ -120,23 +120,28 @@ def __init__(self, inner):
         self._cache = {}
         self._times = None
 
-    def get_work(self, *args):
+    def get_work(self, user, address, desired_share_target,
+                 desired_pseudoshare_target, worker_ip=None, *args):
         if self._times != self.new_work_event.times:
             self._cache = {}
             self._times = self.new_work_event.times
 
-        if args not in self._cache:
-            x, handler = self._inner.get_work(*args)
-            self._cache[args] = x, handler, 0
+        cachekey = (address, desired_share_target, args)
+        if cachekey not in self._cache:
+            x, handler = self._inner.get_work(user, address, desired_share_target,
+                desired_pseudoshare_target, worker_ip, *args)
+            self._cache[cachekey] = x, handler, 0
 
-        x, handler, nonce = self._cache.pop(args)
+        x, handler, nonce = self._cache.pop(cachekey)
 
         res = (
             dict(x, coinb1=x['coinb1'] + pack.IntType(self._my_bits).pack(nonce)),
-            lambda header, user, coinbase_nonce: handler(header, user, pack.IntType(self._my_bits).pack(nonce) + coinbase_nonce),
+            lambda header, user, coinbase_nonce, pseudoshare_target: handler(header, user, pack.IntType(self._my_bits).pack(nonce) + coinbase_nonce, pseudoshare_target),
         )
 
         if nonce + 1 != 2**self._my_bits:
-            self._cache[args] = x, handler, nonce + 1
+            self._cache[cachekey] = x, handler, nonce + 1
 
         return res
+    def __getattr__(self, attr):
+        return getattr(self._inner, attr)

p2pool/data.py

@@ -176,18 +176,19 @@ def generate_transaction(cls, tracker, share_data, block_target, desired_timesta
         transaction_hash_refs = []
         other_transaction_hashes = []
         t1 = time.time()
-        past_shares = list(tracker.get_chain(share_data['previous_share_hash'], min(height, 100)))
         tx_hash_to_this = {}
         if cls.VERSION < 34:
+            past_shares = list(tracker.get_chain(share_data['previous_share_hash'], min(height, 100)))
             for i, share in enumerate(past_shares):
                 for j, tx_hash in enumerate(share.new_transaction_hashes):
                     if tx_hash not in tx_hash_to_this:
                         tx_hash_to_this[tx_hash] = [1+i, j] # share_count, tx_count
+
         t2 = time.time()
         for tx_hash, fee in desired_other_transaction_hashes_and_fees:
             if known_txs is not None:
-                this_stripped_size = bitcoin_data.tx_id_type.packed_size(known_txs[tx_hash])
-                this_real_size     = bitcoin_data.tx_type.packed_size(known_txs[tx_hash])
+                this_stripped_size = bitcoin_data.get_stripped_size(known_txs[tx_hash])
+                this_real_size     = bitcoin_data.get_size(known_txs[tx_hash])
                 this_weight        = this_real_size + 3*this_stripped_size
             else: # we're just verifying someone else's share. We'll calculate sizes in should_punish_reason()
                 this_stripped_size = 0
@@ -612,8 +613,8 @@ def should_punish_reason(self, previous_block, bits, tracker, known_txs):
             pass
         else:
             if not hasattr(self, 'all_tx_size'):
-                self.all_txs_size = sum(bitcoin_data.tx_type.packed_size(tx) for tx in other_txs)
-                self.stripped_txs_size = sum(bitcoin_data.tx_id_type.packed_size(tx) for tx in other_txs)
+                self.all_txs_size = sum(bitcoin_data.get_size(tx) for tx in other_txs)
+                self.stripped_txs_size = sum(bitcoin_data.get_stripped_size(tx) for tx in other_txs)
             if self.all_txs_size + 3 * self.stripped_txs_size + 4*80 + self.gentx_weight > tracker.net.BLOCK_MAX_WEIGHT:
                 return True, 'txs over block weight limit'
             if self.stripped_txs_size + 80 + self.gentx_size > tracker.net.BLOCK_MAX_SIZE:
@@ -797,8 +798,6 @@ def think(self, block_rel_height_func, block_abs_height_func, previous_block, bi
             -self.items[h].should_punish_reason(previous_block, bits, self, known_txs)[0],
         ), h) for h in self.verified.tails.get(best_tail, []))
         punish_aggressively = traditional_sort[-1][0][2] if traditional_sort else False
-        if punish_aggressively > 0:
-            print "Other nodes are going to follow a share we want to punish! Time to hulk up."
 
         if p2pool.DEBUG:
             print len(decorated_heads), 'heads. Top 10:'

p2pool/main.py

@@ -293,7 +293,7 @@ def upnp_thread():
 
         wb = work.WorkerBridge(node, my_address, args.donation_percentage,
                                merged_urls, args.worker_fee, args, pubkeys,
-                               bitcoind)
+                               bitcoind, args.share_rate)
         web_root = web.get_web_root(wb, datadir_path, bitcoind_getinfo_var, static_dir=args.web_static)
         caching_wb = worker_interface.CachingWorkerBridge(wb)
         worker_interface.WorkerInterface(caching_wb).attach_to(web_root, get_handler=lambda request: request.redirect('/static/'))
@@ -544,6 +544,9 @@ def run():
     worker_group.add_argument('-f', '--fee', metavar='FEE_PERCENTAGE',
         help='''charge workers mining to their own bitcoin address (by setting their miner's username to a bitcoin address) this percentage fee to mine on your p2pool instance. Amount displayed at http://127.0.0.1:WORKER_PORT/fee (default: 0)''',
         type=float, action='store', default=0, dest='worker_fee')
+    worker_group.add_argument('-s', '--share-rate', metavar='SECONDS_PER_SHARE',
+        help='Auto-adjust mining difficulty on each connection to target this many seconds per pseudoshare (default: %3.0f)' % 3.,
+        type=float, action='store', default=3., dest='share_rate')
 
     bitcoind_group = parser.add_argument_group('bitcoind interface')
     bitcoind_group.add_argument('--bitcoind-config-path', metavar='BITCOIND_CONFIG_PATH',

p2pool/networks/bitcoincash.py

@@ -31,7 +31,7 @@
 ANNOUNCE_CHANNEL = '#p2pool'
 VERSION_CHECK = lambda v: None if 100000 <= v else 'Bitcoin version too old. Upgrade to 0.11.2 or newer!' # not a bug. BIP65 support is ensured by SOFTFORKS_REQUIRED
 VERSION_WARNING = lambda v: None
-SOFTFORKS_REQUIRED = set(['bip65', 'csv'])
+SOFTFORKS_REQUIRED = set()
 MINIMUM_PROTOCOL_VERSION = 3301
 NEW_MINIMUM_PROTOCOL_VERSION = 3301
 BLOCK_MAX_SIZE = 32000000

p2pool/networks/bitcoincash_testnet.py

@@ -17,7 +17,7 @@
 ANNOUNCE_CHANNEL = '#p2pool-alt'
 VERSION_CHECK = lambda v: None if 100000 <= v else 'Bitcoin version too old. Upgrade to 0.11.2 or newer!' # not a bug. BIP65 support is ensured by SOFTFORKS_REQUIRED
 VERSION_WARNING = lambda v: None
-SOFTFORKS_REQUIRED = set(['bip65', 'csv'])
+SOFTFORKS_REQUIRED = set()
 MINIMUM_PROTOCOL_VERSION = 3301
 NEW_MINIMUM_PROTOCOL_VERSION = 3301
 BLOCK_MAX_SIZE = 32000000

p2pool/node.py

@@ -62,6 +62,7 @@ def handle_share_hashes(self, hashes, peer):
             )
         except:
             log.err(None, 'in handle_share_hashes:')
+            peer.badPeerHappened(30)
         else:
             self.handle_shares([(share, []) for share in shares], peer)
 
@@ -126,6 +127,7 @@ def download_shares():
                     continue
                 except:
                     log.err(None, 'in download_shares:')
+                    peer.badPeerHappened(30)
                     continue
 
                 if not shares:

p2pool/p2p.py

@@ -5,7 +5,7 @@
 import sys
 import time
 
-from twisted.internet import defer, protocol, reactor
+from twisted.internet import defer, protocol, reactor, task
 from twisted.python import failure, log
 
 import p2pool
@@ -93,11 +93,16 @@ def packetReceived(self, command, payload2):
             print 'Peer %s:%i misbehaving, will drop and ban. Reason:' % self.addr, e.message
             self.badPeerHappened()
 
-    def badPeerHappened(self):
+    def badPeerHappened(self, bantime=3600):
         print "Bad peer banned:", self.addr
         self.disconnect()
         if self.transport.getPeer().host != '127.0.0.1': # never ban localhost
-            self.node.bans[self.transport.getPeer().host] = time.time() + 60*60
+            host = self.transport.getPeer().host
+            if not host in self.node.banscores:
+                self.node.banscores[host] = 1
+            else:
+                self.node.banscores[host] += 1
+            self.node.bans[self.transport.getPeer().host] = time.time() + bantime * self.node.banscores[host]**2
 
     def _timeout(self):
         self.timeout_delayed = None
@@ -688,6 +693,7 @@ def __init__(self, best_share_hash_func, port, net, addr_store={}, connect_addrs
         self.nonce = random.randrange(2**64)
         self.peers = {}
         self.bans = {} # address -> end_time
+        self.banscores = {} # address -> how naughty this peer has been recently
         self.clientfactory = ClientFactory(self, desired_outgoing_conns, max_outgoing_attempts)
         self.serverfactory = ServerFactory(self, max_incoming_conns)
         self.running = False
@@ -703,6 +709,14 @@ def start(self):
         self.running = True
 
         self._stop_thinking = deferral.run_repeatedly(self._think)
+        self.forgiveness_task = task.LoopingCall(self.forgive_transgressions)
+        self.forgiveness_task.start(3600.)
+
+    def forgive_transgressions(self):
+        for host in self.banscores:
+            self.banscore[host] -= 1
+            if self.banscore[host] < 0:
+                self.banscore[host] = 0
 
     def _think(self):
         try:

p2pool/web.py

@@ -402,8 +402,8 @@ def get_share_data(share_hash_str):
         'current_payout': graph.DataStreamDescription(dataview_descriptions),
         'current_payouts': graph.DataStreamDescription(dataview_descriptions, multivalues=True),
         'peers': graph.DataStreamDescription(dataview_descriptions, multivalues=True, default_func=graph.make_multivalue_migrator(dict(incoming='incoming_peers', outgoing='outgoing_peers'))),
-        'miner_hash_rates': graph.DataStreamDescription(dataview_descriptions, is_gauge=False, multivalues=True),
-        'miner_dead_hash_rates': graph.DataStreamDescription(dataview_descriptions, is_gauge=False, multivalues=True),
+        'miner_hash_rates': graph.DataStreamDescription(dataview_descriptions, is_gauge=False, multivalues=True, multivalues_keep=10000),
+        'miner_dead_hash_rates': graph.DataStreamDescription(dataview_descriptions, is_gauge=False, multivalues=True, multivalues_keep=10000),
         'desired_version_rates': graph.DataStreamDescription(dataview_descriptions, multivalues=True,
             multivalue_undefined_means_0=True),
         'traffic_rate': graph.DataStreamDescription(dataview_descriptions, is_gauge=False, multivalues=True),