Auto merge of #4074 - zancas:3802_py2_to_py3, r=<try>

Migrate tests from py2 to py3

When executed with the `zcash/qa/pull-tester/rpc-tests.sh` script the Python code in the `zcash/qa/rpc-tests` directory, and subdirectories, runs under Python 3.5.3.

I tried to _only_ make changes that were necessary for this goal, and nothing else (like causing `BitcoinTestFramework` to inherit from `unittest.TestCase`, deduplicating repeated code, replacing bespoke components with standard solutions, etc.)

I've begun collecting ideas related to such potential improvements here:

#4076  <- (Note: This "improvement list" is, itself, much in need of improvement.)

contrib/ci-workers/unix.yml

@@ -57,6 +57,13 @@
         msg: "The Python binary at {{ ansible_python.executable }} is version {{ ansible_python_version }}! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary."
       when: ansible_python.version.major != 2 or ansible_python.version.minor != 7
 
+    - name: Install Python3 for rpc-tests
+      raw: test -e /usr/bin/python3 || test -e /usr/local/bin/python3 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python3) || (test -e /usr/bin/dnf && dnf install -qqy python3) || (test -e /usr/sbin/pkg && pkg install -qqy python3)
+      register: output
+      changed_when:
+        - output.stdout != ""
+        - output.stdout != "\r\n"
+
   tasks:
     - name: Get dependencies for distribution
       include_vars: "{{ item }}"
@@ -132,6 +139,15 @@
       become_user: "{{ buildbot_worker_user }}"
       notify: restart buildbot-worker
 
+    - name: Install required Python3 rpc modules
+      pip3:
+        state: latest
+        virtualenv: "~{{ buildbot_worker_user }}/venv"
+        virtualenv_command: "{{ '/usr/local/bin/virtualenv' if ansible_distribution == 'MacOSX' else omit }}"
+      with_items: "{{ rpc_test_modules }}"
+      become_user: "{{ buildbot_worker_user }}"
+      notify: restart buildbot-worker
+
     - name: Create Buildbot worker
       command: >
         ~{{ buildbot_worker_user }}/venv/bin/buildbot-worker create-worker ~/{{ buildbot_worker_name }}

contrib/ci-workers/vars/default.yml

@@ -51,3 +51,5 @@ buildbot_modules:
 rpc_test_modules:
   - pyblake2
   - pyzmq
+  - pyutils
+  - python-qpid-proton

qa/rpc-tests/bip65-cltv-p2p.py

@@ -1,10 +1,10 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or https://www.opensource.org/licenses/mit-license.php .
 #
 
-import sys; assert sys.version_info < (3,), ur"This script does not run under Python 3. Please use Python 2.7.x."
+
 
 from test_framework.test_framework import ComparisonTestFramework
 from test_framework.util import start_nodes
@@ -13,7 +13,7 @@
 from test_framework.comptool import TestInstance, TestManager
 from test_framework.script import CScript, OP_1NEGATE, OP_NOP2, OP_DROP
 from binascii import unhexlify
-import cStringIO
+import io
 
 
 '''
@@ -38,10 +38,10 @@ def setup_network(self):
         self.is_network_split = False
 
     def run_test(self):
-        test = TestManager(self, self.options.tmpdir)
-        test.add_all_connections(self.nodes)
+        test_manager = TestManager(self, self.options.tmpdir)
+        test_manager.add_all_connections(self.nodes)
         NetworkThread().start() # Start up network handling in another thread
-        test.run()
+        test_manager.run()
 
     def create_transaction(self, node, coinbase, to_address, amount):
         from_txid = node.getblock(coinbase)['tx'][0]
@@ -50,7 +50,7 @@ def create_transaction(self, node, coinbase, to_address, amount):
         rawtx = node.createrawtransaction(inputs, outputs)
         signresult = node.signrawtransaction(rawtx)
         tx = CTransaction()
-        f = cStringIO.StringIO(unhexlify(signresult['hex']))
+        f = io.BytesIO(unhexlify(signresult['hex']))
         tx.deserialize(f)
         return tx
 
@@ -66,7 +66,7 @@ def invalidate_transaction(self, tx):
     def get_tests(self):
         self.coinbase_blocks = self.nodes[0].generate(1)
         self.nodes[0].generate(100)
-        self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0)
+        self.tip = int ("0x" + self.nodes[0].getbestblockhash(), 0)
         self.nodeaddress = self.nodes[0].getnewaddress()
 
         '''Check that the rules are enforced.'''

qa/rpc-tests/bipdersig-p2p.py

@@ -1,10 +1,10 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or https://www.opensource.org/licenses/mit-license.php .
 #
 
-import sys; assert sys.version_info < (3,), ur"This script does not run under Python 3. Please use Python 2.7.x."
+
 
 from test_framework.test_framework import ComparisonTestFramework
 from test_framework.util import start_nodes
@@ -13,7 +13,7 @@
 from test_framework.comptool import TestInstance, TestManager
 from test_framework.script import CScript
 from binascii import unhexlify
-import cStringIO
+import io
 
 
 '''
@@ -49,7 +49,7 @@ def create_transaction(self, node, coinbase, to_address, amount):
         rawtx = node.createrawtransaction(inputs, outputs)
         signresult = node.signrawtransaction(rawtx)
         tx = CTransaction()
-        f = cStringIO.StringIO(unhexlify(signresult['hex']))
+        f = io.BytesIO(unhexlify(signresult['hex']))
         tx.deserialize(f)
         return tx
 
@@ -73,7 +73,7 @@ def invalidate_transaction(self, tx):
     def get_tests(self):
         self.coinbase_blocks = self.nodes[0].generate(1)
         self.nodes[0].generate(100)
-        self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0)
+        self.tip = int ("0x" + self.nodes[0].getbestblockhash(), 0)
         self.nodeaddress = self.nodes[0].getnewaddress()
 
         '''Check that the rules are enforced.'''

qa/rpc-tests/blockchain.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2014 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or https://www.opensource.org/licenses/mit-license.php .
@@ -8,7 +8,7 @@
 # rpc/blockchain.cpp.
 #
 
-import sys; assert sys.version_info < (3,), ur"This script does not run under Python 3. Please use Python 2.7.x."
+
 
 import decimal
 
@@ -29,7 +29,7 @@ class BlockchainTest(BitcoinTestFramework):
     """
 
     def setup_chain(self):
-        print("Initializing test directory " + self.options.tmpdir)
+        print(("Initializing test directory " + self.options.tmpdir))
         initialize_chain(self.options.tmpdir)
 
     def setup_network(self, split=False):
@@ -42,13 +42,13 @@ def run_test(self):
         node = self.nodes[0]
         res = node.gettxoutsetinfo()
 
-        assert_equal(res[u'total_amount'], decimal.Decimal('2181.25000000')) # 150*12.5 + 49*6.25
-        assert_equal(res[u'transactions'], 200)
-        assert_equal(res[u'height'], 200)
-        assert_equal(res[u'txouts'], 349) # 150*2 + 49
-        assert_equal(res[u'bytes_serialized'], 14951), # 32*199 + 48*90 + 49*60 + 27*49
-        assert_equal(len(res[u'bestblock']), 64)
-        assert_equal(len(res[u'hash_serialized']), 64)
+        assert_equal(res['total_amount'], decimal.Decimal('2181.25000000')) # 150*12.5 + 49*6.25
+        assert_equal(res['transactions'], 200)
+        assert_equal(res['height'], 200)
+        assert_equal(res['txouts'], 349) # 150*2 + 49
+        assert_equal(res['bytes_serialized'], 14951), # 32*199 + 48*90 + 49*60 + 27*49
+        assert_equal(len(res['bestblock']), 64)
+        assert_equal(len(res['hash_serialized']), 64)
 
 
 if __name__ == '__main__':