R19101 (#1252)

* prepare rel 19.10.1
* bump version; update changelog
* remove "ethereum" pkg dep for xbr
* partial pypy xbr support

.travis-deploy.sh

@@ -51,10 +51,13 @@ ls -la ./dist
 echo 'uploading package ..'
 aws s3 cp --recursive ./dist s3://${AWS_S3_BUCKET_NAME}/wheels
 
+pip install https://github.com/crossbario/py-eth-sig-utils/archive/master.zip#egg=py-eth-sig-utils
+pip install https://github.com/crossbario/txaio/archive/master.zip#egg=txaio
+pip install https://github.com/crossbario/zlmdb/archive/master.zip#egg=zlmdb
+pip install https://github.com/crossbario/autobahn-python/archive/master.zip#egg=autobahn[twisted,serialization,encryption,xbr]
+
 # tell crossbar-builder about this new wheel push
 # get 'wamp' command, always with latest autobahn master
-
-pip install https://github.com/crossbario/autobahn-python/archive/master.zip#egg=autobahn[twisted,serialization,encryption]
 wamp --max-failures 3 --authid wheel_pusher --url ws://office2dmz.crossbario.com:8008/ --realm webhook call builder.wheel_pushed --keyword name autobahn-python --keyword publish true
 
 # build and deploy latest docs

Makefile

@@ -15,6 +15,7 @@ install:
 	-pip uninstall -y pytest_asyncio # remove the broken shit
 	-pip uninstall -y pytest_cov # remove the broken shit
 	# enforce use of bundled libsodium
+	-pip install --force-reinstall git+https://github.com/crossbario/py-eth-sig-utils.git@master#egg=py-eth-sig-utils
 	AUTOBAHN_USE_NVX=1 SODIUM_INSTALL=bundled pip install -e .[all]
 
 build:

autobahn/_version.py

@@ -24,4 +24,4 @@
 #
 ###############################################################################
 
-__version__ = u'19.9.3'
+__version__ = u'19.10.1'

autobahn/twisted/xbr.py

@@ -24,10 +24,13 @@
 #
 ###############################################################################
 
+import sys
+
 try:
-    from autobahn import xbr
+    from autobahn import xbr  # noqa
     HAS_XBR = True
-except ImportError:
+except ImportError as e:
+    sys.stderr.write('WARNING: could not import autobahn.xbr - {}\n'.format(e))
     HAS_XBR = False
 
 
@@ -42,11 +45,13 @@
 
     from autobahn.xbr._util import hl
     from autobahn.xbr._interfaces import IProvider, ISeller, IConsumer, IBuyer
+    from autobahn.xbr import _seller, _buyer, _blockchain
 
-    class SimpleBlockchain(xbr.SimpleBlockchain):
+    class SimpleBlockchain(_blockchain.SimpleBlockchain):
+        log = txaio.make_logger()
         backgroundCaller = deferToThread
 
-    class KeySeries(xbr.KeySeries):
+    class KeySeries(_seller.KeySeries):
         log = txaio.make_logger()
 
         def __init__(self, api_id, price, interval, on_rotate=None):
@@ -83,15 +88,16 @@ def stop(self):
 
             return self._started
 
-    class SimpleSeller(xbr.SimpleSeller):
+    class SimpleSeller(_seller.SimpleSeller):
         """
         Simple XBR seller component. This component can be used by a XBR seller delegate to
         handle the automated selling of data encryption keys to the XBR market maker.
         """
-        xbr.SimpleSeller.KeySeries = KeySeries
+        log = txaio.make_logger()
+        KeySeries = KeySeries
 
-    class SimpleBuyer(xbr.SimpleBuyer):
-        pass
+    class SimpleBuyer(_buyer.SimpleBuyer):
+        log = txaio.make_logger()
 
     ISeller.register(SimpleSeller)
     IProvider.register(SimpleSeller)

autobahn/xbr/__init__.py

@@ -82,9 +82,6 @@ def process_type(type_str):
 
 from autobahn.xbr._abi import XBR_TOKEN_ABI, XBR_NETWORK_ABI, XBR_CHANNEL_ABI
 from autobahn.xbr._abi import XBR_DEBUG_TOKEN_ADDR, XBR_DEBUG_NETWORK_ADDR
-from autobahn.xbr._blockchain import SimpleBlockchain
-from autobahn.xbr._buyer import SimpleBuyer
-from autobahn.xbr._seller import SimpleSeller, KeySeries
 from autobahn.xbr._interfaces import IMarketMaker, IProvider, IConsumer, ISeller, IBuyer
 from autobahn.xbr._util import sign_eip712_data, recover_eip712_signer, pack_uint256, unpack_uint256
 
@@ -107,7 +104,11 @@ def setProvider(_w3):
     """
     global xbrtoken
     global xbrnetwork
+
+    print('Provider set - xbrtoken={}'.format(XBR_DEBUG_TOKEN_ADDR))
     xbrtoken = _w3.eth.contract(address=XBR_DEBUG_TOKEN_ADDR, abi=XBR_TOKEN_ABI)
+
+    print('Provider set - xbrnetwork={}'.format(XBR_DEBUG_NETWORK_ADDR))
     xbrnetwork = _w3.eth.contract(address=XBR_DEBUG_NETWORK_ADDR, abi=XBR_NETWORK_ABI)
 
 
@@ -181,9 +182,4 @@ class ActorType(object):
     'IConsumer',
     'ISeller',
     'IBuyer',
-
-    'SimpleBlockchain',
-    'SimpleBuyer',
-    'SimpleSeller',
-    'KeySeries',
 )

autobahn/xbr/_abi.py

@@ -26,32 +26,36 @@
 
 import os
 import json
+import binascii
 import pkg_resources
 
 os.environ['ETH_HASH_BACKEND'] = 'pycryptodome'
 
-import web3
-
-
 # from eth_hash.backends.pycryptodome import keccak256  # noqa
 # print('Using eth_hash backend {}'.format(keccak256))
-
+import web3
 
 XBR_TOKEN_FN = pkg_resources.resource_filename('autobahn', 'xbr/contracts/XBRToken.json')
 XBR_NETWORK_FN = pkg_resources.resource_filename('autobahn', 'xbr/contracts/XBRNetwork.json')
 XBR_CHANNEL_FN = pkg_resources.resource_filename('autobahn', 'xbr/contracts/XBRChannel.json')
 
 
 if 'XBR_DEBUG_TOKEN_ADDR' in os.environ:
-    XBR_DEBUG_TOKEN_ADDR = web3.Web3.toChecksumAddress(os.environ['XBR_DEBUG_TOKEN_ADDR'])
+    _token_adr = os.environ['XBR_DEBUG_TOKEN_ADDR']
+    _token_adr = binascii.a2b_hex(_token_adr[2:])
+    _token_adr = web3.Web3.toChecksumAddress(_token_adr)
+    XBR_DEBUG_TOKEN_ADDR = _token_adr
 else:
-    XBR_DEBUG_TOKEN_ADDR = '0x0'
+    XBR_DEBUG_TOKEN_ADDR = '0x0000000000000000000000000000000000000000'
     print('WARNING: The XBR smart contracts are not yet deployed to public networks. Please set XBR_DEBUG_TOKEN_ADDR manually.')
 
 if 'XBR_DEBUG_NETWORK_ADDR' in os.environ:
-    XBR_DEBUG_NETWORK_ADDR = web3.Web3.toChecksumAddress(os.environ['XBR_DEBUG_NETWORK_ADDR'])
+    _netw_adr = os.environ['XBR_DEBUG_NETWORK_ADDR']
+    _netw_adr = binascii.a2b_hex(_netw_adr[2:])
+    _netw_adr = web3.Web3.toChecksumAddress(_netw_adr)
+    XBR_DEBUG_NETWORK_ADDR = _netw_adr
 else:
-    XBR_DEBUG_NETWORK_ADDR = '0x0'
+    XBR_DEBUG_NETWORK_ADDR = '0x0000000000000000000000000000000000000000'
     print('WARNING: The XBR smart contracts are not yet deployed to public networks. Please set XBR_DEBUG_NETWORK_ADDR manually.')
 
 

autobahn/xbr/_buyer.py

@@ -40,9 +40,9 @@
 from autobahn.wamp.protocol import ApplicationSession
 from ._util import unpack_uint256, pack_uint256
 
-import web3
 import eth_keys
-from eth_account import Account
+# import web3
+# from eth_account import Account
 
 from ._util import hl, sign_eip712_data, recover_eip712_signer
 
@@ -110,13 +110,17 @@ def __init__(self, market_maker_adr, buyer_key, max_price):
         self._pkey = eth_keys.keys.PrivateKey(buyer_key)
 
         # buyer delegate ethereum private account from raw private key
-        self._acct = Account.privateKeyToAccount(self._pkey)
+        # FIXME
+        # self._acct = Account.privateKeyToAccount(self._pkey)
+        self._acct = None
 
         # buyer delegate ethereum account canonical address
         self._addr = self._pkey.public_key.to_canonical_address()
 
         # buyer delegate ethereum account canonical checksummed address
-        self._caddr = web3.Web3.toChecksumAddress(self._addr)
+        # FIXME
+        # self._caddr = web3.Web3.toChecksumAddress(self._addr)
+        self._caddr = None
 
         # ephemeral data consumer key
         self._receive_key = nacl.public.PrivateKey.generate()
@@ -167,7 +171,7 @@ async def start(self, session, consumer_id):
         self._running = True
 
         self.log.info('Start buying from consumer delegate address {address} (public key 0x{public_key}..)',
-                      address=hl('0x' + self._acct.address),
+                      address=hl(self._caddr),
                       public_key=binascii.b2a_hex(self._pkey.public_key[:10]).decode())
 
         try:
@@ -284,7 +288,12 @@ async def unwrap(self, key_id, serializer, ciphertext):
         channel_adr = bytes(self._channel['channel'])
 
         # if we don't have the key, buy it!
-        if key_id not in self._keys:
+        if key_id in self._keys:
+            self.log.info('Key {key_id} already in key store (or currently being bought).',
+                          key_id=hl(uuid.UUID(bytes=key_id)))
+        else:
+            self.log.info('Key {key_id} not yet in key store - buying key ..', key_id=hl(uuid.UUID(bytes=key_id)))
+
             # mark the key as currently being bought already (the location of code here is multi-entrant)
             self._keys[key_id] = False
 
@@ -294,6 +303,9 @@ async def unwrap(self, key_id, serializer, ciphertext):
             # set price we pay set to the (current) quoted price
             amount = unpack_uint256(quote['price'])
 
+            self.log.info('Key {key_id} has current price quote {amount}',
+                          key_id=hl(uuid.UUID(bytes=key_id)), amount=hl(int(amount / 10**18)))
+
             if amount > self._max_price:
                 raise ApplicationError('xbr.error.max_price_exceeded',
                                        '{}.unwrap() - key {} needed cannot be bought: price {} exceeds maximum price of {}'.format(self.__class__.__name__, uuid.UUID(bytes=key_id), int(amount / 10 ** 18), int(self._max_price / 10 ** 18)))
@@ -347,9 +359,10 @@ async def unwrap(self, key_id, serializer, ciphertext):
 
             buyer_pubkey = self._receive_key.public_key.encode(encoder=nacl.encoding.RawEncoder)
             channel_seq = self._seq + 1
+            is_final = False
 
             # XBRSIG[1/8]: compute EIP712 typed data signature
-            signature = sign_eip712_data(self._pkey_raw, channel_adr, channel_seq, balance, is_final=False)
+            signature = sign_eip712_data(self._pkey_raw, channel_adr, channel_seq, balance, is_final=is_final)
 
             # persist 1st phase of the transaction locally
             self._save_transaction_phase1(channel_adr, self._addr, buyer_pubkey, key_id, channel_seq, amount, balance, signature)

autobahn/xbr/_seller.py

@@ -37,12 +37,12 @@
 
 import cbor2
 import eth_keys
-from eth_account import Account
+# from eth_account import Account
 import nacl.secret
 import nacl.utils
 import nacl.public
 import txaio
-import web3
+# import web3
 
 from ._util import hl, recover_eip712_signer, sign_eip712_data
 
@@ -216,13 +216,17 @@ def __init__(self, market_maker_adr, seller_key, provider_id=None):
         self._pkey = eth_keys.keys.PrivateKey(seller_key)
 
         # seller ethereum private account from raw private key
-        self._acct = Account.privateKeyToAccount(self._pkey)
+        # FIXME
+        # self._acct = Account.privateKeyToAccount(self._pkey)
+        self._acct = None
 
         # seller ethereum account canonical address
         self._addr = self._pkey.public_key.to_canonical_address()
 
         # seller ethereum account canonical checksummed address
-        self._caddr = web3.Web3.toChecksumAddress(self._addr)
+        # FIXME
+        # self._caddr = web3.Web3.toChecksumAddress(self._addr)
+        self._caddr = None
 
         # seller provider ID
         self._provider_id = provider_id or str(self._pkey.public_key)
@@ -334,8 +338,9 @@ async def on_rotate(key_series):
                 self.log.warn('Failed to place offer for key! Retrying {retries}/5 ..', retries=retries)
                 await asyncio.sleep(1)
 
-        key_series = SimpleSeller.KeySeries(api_id, price, interval, on_rotate)
+        key_series = self.KeySeries(api_id, price, interval, on_rotate)
         self._keys[api_id] = key_series
+        self.log.info('Created new key series {key_series}', key_series=key_series)
 
         return key_series
 
@@ -354,7 +359,7 @@ async def start(self, session):
         self._session_regs = []
 
         self.log.info('Start selling from seller delegate address {address} (public key 0x{public_key}..)',
-                      address=hl('0x' + self._acct.address),
+                      address=hl(self._caddr),
                       public_key=binascii.b2a_hex(self._pkey.public_key[:10]).decode())
 
         procedure = 'xbr.provider.{}.sell'.format(self._provider_id)

autobahn/xbr/_util.py

@@ -84,6 +84,7 @@ def hl(text, bold=True, color='yellow'):
 
 
 def _create_eip712_data(verifying_adr, channel_adr, channel_seq, balance, is_final):
+    assert type(verifying_adr) == bytes and len(verifying_adr) == 20
     assert type(channel_adr) == bytes and len(channel_adr) == 20
     assert type(channel_seq) == int
     assert type(balance) == int
@@ -156,7 +157,7 @@ def sign_eip712_data(eth_privkey, channel_adr, channel_seq, balance, is_final=Fa
     assert type(balance) == int and balance >= 0
     assert type(is_final) == bool
 
-    verifying_adr = '0x254dffcd3277C0b1660F6d42EFbB754edaBAbC2B'
+    verifying_adr = a2b_hex('0x254dffcd3277C0b1660F6d42EFbB754edaBAbC2B'[2:])
 
     # make a private key object from the raw private key bytes
     # pkey = eth_keys.keys.PrivateKey(eth_privkey)
@@ -168,7 +169,11 @@ def sign_eip712_data(eth_privkey, channel_adr, channel_seq, balance, is_final=Fa
     # create EIP712 typed data object
     data = _create_eip712_data(verifying_adr, channel_adr, channel_seq, balance, is_final)
 
-    signature = signing.v_r_s_to_signature(*signing.sign_typed_data(data, eth_privkey))
+    # FIXME: this fails on PyPy (but ot on CPy!) with
+    #  Unknown format b'%M\xff\xcd2w\xc0\xb1f\x0fmB\xef\xbbuN\xda\xba\xbc+', attempted to normalize to 0x254dffcd3277c0b1660f6d42efbb754edababc2b
+    _args = signing.sign_typed_data(data, eth_privkey)
+
+    signature = signing.v_r_s_to_signature(*_args)
     assert len(signature) == _EIP712_SIG_LEN
 
     return signature
@@ -202,7 +207,7 @@ def recover_eip712_signer(channel_adr, channel_seq, balance, is_final, signature
     assert type(is_final) == bool
     assert type(signature) == bytes and len(signature) == _EIP712_SIG_LEN
 
-    verifying_adr = '0x254dffcd3277C0b1660F6d42EFbB754edaBAbC2B'
+    verifying_adr = a2b_hex('0x254dffcd3277C0b1660F6d42EFbB754edaBAbC2B'[2:])
 
     # recreate EIP712 typed data object
     data = _create_eip712_data(verifying_adr, channel_adr, channel_seq, balance, is_final)

docs/changelog.rst

@@ -5,6 +5,14 @@
 Changelog
 =========
 
+19.10.1
+-------
+
+* new: updated docker image scripts
+* new: add WAMP serializer in use to SessionDetails
+* fix: partial support for xb buyers/sellers in pypy
+* fix: remove dependency on "ethereum" package (part of pypy support)
+
 19.9.3
 ------