From 7e76c4a44ab33c2f55c65b90fa865190d698b7c2 Mon Sep 17 00:00:00 2001 From: Levente Pap Date: Mon, 7 Oct 2019 18:09:36 +0200 Subject: [PATCH 1/3] Support for LocalPow via ccurl.interface.py To perform PoW (attach_to_tangle) locally. BaseAdatper class has a new attribute, local_pow, that is set to False by default. By declaring a new api instance with local_pow=True, attach_to_tangle calls will be handled by an interface to the ccurl library. The change is backward compatible, since if local_pow argument is omitted, behavior defaults back to 'normal'. Feature can be disabled/enabled dynamically using the api's set_local_pow() function. --- README.rst | 19 +++ examples/local_pow.py | 39 +++++ iota/adapter/__init__.py | 10 ++ iota/api.py | 33 ++++- iota/commands/core/attach_to_tangle.py | 12 ++ setup.py | 1 + test/local_pow_test.py | 193 +++++++++++++++++++++++++ 7 files changed, 302 insertions(+), 5 deletions(-) create mode 100644 examples/local_pow.py create mode 100644 test/local_pow_test.py diff --git a/README.rst b/README.rst index cbbe1d5e..eaf037c0 100644 --- a/README.rst +++ b/README.rst @@ -42,6 +42,23 @@ To install this extension, use the following command:: pip install pyota[ccurl] +Optional Local Pow +================== +To perform Proof-of-Work locally without relying on any node, +you can install an extension module called `PyOTA-PoW`_ . + +Specifiy the ``local_pow=True`` argument when creating an +api instance, that will redirect all ``attach_to_tangle`` +API calls to an interface function in the ``pow`` package. +Returns the same as a node would for the API call. + +To install this extension, use the following command:: + + pip install pyota[pow] + +Alternativley you can take a look on the repository +`Ccurl.interface.py`_ to install Pyota-PoW. +Follow the steps depicted in the repo's README file. Installing from Source ====================== @@ -101,3 +118,5 @@ can also build the documentation locally: .. _ReadTheDocs: https://pyota.readthedocs.io/ .. _official API: https://docs.iota.org/docs/node-software/0.1/iri/references/api-reference .. _tox: https://tox.readthedocs.io/ +.. _Ccurl.interface.py: https://github.com/lzpap/ccurl.interface.py +.. _PyOTA-PoW: https://pypi.org/project/PyOTA-PoW/ diff --git a/examples/local_pow.py b/examples/local_pow.py new file mode 100644 index 00000000..afea778c --- /dev/null +++ b/examples/local_pow.py @@ -0,0 +1,39 @@ +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import iota +from pprint import pprint +from datetime import datetime + +# Generate a random seed. +myseed = iota.crypto.types.Seed.random() +# Get an address generator. +addres_generator = iota.crypto.addresses.AddressGenerator(myseed) + +# Instantiate API. Note the `local_pow=True` argument. +# This will cause PyOTA to do proof-of-work locally, +# by using the pyota-pow extension package. (if installed) +# Find it at: https://pypi.org/project/PyOTA-PoW/ +api = iota.Iota("https://nodes.thetangle.org:443",myseed,local_pow=True) + +# Generate two addresses +addys = addres_generator.get_addresses(1, count=2) +pprint('Generated addresses are:') +pprint(addys) + +# Preparing transactions +pt = iota.ProposedTransaction(address = iota.Address(addys[0]), + message = iota.TryteString.from_unicode('Tx1: The PoW for this transaction was done by Pyota-Pow.'), + tag = iota.Tag(b'LOCALATTACHINTERFACE99999'), # Up to 27 trytes + value = 0) + +pt2 = iota.ProposedTransaction(address = iota.Address(addys[1]), + message = iota.TryteString.from_unicode('Tx2: The PoW for this transaction was done by Pyota-Pow.'), + tag = iota.Tag(b'LOCALATTACHINTERFACE99999'), # Up to 27 trytes + value = 0) + +# `send_transfer` will take care of the rest +response = api.send_transfer([pt,pt2]) + +pprint('Broadcasted bundle:') +pprint(response['bundle'].as_json_compatible()) \ No newline at end of file diff --git a/iota/adapter/__init__.py b/iota/adapter/__init__.py index 61ae1bae..48457fb8 100644 --- a/iota/adapter/__init__.py +++ b/iota/adapter/__init__.py @@ -157,6 +157,7 @@ def __init__(self): super(BaseAdapter, self).__init__() self._logger = None # type: Logger + self.local_pow = False # type: boolean @abstract_method def get_uri(self): @@ -209,6 +210,15 @@ def _log(self, level, message, context=None): if self._logger: self._logger.log(level, message, extra={'context': context or {}}) + def set_local_pow(self, local_pow): + # type: (bool) -> None + """ + Sets the local_pow attribute of the adapter. If it is true, + attach_to_tangle command calls external interface to perform + pow, instead of sending the request to a node. + By default, it is set to false. + """ + self.local_pow = local_pow class HttpAdapter(BaseAdapter): """ diff --git a/iota/api.py b/iota/api.py index 9cfeb26f..0a0c064f 100644 --- a/iota/api.py +++ b/iota/api.py @@ -67,8 +67,8 @@ class StrictIota(object): """ commands = discover_commands('iota.commands.core') - def __init__(self, adapter, testnet=False): - # type: (AdapterSpec, bool) -> None + def __init__(self, adapter, testnet=False, local_pow=False): + # type: (AdapterSpec, bool, bool) -> None """ :param adapter: URI string or BaseAdapter instance. @@ -82,6 +82,17 @@ def __init__(self, adapter, testnet=False): adapter = resolve_adapter(adapter) self.adapter = adapter # type: BaseAdapter + # Note that the `local_pow` parameter is passed to adapter, + # the api class has no notion about it. The reason being, + # that this parameter is used in `AttachToTangeCommand` calls, + # that is called from various api calls (`attach_to_tangle`, + # `send_trytes` or `send_transfer`). Inside `AttachToTangeCommand`, + # we no longer have access to the attributes of the API class, therefore + # `local_pow` needs to be associated with the adapter. + # Logically, `local_pow` will decide if the api call does pow + # via pyota-pow extension, or sends the request to a node. + # But technically, the parameter belongs to the adapter. + self.adapter.set_local_pow(local_pow) self.testnet = testnet def __getattr__(self, command): @@ -139,6 +150,18 @@ def create_command(self, command): """ return CustomCommand(self.adapter, command) + def set_local_pow(self, local_pow): + # type: (bool) -> None + """ + Sets the local_pow attribute of the adapter of the api instance. + If it is true, attach_to_tangle command calls external interface + to perform pow, instead of sending the request to a node. + By default, it is set to false. + This particular method is needed if one wants to change + local_pow behavior dynamically. + """ + self.adapter.set_local_pow(local_pow) + @property def default_min_weight_magnitude(self): # type: () -> int @@ -527,8 +550,8 @@ class Iota(StrictIota): """ commands = discover_commands('iota.commands.extended') - def __init__(self, adapter, seed=None, testnet=False): - # type: (AdapterSpec, Optional[TrytesCompatible], bool) -> None + def __init__(self, adapter, seed=None, testnet=False, local_pow=False): + # type: (AdapterSpec, Optional[TrytesCompatible], bool, bool) -> None """ :param seed: Seed used to generate new addresses. @@ -537,7 +560,7 @@ def __init__(self, adapter, seed=None, testnet=False): .. note:: This value is never transferred to the node/network. """ - super(Iota, self).__init__(adapter, testnet) + super(Iota, self).__init__(adapter, testnet, local_pow) self.seed = Seed(seed) if seed else Seed.random() self.helpers = Helpers(self) diff --git a/iota/commands/core/attach_to_tangle.py b/iota/commands/core/attach_to_tangle.py index daefec5e..8c2644fc 100644 --- a/iota/commands/core/attach_to_tangle.py +++ b/iota/commands/core/attach_to_tangle.py @@ -27,6 +27,18 @@ def get_request_filter(self): def get_response_filter(self): return AttachToTangleResponseFilter() + def _execute(self, request): + if self.adapter.local_pow is True: + from pow import ccurl_interface + powed_trytes = ccurl_interface.attach_to_tangle( + request['trytes'], + request['branchTransaction'], + request['trunkTransaction'], + request['minWeightMagnitude'] + ) + return {'trytes': powed_trytes} + else: + return super(FilterCommand, self)._execute(request) class AttachToTangleRequestFilter(RequestFilter): def __init__(self): diff --git a/setup.py b/setup.py index 168e101a..a7ebc667 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ extras_require={ 'ccurl': ['pyota-ccurl'], + 'pow': ['pyota-pow >= 1.0.1'], 'docs-builder': ['sphinx', 'sphinx_rtd_theme'], # tox is able to run the tests in parallel since version 3.7 'test-runner': ['tox >= 3.7'] + tests_require, diff --git a/test/local_pow_test.py b/test/local_pow_test.py new file mode 100644 index 00000000..3b60b7dd --- /dev/null +++ b/test/local_pow_test.py @@ -0,0 +1,193 @@ +# coding=utf-8 +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from iota import Iota, TryteString, TransactionHash, TransactionTrytes, \ + HttpAdapter, MockAdapter +from iota.adapter.sandbox import SandboxAdapter +from iota.adapter.wrappers import RoutingWrapper +from unittest import TestCase +import sys + +from unittest.mock import MagicMock, patch + +# Load mocked package on import from pow pkg. +# Therefore we can test without having to install it. +sys.modules['pow'] = MagicMock() + +class LocalPowTestCase(TestCase): + """ + Unit tests for `local_pow` feature using `pow` package + from `ccurl.inteface.py`. + """ + # We are only interested in if the ccurl interface is called. + # Don't care about the values, and there is no actual PoW + # calculation in these tests. Testing the functional correctness + # of the PoW calculation is done in iotaledger/ccurl.interface.py. + # Filters are thoroughly tested in `attach_to_tangle_test.py`. + def setUp(self): + """ + These values will be used in the tests. + """ + # Will be padded to transaction length by TransactionTrytes() + self.trytes1 ='CCLDVADBEACCWCTCEAZBCDFCE' + # Will be padded to transaction length by TransactionTrytes() + self.trytes2 ='CGDEAHDFDPCBDGDPCRCHDXCCDBDEAKDPC' + # Will be padded to hash length by TransactionHash() + self.trunk ='EWSQPV9AGXUQRYAZIUONVBXFNWRWIGVCFT' + self.branch ='W9VELHQPPERYSG9ZLLAHQKDLJQBKYYZOS' + self.mwm = 14 + + # Create real objects so that we pass the filters + self.bundle = [TransactionTrytes(self.trytes1), TransactionTrytes(self.trytes2)] + # ccurl_bundle is only needed to differentiate between response + # from mocked pow and MockAdapter in some test cases. + self.ccurl_bundle = [TransactionTrytes(self.trytes1)] + self.trunk = TransactionHash(self.trunk) + self.branch = TransactionHash(self.branch) + + def test_backward_compatibility(self): + """ + Test that the local_pow feature is backward compatible. + That is, if `local_pow` argument is omitted, it takes no + effect and the pow extension package is not called. + """ + with patch('pow.ccurl_interface.attach_to_tangle', + MagicMock(return_value=self.ccurl_bundle)) as mocked_ccurl: + self.adapter = MockAdapter() + self.adapter.seed_response('attachToTangle',{ + 'trytes': self.bundle, + }) + # No `local_pow` argument is passed to the api! + api = Iota(self.adapter) + result = api.attach_to_tangle( + self.trunk, + self.branch, + self.bundle, + self.mwm) + # Ccurl interface was not called + self.assertFalse(mocked_ccurl.called) + # Result is the one returned by MockAdapter + self.assertEqual(result['trytes'], self.bundle) + # And not by mocked pow pkg + self.assertNotEqual(result['trytes'], self.ccurl_bundle) + + def test_http_adapter(self): + """ + Test if local_pow feature works with HttpAdapter. + """ + # Note that we need correct return value to pass the + # repsonse filter. + with patch('pow.ccurl_interface.attach_to_tangle', + MagicMock(return_value=self.bundle)) as mocked_ccurl: + api = Iota(HttpAdapter('http://localhost:14265/'),local_pow=True) + result = api.attach_to_tangle( + self.trunk, + self.branch, + self.bundle, + self.mwm) + self.assertTrue(mocked_ccurl.called) + self.assertEqual(result['trytes'], self.bundle) + + def test_mock_adapter(self): + """ + Test if local_pow feature works with MockAdapter. + """ + # Note that we need correct return value to pass the + # repsonse filter. + with patch('pow.ccurl_interface.attach_to_tangle', + MagicMock(return_value=self.bundle)) as mocked_ccurl: + api = Iota(MockAdapter(),local_pow=True) + result = api.attach_to_tangle( + self.trunk, + self.branch, + self.bundle, + self.mwm) + self.assertTrue(mocked_ccurl.called) + self.assertEqual(result['trytes'], self.bundle) + + def test_sandbox_adapter(self): + """ + Test if local_pow feature works with SandboxAdapter. + """ + # Note that we need correct return value to pass the + # repsonse filter. + with patch('pow.ccurl_interface.attach_to_tangle', + MagicMock(return_value=self.bundle)) as mocked_ccurl: + api = Iota(SandboxAdapter('https://sandbox.iota:14265/api/v1/', auth_token=None), + local_pow=True) + result = api.attach_to_tangle( + self.trunk, + self.branch, + self.bundle, + self.mwm) + self.assertTrue(mocked_ccurl.called) + self.assertEqual(result['trytes'], self.bundle) + + def test_routing_wrapper(self): + """ + Test if local_pow feature works with RoutingWrapper. + """ + # Note that we need correct return value to pass the + # repsonse filter. + with patch('pow.ccurl_interface.attach_to_tangle', + MagicMock(return_value=self.bundle)) as mocked_ccurl: + # We are trying to redirect `attach_to_tangle` calls to localhost + # with a RoutingWrapper. However, if local_pow=true, the pow + # request will not reach the adapter, but will be directed to + # ccurl interface. + api = Iota(RoutingWrapper('http://12.34.56.78:14265') + .add_route('attachToTangle', 'http://localhost:14265'), + local_pow=True) + result = api.attach_to_tangle( + self.trunk, + self.branch, + self.bundle, + self.mwm) + self.assertTrue(mocked_ccurl.called) + self.assertEqual(result['trytes'], self.bundle) + + def test_set_local_pow(self): + """ + Test if local_pow can be enabled/disabled dynamically. + """ + with patch('pow.ccurl_interface.attach_to_tangle', + MagicMock(return_value=self.ccurl_bundle)) as mocked_ccurl: + self.adapter = MockAdapter() + self.adapter.seed_response('attachToTangle',{ + 'trytes': self.bundle, + }) + # First, we enable local_pow + api = Iota(self.adapter, local_pow=True) + result = api.attach_to_tangle( + self.trunk, + self.branch, + self.bundle, + self.mwm) + # Ccurl was called + self.assertTrue(mocked_ccurl.called) + # Result comes from ccurl + self.assertEqual(result['trytes'], self.ccurl_bundle) + + # Reset mock, this clears the called attribute + mocked_ccurl.reset_mock() + + # Disable local_pow + api.set_local_pow(local_pow=False) + # Try again + result = api.attach_to_tangle( + self.trunk, + self.branch, + self.bundle, + self.mwm) + # Ccurl interface was not called + self.assertFalse(mocked_ccurl.called) + # Result is the one returned by MockAdapter + self.assertEqual(result['trytes'], self.bundle) + # And not by mocked pow pkg + self.assertNotEqual(result['trytes'], self.ccurl_bundle) + + + + + From 5ac4aeb3ee1748efa1103e6ddd7d07c52045718b Mon Sep 17 00:00:00 2001 From: Levente Pap Date: Wed, 16 Oct 2019 16:16:08 +0200 Subject: [PATCH 2/3] Support Python 2.7 in local_pow_test.py --- test/local_pow_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/local_pow_test.py b/test/local_pow_test.py index 3b60b7dd..c8625e3d 100644 --- a/test/local_pow_test.py +++ b/test/local_pow_test.py @@ -8,8 +8,12 @@ from iota.adapter.wrappers import RoutingWrapper from unittest import TestCase import sys +from six import PY2 -from unittest.mock import MagicMock, patch +if PY2: + from mock import MagicMock, patch +else: + from unittest.mock import MagicMock, patch # Load mocked package on import from pow pkg. # Therefore we can test without having to install it. From f80f433e8d6dc28751dc7a48c0fedad6879c4c60 Mon Sep 17 00:00:00 2001 From: Levente Pap Date: Thu, 17 Oct 2019 11:51:28 +0200 Subject: [PATCH 3/3] Update docs and format fixes --- README.rst | 5 ++--- docs/adapters.rst | 16 ++++++++++++++++ docs/getting_started.rst | 18 +++++++++++++++++- examples/local_pow.py | 1 - test/local_pow_test.py | 15 +++++---------- 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/README.rst b/README.rst index eaf037c0..e143fd21 100644 --- a/README.rst +++ b/README.rst @@ -44,13 +44,12 @@ To install this extension, use the following command:: Optional Local Pow ================== -To perform Proof-of-Work locally without relying on any node, +To perform proof-of-work locally without relying on a node, you can install an extension module called `PyOTA-PoW`_ . Specifiy the ``local_pow=True`` argument when creating an api instance, that will redirect all ``attach_to_tangle`` API calls to an interface function in the ``pow`` package. -Returns the same as a node would for the API call. To install this extension, use the following command:: @@ -118,5 +117,5 @@ can also build the documentation locally: .. _ReadTheDocs: https://pyota.readthedocs.io/ .. _official API: https://docs.iota.org/docs/node-software/0.1/iri/references/api-reference .. _tox: https://tox.readthedocs.io/ -.. _Ccurl.interface.py: https://github.com/lzpap/ccurl.interface.py +.. _Ccurl.interface.py: https://github.com/iotaledger/ccurl.interface.py .. _PyOTA-PoW: https://pypi.org/project/PyOTA-PoW/ diff --git a/docs/adapters.rst b/docs/adapters.rst index be52b471..22ee2ade 100644 --- a/docs/adapters.rst +++ b/docs/adapters.rst @@ -213,6 +213,22 @@ depending on the command name. For example, you could use this wrapper to direct all PoW requests to a local node, while sending the other requests to a light wallet node. +.. note:: + + A common use case for ``RoutingWrapper`` is to perform proof-of-work on + a specific (local) node, but let all other requests go to another node. + Take care when you use ``RoutingWrapper`` adapter and ``local_pow`` + parameter together in an API instance, because the behavior might not + be obvious. + + ``local_pow`` tells the API to perform proof-of-work (``attach_to_tangle``) + without relying on an actual node. It does this by calling an extension + package `PyOTA-PoW `_ that does the + job. In PyOTA, this means the request doesn't reach the adapter, it + is redirected before. + As a consequence, ``local_pow`` has precedence over the route that is + defined in ``RoutingWrapper``. + ``RoutingWrapper`` must be initialized with a default URI/adapter. This is the adapter that will be used for any command that doesn't have a route associated with it. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index da254bcf..0602b179 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -6,7 +6,7 @@ Install PyOTA using `pip`: .. code-block:: bash - pip install pyota[ccurl] + pip install pyota[ccurl,pow] .. note:: @@ -15,6 +15,21 @@ Install PyOTA using `pip`: This extension boosts the performance of certain crypto operations significantly (speedups of 60x are common). +.. note:: + + The ``[pow]`` extra installs the optional `PyOTA-PoW extension`_. + + This extension makes it possible to perform proof-of-work + (api call ``attach_to_tangle``) locally, without relying on an iota node. + Use the ``local_pow`` parameter at api instantiation: + + .. code:: + + api = Iota('https://nodes.thetangle.org:443', local_pow=True) + + Or the ``set_local_pow`` method of the api class to dynamically enable/disable + the local proof-of-work feature. + Getting Started =============== In order to interact with the IOTA network, you will need access to a node. @@ -78,6 +93,7 @@ your API requests so that they contain the necessary authentication metadata. .. _forum: https://forum.iota.org/ .. _official api: https://docs.iota.org/docs/node-software/0.1/iri/references/api-reference .. _pyota-ccurl extension: https://pypi.python.org/pypi/PyOTA-CCurl +.. _pyota-pow extension: https://pypi.org/project/PyOTA-PoW/ .. _run your own node.: http://iotasupport.com/headlessnode.shtml .. _slack: http://slack.iota.org/ .. _use a light wallet node.: http://iotasupport.com/lightwallet.shtml diff --git a/examples/local_pow.py b/examples/local_pow.py index afea778c..1421018e 100644 --- a/examples/local_pow.py +++ b/examples/local_pow.py @@ -3,7 +3,6 @@ import iota from pprint import pprint -from datetime import datetime # Generate a random seed. myseed = iota.crypto.types.Seed.random() diff --git a/test/local_pow_test.py b/test/local_pow_test.py index c8625e3d..ab494785 100644 --- a/test/local_pow_test.py +++ b/test/local_pow_test.py @@ -81,7 +81,7 @@ def test_http_adapter(self): Test if local_pow feature works with HttpAdapter. """ # Note that we need correct return value to pass the - # repsonse filter. + # response filter. with patch('pow.ccurl_interface.attach_to_tangle', MagicMock(return_value=self.bundle)) as mocked_ccurl: api = Iota(HttpAdapter('http://localhost:14265/'),local_pow=True) @@ -98,7 +98,7 @@ def test_mock_adapter(self): Test if local_pow feature works with MockAdapter. """ # Note that we need correct return value to pass the - # repsonse filter. + # response filter. with patch('pow.ccurl_interface.attach_to_tangle', MagicMock(return_value=self.bundle)) as mocked_ccurl: api = Iota(MockAdapter(),local_pow=True) @@ -115,7 +115,7 @@ def test_sandbox_adapter(self): Test if local_pow feature works with SandboxAdapter. """ # Note that we need correct return value to pass the - # repsonse filter. + # response filter. with patch('pow.ccurl_interface.attach_to_tangle', MagicMock(return_value=self.bundle)) as mocked_ccurl: api = Iota(SandboxAdapter('https://sandbox.iota:14265/api/v1/', auth_token=None), @@ -133,7 +133,7 @@ def test_routing_wrapper(self): Test if local_pow feature works with RoutingWrapper. """ # Note that we need correct return value to pass the - # repsonse filter. + # response filter. with patch('pow.ccurl_interface.attach_to_tangle', MagicMock(return_value=self.bundle)) as mocked_ccurl: # We are trying to redirect `attach_to_tangle` calls to localhost @@ -189,9 +189,4 @@ def test_set_local_pow(self): # Result is the one returned by MockAdapter self.assertEqual(result['trytes'], self.bundle) # And not by mocked pow pkg - self.assertNotEqual(result['trytes'], self.ccurl_bundle) - - - - - + self.assertNotEqual(result['trytes'], self.ccurl_bundle) \ No newline at end of file