From fa9fe2f1b1e7f7dabb135d634d919541e198f06a Mon Sep 17 00:00:00 2001 From: enrique Date: Mon, 13 Jul 2020 17:36:44 +0200 Subject: [PATCH] Integration of compute case --- .github/workflows/release-github.yml | 3 - examples/buy_asset.py | 6 +- examples/buy_asset_new_flow.py | 133 +++++++++++++++++++++++++++ examples/compute_example.py | 104 +++++++++++++++++++++ examples/example_metadata.py | 85 ++++++++++++++++- nevermined_sdk_py/gateway/gateway.py | 21 +++++ setup.py | 4 +- tests/nevermined/test_agreements.py | 4 +- 8 files changed, 348 insertions(+), 12 deletions(-) create mode 100644 examples/buy_asset_new_flow.py create mode 100644 examples/compute_example.py diff --git a/.github/workflows/release-github.yml b/.github/workflows/release-github.yml index 1121cf2..794f2e4 100644 --- a/.github/workflows/release-github.yml +++ b/.github/workflows/release-github.yml @@ -11,9 +11,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Release documentation - run: | - make github - name: Release uses: softprops/action-gh-release@v1 if: startsWith(github.ref, 'refs/tags/') diff --git a/examples/buy_asset.py b/examples/buy_asset.py index 60e6171..39f6790 100755 --- a/examples/buy_asset.py +++ b/examples/buy_asset.py @@ -40,7 +40,7 @@ def buy_asset(): # make nevermined instance nevermined = Nevermined() Diagnostics.verify_contracts() - acc = get_account(0) + acc = get_account(1) if not acc: acc = ([acc for acc in nevermined.accounts.list() if acc.password] or nevermined.accounts.list())[0] @@ -51,7 +51,7 @@ def buy_asset(): ddo = nevermined.assets.resolve(did) logging.info(f'using ddo: {did}') else: - ddo = nevermined.assets.create(example_metadata.metadata, acc, providers=[], authorization_type='SecretStore') + ddo = nevermined.assets.create(example_metadata.metadata, acc, providers=[], authorization_type='SecretStore', use_secret_store=True) assert ddo is not None, f'Registering asset on-chain failed.' did = ddo.did logging.info(f'registered ddo: {did}') @@ -84,7 +84,7 @@ def buy_asset(): f'{keeper.did_registry.is_did_provider(ddo.asset_id, provider)}') nevermined_cons = Nevermined() - consumer_account = get_account(1) + consumer_account = get_account(0) # sign agreement using the registered asset did above service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS) diff --git a/examples/buy_asset_new_flow.py b/examples/buy_asset_new_flow.py new file mode 100644 index 0000000..040d772 --- /dev/null +++ b/examples/buy_asset_new_flow.py @@ -0,0 +1,133 @@ +import logging +import os +import time + +from contracts_lib_py.diagnostics import Diagnostics +from contracts_lib_py.web3_provider import Web3Provider +from common_utils_py.agreements.service_agreement import ServiceAgreement +from common_utils_py.agreements.service_types import ServiceTypes + +from examples import ExampleConfig, example_metadata +from nevermined_sdk_py import ConfigProvider, Nevermined +from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper +from contracts_lib_py.utils import get_account + + +def _log_event(event_name): + def _process_event(event): + print(f'Received event {event_name}: {event}') + + return _process_event + + +if 'TEST_NILE' in os.environ and os.environ['TEST_NILE'] == '1': + ASYNC_DELAY = 5 # seconds +else: + ASYNC_DELAY = 1 # seconds + + +def buy_asset_new_flow(): + """ + Requires all Nevermined services running. + + """ + ConfigProvider.set_config(ExampleConfig.get_config()) + config = ConfigProvider.get_config() + providers = { + 'duero': '0xfEF2d5e1670342b9EF22eeeDcb287EC526B48095', + 'nile': '0x4aaab179035dc57b35e2ce066919048686f82972' + } + # make nevermined instance + nevermined = Nevermined() + Diagnostics.verify_contracts() + acc = get_account(0) + if not acc: + acc = ([acc for acc in nevermined.accounts.list() if acc.password] or nevermined.accounts.list())[0] + + keeper = Keeper.get_instance() + # Register ddo + did = '' # 'did:nv:7648596b60f74301ae1ef9baa5d637255d517ff362434754a3779e1de4c8219b' + if did: + ddo = nevermined.assets.resolve(did) + logging.info(f'using ddo: {did}') + else: + ddo = nevermined.assets.create(example_metadata.metadata, acc, providers=[]) + assert ddo is not None, f'Registering asset on-chain failed.' + did = ddo.did + logging.info(f'registered ddo: {did}') + # nevermined here will be used only to publish the asset. Handling the asset by the publisher + # will be performed by the Gateway server running locally + test_net = os.environ.get('TEST_NET', '') + if test_net.startswith('nile'): + provider = keeper.did_registry.to_checksum_address(providers['nile']) + elif test_net.startswith('duero'): + provider = keeper.did_registry.to_checksum_address(providers['duero']) + else: + provider = '0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0' + + # Wait for did registry event + event = keeper.did_registry.subscribe_to_event( + keeper.did_registry.DID_REGISTRY_EVENT_NAME, + 30, + event_filter={ + '_did': Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), + '_owner': acc.address}, + wait=True + ) + if not event: + logging.warning(f'Failed to get the did registry event for asset with did {did}.') + assert keeper.did_registry.get_block_number_updated(ddo.asset_id) > 0, \ + f'There is an issue in registering asset {did} on-chain.' + + keeper.did_registry.add_provider(ddo.asset_id, provider, acc) + logging.info(f'is {provider} set as did provider: ' + f'{keeper.did_registry.is_did_provider(ddo.asset_id, provider)}') + + nevermined_cons = Nevermined() + consumer_account = get_account(1) + + # sign agreement using the registered asset did above + service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS) + # This will send the order request to Gateway which in turn will execute the agreement on-chain + nevermined_cons.accounts.request_tokens(consumer_account, 10) + sa = ServiceAgreement.from_service_dict(service.as_dictionary()) + agreement_id = '' + if not agreement_id: + agreement_id = nevermined_cons.assets.order_direct( + did, sa.index, consumer_account, consumer_account) + + nevermined.assets.access( + agreement_id, + did, + sa.index, + consumer_account, + config.downloads_path, + index=0) + logging.info('Success buying asset.') + + event = keeper.access_secret_store_condition.subscribe_condition_fulfilled( + agreement_id, 15, None, (), wait=True + ) + logging.info(f'Got access event {event}') + i = 0 + while nevermined.agreements.is_access_granted( + agreement_id, did, consumer_account.address) is not True and i < 15: + time.sleep(1) + i += 1 + + assert nevermined.agreements.is_access_granted(agreement_id, did, consumer_account.address) + + event = keeper.escrow_reward_condition.subscribe_condition_fulfilled( + agreement_id, + 30, + None, + (), + wait=True + ) + assert event, 'no event for EscrowReward.Fulfilled' + logging.info(f'got EscrowReward.FULFILLED event: {event}') + logging.info('Done buy asset.') + + +if __name__ == '__main__': + buy_asset_new_flow() diff --git a/examples/compute_example.py b/examples/compute_example.py new file mode 100644 index 0000000..f3d82c3 --- /dev/null +++ b/examples/compute_example.py @@ -0,0 +1,104 @@ +import logging +import time + +from common_utils_py.agreements.service_agreement import ServiceAgreement +from common_utils_py.agreements.service_types import ServiceTypes +from contracts_lib_py.utils import get_account + +from examples import example_metadata, ExampleConfig +from nevermined_sdk_py import ConfigProvider, Nevermined +from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper + + +def _log_event(event_name): + def _process_event(event): + print(f'Received event {event_name}: {event}') + + return _process_event + + +def compute_example(): + ConfigProvider.set_config(ExampleConfig.get_config()) + config = ConfigProvider.get_config() + + # make nevermined instance + nevermined = Nevermined() + acc = get_account(0) + if not acc: + acc = \ + ([acc for acc in nevermined.accounts.list() if acc.password] or nevermined.accounts.list())[ + 0] + keeper = Keeper.get_instance() + provider = '0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0' + + ddo = nevermined.assets.create(example_metadata.metadata, acc, providers=[provider], + authorization_type='SecretStore') + assert ddo is not None, f'Registering asset on-chain failed.' + did = ddo.did + logging.info(f'registered ddo: {did}') + + compute_ddo = nevermined.assets.create(example_metadata.compute_ddo, acc, providers=[provider], + authorization_type='SecretStore') + assert compute_ddo is not None, f'Registering asset on-chain failed.' + compute_did = compute_ddo.did + logging.info(f'registered ddo: {compute_did}') + + algo_ddo = nevermined.assets.create(example_metadata.algo_metadata, acc, providers=[provider], + authorization_type='SecretStore') + assert algo_ddo is not None, f'Registering algorithm on-chain failed.' + algo_did = algo_ddo.did + logging.info(f'registered ddo: {algo_did}') + + workflow_metadata = example_metadata.workflow_ddo + workflow_metadata['main']['workflow']['stages'][0]['input'][0]['id'] = did + workflow_metadata['main']['workflow']['stages'][0]['input'][1]['id'] = compute_did + workflow_metadata['main']['workflow']['stages'][0]['transformation']['id'] = algo_did + workflow_ddo = nevermined.assets.create(workflow_metadata, acc, providers=[provider], + authorization_type='SecretStore') + assert workflow_ddo is not None, f'Registering algorithm on-chain failed.' + workflow_did = workflow_ddo.did + logging.info(f'registered ddo: {workflow_did}') + + + nevermined_cons = Nevermined() + consumer_account = get_account(1) + + service = compute_ddo.get_service(service_type=ServiceTypes.CLOUD_COMPUTE) + nevermined_cons.accounts.request_tokens(consumer_account, 10) + sa = ServiceAgreement.from_service_dict(service.as_dictionary()) + agreement_id = '' + if not agreement_id: + agreement_id = nevermined_cons.assets.order( + compute_did, sa.index, consumer_account) + + logging.info('placed order: %s, %s', did, agreement_id) + + event = keeper.lock_reward_condition.subscribe_condition_fulfilled( + agreement_id, 60, None, (), wait=True + ) + assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs" + logging.info('Got lock reward event, next: wait for the access condition..') + + event = keeper.compute_execution_condition.subscribe_condition_fulfilled( + agreement_id, 15, None, (), wait=True + ) + logging.info(f'Got access event {event}') + + nevermined_cons.assets.execute(agreement_id, compute_did, sa.index, consumer_account, + workflow_did) + logging.info('Success executing workflow.') + + event = keeper.escrow_reward_condition.subscribe_condition_fulfilled( + agreement_id, + 30, + None, + (), + wait=True + ) + assert event, 'no event for EscrowReward.Fulfilled' + logging.info(f'got EscrowReward.FULFILLED event: {event}') + logging.info('Done buy asset.') + + +if __name__ == '__main__': + compute_example() diff --git a/examples/example_metadata.py b/examples/example_metadata.py index 4a57689..970735f 100644 --- a/examples/example_metadata.py +++ b/examples/example_metadata.py @@ -16,9 +16,92 @@ "checksum": str(uuid.uuid4()), "checksumType": "MD5", "contentLength": "12057507", - "url": "https://raw.githubusercontent.com/oceanprotocol/barge/master/README.md" + "url": "https://raw.githubusercontent.com/keyko-io/nevermined-tools/master/README.md" } ], "type": "dataset" } } + +algo_metadata = { + "main": { + "author": "John Doe", + "checksum": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", + "dateCreated": "2019-02-08T08:13:49Z", + "license": "CC-BY", + "name": "My workflow", + "price": "1", + "type": "algorithm", + "algorithm": { + "language": "scala", + "format": "jar", + "version": "0.1", + "entrypoint": "nv-entrypoint.sh", + "requirements": [ + { + "requirement": "java", + "version": "1.8" + } + ] + } + } +} + +compute_ddo = { + "main": { + "name": "10 Monkey Species Small", + "dateCreated": "2012-02-01T10:55:11Z", + "author": "Mario", + "license": "CC0: Public Domain", + "price": "10", + "files": [], + "type": "compute" + } +} + +workflow_ddo = { + "main": { + "author": "John Doe", + "checksum": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", + "dateCreated": "2019-02-08T08:13:49Z", + "datePublished": "2019-05-08T08:13:49Z", + "license": "CC-BY", + "name": "My workflow", + "price": "1", + "type": "workflow", + "workflow": { + "stages": [ + { + "index": 0, + "stageType": "Filtering", + "requirements": { + "container": { + "image": "openjdk", + "tag": "14-jdl", + "checksum":"sha256:53ad3a03b2fb240b6c494339821e6638cd44c989bcf26ec4d51a6a52f7518c1d" + } + }, + "input": [ + { + "index": 0, + "id": "did:nv:12345" + }, + { + "index": 1, + "id": "did:nv:67890" + } + ], + "transformation": { + "id": "did:nv:abcde" + }, + "output": { + "metadataUrl": "https://localhost:5000/api/v1/metadata/assets/ddo/", + "secretStoreUrl": "http://localhost:12001", + "accessProxyUrl": "https://localhost:8030/api/v1/gateway/", + "metadata": {} + } + } + ] + } +} +} diff --git a/nevermined_sdk_py/gateway/gateway.py b/nevermined_sdk_py/gateway/gateway.py index d72648f..6c9043e 100644 --- a/nevermined_sdk_py/gateway/gateway.py +++ b/nevermined_sdk_py/gateway/gateway.py @@ -155,6 +155,23 @@ def execute_service(service_agreement_id, service_endpoint, account, workflow_dd workflow_ddo.did, signature) logger.info(f'invoke execute endpoint with this url: {execute_url}') response = Gateway._http_client.post(execute_url) + return response + + @staticmethod + def execute_compute_service(service_agreement_id, service_endpoint, account, workflow_ddo): + signature = Keeper.get_instance().sign_hash( + add_ethereum_prefix_and_hash_msg(service_agreement_id), + account) + headers = dict({ + 'X-Consumer-Address': account.address, + 'X-Signature': signature, + 'X-DID': workflow_ddo.did + }) + execute_url = Gateway._create_compute_url(service_endpoint, service_agreement_id, account, + workflow_ddo.did, signature) + response = Gateway._http_client.post(execute_url, headers= headers) + return response + @staticmethod def _prepare_consume_payload(did, service_agreement_id, service_index, signature, @@ -321,3 +338,7 @@ def _create_execute_url(service_endpoint, service_agreement_id, account, workflo f'&serviceAgreementId={service_agreement_id}' f'&consumerAddress={account.address}' f'&workflowDID={workflow_did}') + + @staticmethod + def _create_compute_url(service_endpoint, service_agreement_id): + return f'{service_endpoint}/{service_agreement_id}' diff --git a/setup.py b/setup.py index 255d9be..d4c0c4a 100644 --- a/setup.py +++ b/setup.py @@ -20,8 +20,8 @@ 'pyopenssl', 'PyJWT', # not jwt 'PyYAML==4.2b4', - 'common-utils-py==0.2.7', - 'contracts-lib-py==0.2.3', + 'common-utils-py==0.2.8', + 'contracts-lib-py==0.2.4', 'ocean-secret-store-client==0.0.1', 'requests==2.21.0', 'deprecated', diff --git a/tests/nevermined/test_agreements.py b/tests/nevermined/test_agreements.py index 0c39e65..f36ee17 100644 --- a/tests/nevermined/test_agreements.py +++ b/tests/nevermined/test_agreements.py @@ -150,9 +150,7 @@ def test_sign_agreement(publisher_instance, consumer_instance, registered_ddo): # print('All good, files are here: %s' % path) - - -@pytest.mark.skip(reason="Failing some times with actions") +# @pytest.mark.skip(reason="Failing some times with actions") def test_agreement_status(setup_agreements_enviroment, agreements): ( keeper,