Integration of compute case

.github/workflows/release-github.yml

@@ -11,9 +11,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Release documentation
-        run: |
-          make github
       - name: Release
         uses: softprops/action-gh-release@v1
         if: startsWith(github.ref, 'refs/tags/')

examples/buy_asset.py

@@ -40,7 +40,7 @@ def buy_asset():
     # make nevermined instance
     nevermined = Nevermined()
     Diagnostics.verify_contracts()
-    acc = get_account(0)
+    acc = get_account(1)
     if not acc:
         acc = ([acc for acc in nevermined.accounts.list() if acc.password] or nevermined.accounts.list())[0]
 
@@ -51,7 +51,7 @@ def buy_asset():
         ddo = nevermined.assets.resolve(did)
         logging.info(f'using ddo: {did}')
     else:
-        ddo = nevermined.assets.create(example_metadata.metadata, acc, providers=[], authorization_type='SecretStore')
+        ddo = nevermined.assets.create(example_metadata.metadata, acc, providers=[], authorization_type='SecretStore', use_secret_store=True)
         assert ddo is not None, f'Registering asset on-chain failed.'
         did = ddo.did
         logging.info(f'registered ddo: {did}')
@@ -84,7 +84,7 @@ def buy_asset():
                      f'{keeper.did_registry.is_did_provider(ddo.asset_id, provider)}')
 
     nevermined_cons = Nevermined()
-    consumer_account = get_account(1)
+    consumer_account = get_account(0)
 
     # sign agreement using the registered asset did above
     service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS)

examples/buy_asset_new_flow.py

@@ -0,0 +1,133 @@
+import logging
+import os
+import time
+
+from contracts_lib_py.diagnostics import Diagnostics
+from contracts_lib_py.web3_provider import Web3Provider
+from common_utils_py.agreements.service_agreement import ServiceAgreement
+from common_utils_py.agreements.service_types import ServiceTypes
+
+from examples import ExampleConfig, example_metadata
+from nevermined_sdk_py import ConfigProvider, Nevermined
+from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper
+from contracts_lib_py.utils import get_account
+
+
+def _log_event(event_name):
+    def _process_event(event):
+        print(f'Received event {event_name}: {event}')
+
+    return _process_event
+
+
+if 'TEST_NILE' in os.environ and os.environ['TEST_NILE'] == '1':
+    ASYNC_DELAY = 5  # seconds
+else:
+    ASYNC_DELAY = 1  # seconds
+
+
+def buy_asset_new_flow():
+    """
+    Requires all Nevermined services running.
+
+    """
+    ConfigProvider.set_config(ExampleConfig.get_config())
+    config = ConfigProvider.get_config()
+    providers = {
+        'duero': '0xfEF2d5e1670342b9EF22eeeDcb287EC526B48095',
+        'nile': '0x4aaab179035dc57b35e2ce066919048686f82972'
+    }
+    # make nevermined instance
+    nevermined = Nevermined()
+    Diagnostics.verify_contracts()
+    acc = get_account(0)
+    if not acc:
+        acc = ([acc for acc in nevermined.accounts.list() if acc.password] or nevermined.accounts.list())[0]
+
+    keeper = Keeper.get_instance()
+    # Register ddo
+    did = ''  # 'did:nv:7648596b60f74301ae1ef9baa5d637255d517ff362434754a3779e1de4c8219b'
+    if did:
+        ddo = nevermined.assets.resolve(did)
+        logging.info(f'using ddo: {did}')
+    else:
+        ddo = nevermined.assets.create(example_metadata.metadata, acc, providers=[])
+        assert ddo is not None, f'Registering asset on-chain failed.'
+        did = ddo.did
+        logging.info(f'registered ddo: {did}')
+        # nevermined here will be used only to publish the asset. Handling the asset by the publisher
+        # will be performed by the Gateway server running locally
+        test_net = os.environ.get('TEST_NET', '')
+        if test_net.startswith('nile'):
+            provider = keeper.did_registry.to_checksum_address(providers['nile'])
+        elif test_net.startswith('duero'):
+            provider = keeper.did_registry.to_checksum_address(providers['duero'])
+        else:
+            provider = '0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0'
+
+        # Wait for did registry event
+        event = keeper.did_registry.subscribe_to_event(
+            keeper.did_registry.DID_REGISTRY_EVENT_NAME,
+            30,
+            event_filter={
+                '_did': Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
+                '_owner': acc.address},
+            wait=True
+        )
+        if not event:
+            logging.warning(f'Failed to get the did registry event for asset with did {did}.')
+        assert keeper.did_registry.get_block_number_updated(ddo.asset_id) > 0, \
+            f'There is an issue in registering asset {did} on-chain.'
+
+        keeper.did_registry.add_provider(ddo.asset_id, provider, acc)
+        logging.info(f'is {provider} set as did provider: '
+                     f'{keeper.did_registry.is_did_provider(ddo.asset_id, provider)}')
+
+    nevermined_cons = Nevermined()
+    consumer_account = get_account(1)
+
+    # sign agreement using the registered asset did above
+    service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS)
+    # This will send the order request to Gateway which in turn will execute the agreement on-chain
+    nevermined_cons.accounts.request_tokens(consumer_account, 10)
+    sa = ServiceAgreement.from_service_dict(service.as_dictionary())
+    agreement_id = ''
+    if not agreement_id:
+        agreement_id = nevermined_cons.assets.order_direct(
+            did, sa.index, consumer_account, consumer_account)
+
+    nevermined.assets.access(
+        agreement_id,
+        did,
+        sa.index,
+        consumer_account,
+        config.downloads_path,
+        index=0)
+    logging.info('Success buying asset.')
+
+    event = keeper.access_secret_store_condition.subscribe_condition_fulfilled(
+        agreement_id, 15, None, (), wait=True
+    )
+    logging.info(f'Got access event {event}')
+    i = 0
+    while nevermined.agreements.is_access_granted(
+            agreement_id, did, consumer_account.address) is not True and i < 15:
+        time.sleep(1)
+        i += 1
+
+    assert nevermined.agreements.is_access_granted(agreement_id, did, consumer_account.address)
+
+    event = keeper.escrow_reward_condition.subscribe_condition_fulfilled(
+        agreement_id,
+        30,
+        None,
+        (),
+        wait=True
+    )
+    assert event, 'no event for EscrowReward.Fulfilled'
+    logging.info(f'got EscrowReward.FULFILLED event: {event}')
+    logging.info('Done buy asset.')
+
+
+if __name__ == '__main__':
+    buy_asset_new_flow()

examples/compute_example.py

@@ -0,0 +1,104 @@
+import logging
+import time
+
+from common_utils_py.agreements.service_agreement import ServiceAgreement
+from common_utils_py.agreements.service_types import ServiceTypes
+from contracts_lib_py.utils import get_account
+
+from examples import example_metadata, ExampleConfig
+from nevermined_sdk_py import ConfigProvider, Nevermined
+from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper
+
+
+def _log_event(event_name):
+    def _process_event(event):
+        print(f'Received event {event_name}: {event}')
+
+    return _process_event
+
+
+def compute_example():
+    ConfigProvider.set_config(ExampleConfig.get_config())
+    config = ConfigProvider.get_config()
+
+    # make nevermined instance
+    nevermined = Nevermined()
+    acc = get_account(0)
+    if not acc:
+        acc = \
+        ([acc for acc in nevermined.accounts.list() if acc.password] or nevermined.accounts.list())[
+            0]
+    keeper = Keeper.get_instance()
+    provider = '0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0'
+
+    ddo = nevermined.assets.create(example_metadata.metadata, acc, providers=[provider],
+                                   authorization_type='SecretStore')
+    assert ddo is not None, f'Registering asset on-chain failed.'
+    did = ddo.did
+    logging.info(f'registered ddo: {did}')
+
+    compute_ddo = nevermined.assets.create(example_metadata.compute_ddo, acc, providers=[provider],
+                                           authorization_type='SecretStore')
+    assert compute_ddo is not None, f'Registering asset on-chain failed.'
+    compute_did = compute_ddo.did
+    logging.info(f'registered ddo: {compute_did}')
+
+    algo_ddo = nevermined.assets.create(example_metadata.algo_metadata, acc, providers=[provider],
+                                        authorization_type='SecretStore')
+    assert algo_ddo is not None, f'Registering algorithm on-chain failed.'
+    algo_did = algo_ddo.did
+    logging.info(f'registered ddo: {algo_did}')
+
+    workflow_metadata = example_metadata.workflow_ddo
+    workflow_metadata['main']['workflow']['stages'][0]['input'][0]['id'] = did
+    workflow_metadata['main']['workflow']['stages'][0]['input'][1]['id'] = compute_did
+    workflow_metadata['main']['workflow']['stages'][0]['transformation']['id'] = algo_did
+    workflow_ddo = nevermined.assets.create(workflow_metadata, acc, providers=[provider],
+                                            authorization_type='SecretStore')
+    assert workflow_ddo is not None, f'Registering algorithm on-chain failed.'
+    workflow_did = workflow_ddo.did
+    logging.info(f'registered ddo: {workflow_did}')
+
+
+    nevermined_cons = Nevermined()
+    consumer_account = get_account(1)
+
+    service = compute_ddo.get_service(service_type=ServiceTypes.CLOUD_COMPUTE)
+    nevermined_cons.accounts.request_tokens(consumer_account, 10)
+    sa = ServiceAgreement.from_service_dict(service.as_dictionary())
+    agreement_id = ''
+    if not agreement_id:
+        agreement_id = nevermined_cons.assets.order(
+            compute_did, sa.index, consumer_account)
+
+    logging.info('placed order: %s, %s', did, agreement_id)
+
+    event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
+        agreement_id, 60, None, (), wait=True
+    )
+    assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
+    logging.info('Got lock reward event, next: wait for the access condition..')
+
+    event = keeper.compute_execution_condition.subscribe_condition_fulfilled(
+        agreement_id, 15, None, (), wait=True
+    )
+    logging.info(f'Got access event {event}')
+
+    nevermined_cons.assets.execute(agreement_id, compute_did, sa.index, consumer_account,
+                                   workflow_did)
+    logging.info('Success executing workflow.')
+
+    event = keeper.escrow_reward_condition.subscribe_condition_fulfilled(
+        agreement_id,
+        30,
+        None,
+        (),
+        wait=True
+    )
+    assert event, 'no event for EscrowReward.Fulfilled'
+    logging.info(f'got EscrowReward.FULFILLED event: {event}')
+    logging.info('Done buy asset.')
+
+
+if __name__ == '__main__':
+    compute_example()

examples/example_metadata.py

@@ -16,9 +16,92 @@
                 "checksum": str(uuid.uuid4()),
                 "checksumType": "MD5",
                 "contentLength": "12057507",
-                "url": "https://raw.githubusercontent.com/oceanprotocol/barge/master/README.md"
+                "url": "https://raw.githubusercontent.com/keyko-io/nevermined-tools/master/README.md"
             }
         ],
         "type": "dataset"
     }
 }
+
+algo_metadata = {
+        "main": {
+          "author": "John Doe",
+          "checksum": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377",
+          "dateCreated": "2019-02-08T08:13:49Z",
+          "license": "CC-BY",
+          "name": "My workflow",
+          "price": "1",
+          "type": "algorithm",
+          "algorithm": {
+            "language": "scala",
+            "format": "jar",
+            "version": "0.1",
+            "entrypoint": "nv-entrypoint.sh",
+            "requirements": [
+              {
+                "requirement": "java",
+                "version": "1.8"
+              }
+            ]
+          }
+        }
+}
+
+compute_ddo = {
+  "main": {
+    "name": "10 Monkey Species Small",
+    "dateCreated": "2012-02-01T10:55:11Z",
+    "author": "Mario",
+    "license": "CC0: Public Domain",
+    "price": "10",
+    "files": [],
+    "type": "compute"
+  }
+}
+
+workflow_ddo = {
+    "main": {
+    "author": "John Doe",
+    "checksum": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377",
+    "dateCreated": "2019-02-08T08:13:49Z",
+    "datePublished": "2019-05-08T08:13:49Z",
+    "license": "CC-BY",
+    "name": "My workflow",
+    "price": "1",
+    "type": "workflow",
+    "workflow": {
+        "stages": [
+            {
+                "index": 0,
+                "stageType": "Filtering",
+                "requirements": {
+                    "container": {
+                        "image": "openjdk",
+                        "tag": "14-jdl",
+                        "checksum":"sha256:53ad3a03b2fb240b6c494339821e6638cd44c989bcf26ec4d51a6a52f7518c1d"
+                    }
+                },
+                "input": [
+                    {
+                        "index": 0,
+                        "id": "did:nv:12345"
+                    },
+                    {
+                        "index": 1,
+                        "id": "did:nv:67890"
+                    }
+                ],
+                "transformation": {
+                    "id": "did:nv:abcde"
+                },
+                "output": {
+                    "metadataUrl": "https://localhost:5000/api/v1/metadata/assets/ddo/",
+                    "secretStoreUrl": "http://localhost:12001",
+                    "accessProxyUrl": "https://localhost:8030/api/v1/gateway/",
+                    "metadata": {}
+                }
+            }
+        ]
+    }
+}
+}