Merge pull request #21 from keyko-io/fix/file-download

Fixed an issue with ownerDownload where the files were not being saved

src/main/java/io/keyko/nevermined/api/impl/AssetsImpl.java

@@ -138,6 +138,10 @@ public Boolean ownerDownload(DID did, int serviceIndex, String basePath) throws
         return neverminedManager.downloadAssetByOwner(did, serviceIndex, basePath);
     }
 
+    public Boolean ownerDownload(DID did, int serviceIndex, String basePath, int fileIndex) throws ServiceException, ConsumeServiceException {
+        return neverminedManager.downloadAssetByOwner(did, serviceIndex, basePath, fileIndex);
+    }
+
     @Override
     public Flowable<OrderResult> order(DID did, int serviceDefinitionId) throws OrderException {
         return neverminedManager.purchaseAssetFlowable(did, serviceDefinitionId);

src/main/java/io/keyko/nevermined/external/GatewayService.java

@@ -279,6 +279,34 @@ public static void downloadToPath(String serviceEndpoint, String consumerAddress
 
     }
 
+    /**
+     * Calls a Gateway endpoint to download an asset
+     * @param serviceEndpoint the service endpoint
+     * @param consumerAddress the address of the consumer
+     * @param did asset decentralized identifier
+     * @param index index position of the file in the DDO
+     * @param signature User signature of the service agreement
+     * @param destinationPath path where the downloaded asset will be stored
+     * @param startRange  the start of the bytes range
+     * @param endRange  the end of the bytes range
+     * @param isRangeRequest indicates if is a range request
+     * @throws IOException Exception during the download process
+     */
+    public static void downloadToPathByOwner(String serviceEndpoint, String consumerAddress, String did, int index,
+                                             String signature, String destinationPath, Boolean isRangeRequest,
+                                             Integer startRange, Integer endRange) throws IOException {
+
+
+        InputStream inputStream = downloadUrlByOwner(serviceEndpoint, consumerAddress, did, index, signature,
+                isRangeRequest, startRange, endRange);
+        ReadableByteChannel readableByteChannel = Channels.newChannel(inputStream);
+        FileOutputStream fileOutputStream = FileUtils.openOutputStream(new File(destinationPath));
+
+        fileOutputStream.getChannel()
+                .transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
+
+    }
+
     /**
      * Calls a Gateway endpoint to download an asset
      * @param serviceEndpoint the service endpoint

src/main/java/io/keyko/nevermined/manager/NeverminedManager.java

@@ -552,19 +552,35 @@ else if  (service.type.equals(Service.ServiceTypes.COMPUTE.toString()))
 
     }
 
-
     /**
      * Purchases an Asset represented by a DID. It implies to initialize a Service Agreement between publisher and consumer
      *
-     * @param did  the did
+     * @param did          the did
      * @param serviceIndex the service index in the ddo to download
-     * @param basePath path where we want to download the asset files
+     * @param basePath     path where we want to download the asset files
      * @return true if the asset was purchased successfully, if not false
-     * @throws ServiceException ServiceException
+     * @throws ServiceException        ServiceException
      * @throws ConsumeServiceException ConsumeServiceException
      */
     public boolean downloadAssetByOwner(DID did, int serviceIndex, String basePath)
             throws ServiceException, ConsumeServiceException {
+        return downloadAssetByOwner(did, serviceIndex, basePath, 0);
+    }
+
+
+    /**
+     * Purchases an Asset represented by a DID. It implies to initialize a Service Agreement between publisher and consumer
+     *
+     * @param did          the did
+     * @param serviceIndex the service index in the ddo to download
+     * @param basePath     path where we want to download the asset files
+     * @param fileIndex    index of the file inside the files definition in metadata
+     * @return true if the asset was purchased successfully, if not false
+     * @throws ServiceException        ServiceException
+     * @throws ConsumeServiceException ConsumeServiceException
+     */
+    public boolean downloadAssetByOwner(DID did, int serviceIndex, String basePath, int fileIndex)
+            throws ServiceException, ConsumeServiceException {
 
         Service service;
         DDO ddo;
@@ -603,14 +619,10 @@ public boolean downloadAssetByOwner(DID did, int serviceIndex, String basePath)
             for (AssetMetadata.File file : files) {
 
                 try {
-                    String destinationPath = basePath + File.separator + did.getHash() + File.separator;
-                    if (null != file.name && !file.name.isEmpty())
-                        destinationPath = destinationPath + file.name;
-                    else
-                        destinationPath = destinationPath + file.index;
-
-                    GatewayService.downloadUrlByOwner(serviceEndpoint, checkConsumerAddress,
-                            did.getDid(), file.index, signature, false, 0, 0);
+                    String destinationPath = buildDestinationPath(basePath, did, fileIndex, file);
+                    GatewayService.downloadToPathByOwner(serviceEndpoint, checkConsumerAddress,
+                            did.getDid(), file.index, signature, destinationPath, false,
+                            0, 0);
 
                 } catch (IOException e) {
                     String msg = "Error downloading asset by owner with DID " + did.getDid();
@@ -960,12 +972,7 @@ public boolean access(String serviceAgreementId, DID did, int serviceIndex, int
 
             // For each url we call to consume Gateway endpoint that requires consumerAddress, serviceAgreementId and url as a parameters
             try {
-                String destinationPath = basePath + File.separator + did.getHash() + File.separator;
-                if (null != file.name && !file.name.isEmpty())
-                    destinationPath = destinationPath + file.name;
-                else
-                    destinationPath = destinationPath + fileIndex;
-
+                String destinationPath = buildDestinationPath(basePath, did, fileIndex, file);
                 GatewayService.downloadToPath(serviceEndpoint, checkConsumerAddress, agreementId, did.getDid(),
                         fileIndex, signature, destinationPath, false, 0, 0);
 
@@ -981,6 +988,25 @@ public boolean access(String serviceAgreementId, DID did, int serviceIndex, int
         return true;
     }
 
+    /**
+     * Constructs the final path where the file should be downloaded
+     *
+     * @param basePath  the path where the asset will be downloaded
+     * @param did       the did
+     * @param fileIndex index of the file inside the files definition in metadata
+     * @param file      the asset metadata file
+     * @return the destination path to download the file
+     */
+    private String buildDestinationPath(String basePath, DID did, int fileIndex, AssetMetadata.File file) {
+        String destinationPath = basePath + File.separator + "datafile." + did.getHash() + "." + fileIndex + File.separator;
+        if (null != file.name && !file.name.isEmpty())
+            destinationPath = destinationPath + file.name;
+        else
+            destinationPath = destinationPath + fileIndex;
+
+        return destinationPath;
+    }
+
     public String generateSignature(String message) throws IOException, CipherException {
         return EncodingHelper.signatureToString(
                 EthereumHelper.signMessage(message, getKeeperService().getCredentials()));

src/main/java/io/keyko/nevermined/models/asset/AssetMetadata.java

@@ -1,9 +1,7 @@
 package io.keyko.nevermined.models.asset;
 
 
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.annotation.*;
 import io.keyko.common.helpers.CryptoHelper;
 import io.keyko.nevermined.models.AbstractModel;
 import io.keyko.nevermined.models.service.Service;
@@ -13,7 +11,6 @@
 
 import java.util.stream.Collectors;
 
-
 @JsonIgnoreProperties(ignoreUnknown = true)
 @JsonPropertyOrder(alphabetic = true)
 public class AssetMetadata extends AbstractModel {
@@ -82,6 +79,12 @@ public static class File {
         @JsonProperty
         public String resourceId;
 
+        @JsonSetter("url")
+        public void setUrl(String url) {
+            this.name = new java.io.File(url).getName();
+            this.url = url;
+        }
+
 
         public File() {
         }

src/test/java/io/keyko/nevermined/api/AssetsApiIT.java

@@ -22,11 +22,10 @@
 import io.reactivex.Flowable;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
+import org.junit.*;
+import org.junit.rules.TemporaryFolder;
 
+import java.io.File;
 import java.io.InputStream;
 import java.math.BigInteger;
 import java.nio.file.Files;
@@ -63,6 +62,9 @@ public class AssetsApiIT {
 
     private static Config config;
 
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
     @BeforeClass
     public static void setUp() throws Exception {
 
@@ -271,13 +273,19 @@ public void ownerDownload() throws Exception {
         DDO ddo = neverminedAPI.getAssetsAPI().create(metadataBase, providerConfig);
         DID did = new DID(ddo.id);
 
+        final Boolean downloaded = neverminedAPI.getAssetsAPI().ownerDownload(did, Service.DEFAULT_ACCESS_INDEX,
+                tempFolder.getRoot().getAbsolutePath());
 
-        final Boolean downloaded = neverminedAPI.getAssetsAPI().ownerDownload(did, Service.DEFAULT_ACCESS_INDEX, "/tmp");
+        String expectedDestinationPath = tempFolder.getRoot().getAbsolutePath() + File.separator + "datafile."
+                + did.getHash() + ".0" + File.separator + "shs_dataset_test.txt";
         assertTrue(downloaded);
+        assertTrue(new File(expectedDestinationPath).exists());
+
 
         Boolean shouldntBeDownloaded = false;
         try {
-            shouldntBeDownloaded = neverminedAPIConsumer.getAssetsAPI().ownerDownload(did, Service.DEFAULT_ACCESS_INDEX, "/tmp");
+            shouldntBeDownloaded = neverminedAPIConsumer.getAssetsAPI().ownerDownload(did, Service.DEFAULT_ACCESS_INDEX,
+                    tempFolder.getRoot().getAbsolutePath());
         } catch (ServiceException | ConsumeServiceException e) {
         }
         assertFalse(shouldntBeDownloaded);

src/test/java/io/keyko/nevermined/models/DDOTest.java

@@ -120,7 +120,18 @@ public void checkDateFormat() throws Exception {
         assertTrue(AbstractModel.DATE_FORMAT.format(newDDO.updated).startsWith("20"));
     }
 
+    @Test
+    public void checkFiles() throws Exception {
+        DDO ddo = DDO.fromJSON(new TypeReference<DDO>() {}, DDO_JSON_CONTENT);
+        String ddoJson = ddo.toJson();
+        ddo = DDO.fromJSON(new TypeReference<DDO>() {
+        }, ddoJson);
 
+        assertEquals(1, ddo.services.get(0).attributes.main.files.size());
+        assertEquals("https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt",
+                ddo.services.get(0).attributes.main.files.get(0).url);
+        assertEquals("shs_dataset_test.txt", ddo.services.get(0).attributes.main.files.get(0).name);
+    }
 
     @Test
     public void testWorkflow() throws Exception {

src/test/java/io/keyko/nevermined/models/DdoIT.java

@@ -82,7 +82,7 @@ public void testAccessDatasetMetadata() throws Exception {
         DDO ddo = DDO.fromJSON(new TypeReference<DDO>() {}, ACCESS_DATASET_EXAMPLE_CONTENT);
         SortedMap<String, String> checksums = ddo.generateChecksums();
         assertEquals(4, checksums.size());
-        assertEquals("e8cf331dc623f4c470698fea8eec786f5e77850e9b5119bbe1eef3f1fc3fa8a5", checksums.get("0"));
+        assertEquals("7daf1d183628a0e203d75497a6b8ac081a8c64427e35c0dfe753753e11bdf3ce", checksums.get("0"));
         assertEquals("e4ac07c241b4a113e1ad6ef046b8fdf61c5e2c6e6a1745d352f8f30ffa84d704", checksums.get("1"));
 
         DID did = DID.builder(ddo.toJson(ddo.proof.checksum));
@@ -99,7 +99,7 @@ public void testAccessAlgorithmMetadata() throws Exception {
         DDO ddo = DDO.fromJSON(new TypeReference<DDO>() {}, ACCESS_ALGORITHM_EXAMPLE_CONTENT);
         SortedMap<String, String> checksums = ddo.generateChecksums();
         assertEquals(4, checksums.size());
-        assertEquals("1d9d75356a502bcead9da02cc2408562d169d2cdaec4de5398b19f1deea67fc2", checksums.get("0"));
+        assertEquals("ed1f3099c8ced660662c90ab0f1d1cc437b6968f9875bbdf6a6120be445dc638", checksums.get("0"));
 
         DID did = DID.builder(ddo.toJson(ddo.proof.checksum));
         log.debug("Did generated from checksums: " + did.did);

src/test/resources/application.conf

@@ -24,20 +24,20 @@ provider.address="0x068ed00cf0441e4829d9784fcbe7b9e26d4bd8d0"
 
 # Contracts addresses
 
-contract.SignCondition.address="0x208aca4B0316C9996F085cbD57E01c11Bc0E7cb1"
-contract.HashLockCondition.address="0x0123aFa89725EeE631aaa73b243e68589E70f2AE"
-contract.LockRewardCondition.address="0x521E247a99BB7F6a624212148dad3c7a8A232a08"
-contract.AccessSecretStoreCondition.address="0x8C8e79260ce44755a4Fff564dC4c414084629fbB"
-contract.EscrowReward.address="0xC87a8c97E70bc082A2B4b64818511d96cEdc3e7d"
-contract.EscrowAccessSecretStoreTemplate.address="0x90988436A75c308fBfc52D27fca65d28D5A3b919"
-contract.NeverminedToken.address="0x4714b1Bb67EcF87Df7A977917f87f46eA49eA631"
-contract.Dispenser.address="0xc6dbC055422dD656e36D8ad66d4BcDC436C11DDE"
-contract.DIDRegistry.address="0x06768E184cB8f0fD729F6B96b552599487De3D70"
-contract.ConditionStoreManager.address="0x2AaC920AA4D10b80db9ed0E4EC04A3ff612F2bc6"
-contract.TemplateStoreManager.address="0x06F712732acfC7Be52997C94D12A4313C83d6bB7"
-contract.AgreementStoreManager.address="0xC244586E1675E4A98755bC1CE6F6Dead2CD55742"
-contract.ComputeExecutionCondition.address="0x176EC9B938d0ecE9aE1252EceB84c6bEf9a68576"
-contract.EscrowComputeExecutionTemplate.address="0xeB0E4AEa7BEb29AD1018185A3e097b4588eA902C"
+contract.SignCondition.address="0x41A1833BeaB93f5e2bd83F45A2F42E3799b04C6F"
+contract.HashLockCondition.address="0x08Ec8709A3fe3cD4b9A160Eab3be62ccF0754410"
+contract.LockRewardCondition.address="0x3cAD3F3bF241757808790EA7A64A41949fCA8F6c"
+contract.AccessSecretStoreCondition.address="0x970cE99E03d799F42db329baf74A8863d5D8380b"
+contract.EscrowReward.address="0xFC5Aa469b708A069FE1027679453B9b89b303D6A"
+contract.EscrowAccessSecretStoreTemplate.address="0x020F9DD5558255d32D53cf55b80aD98DdCFF8ce8"
+contract.NeverminedToken.address="0x7B14E41EEc3802f522637674a1De3844AC54Bd46"
+contract.Dispenser.address="0xc0579b701EBBcff139340e8e49b4fFdE7F4536BC"
+contract.DIDRegistry.address="0x7829d944551D51F99a52a608C6aDfFda01544342"
+contract.ConditionStoreManager.address="0x517f5218Edff5E13bE22914Fca51D696ac87748E"
+contract.TemplateStoreManager.address="0xC80776ad641BDA91BE8BC2ea100B1cBf708A8CD1"
+contract.AgreementStoreManager.address="0x0a3708Ea0FEF4e98E08B62e5cB60b2Df493Eeab3"
+contract.ComputeExecutionCondition.address="0x6F8D287a2898d7A2aff723CaD4D56aD9E94CB405"
+contract.EscrowComputeExecutionTemplate.address="0xB7BA5d9a934e4b6c581af7bC76bf2cd274f5B7be"
 
 
 consume.basePath = "/tmp"