Corrected tile filter for products

src/main/java/ro/cs/products/ProductDownloader.java

@@ -73,7 +73,7 @@ public abstract class ProductDownloader<T extends ProductDescriptor> {
     protected boolean shouldDeleteAfterCompression;
     protected DownloadMode downloadMode;
     protected ProductStore store;
-
+    protected double[] averageDownloadSpeed;
     protected Logger.ScopeLogger productLogger;
 
     protected BatchProgressListener batchProgressListener;
@@ -332,6 +332,10 @@ protected Path downloadFile(String remoteUrl, Path file, String authToken) throw
         return downloadFile(remoteUrl, file, this.downloadMode, authToken);
     }
 
+    protected void resetCounter() { this.averageDownloadSpeed = new double[] { 0.0, 0.0 }; }
+
+    protected double getAverageSpeed() { return this.averageDownloadSpeed[0]; }
+
     private Path downloadFile(String remoteUrl, Path file, DownloadMode mode, String authToken) throws IOException {
         HttpURLConnection connection = null;
         try {
@@ -388,6 +392,9 @@ private Path downloadFile(String remoteUrl, Path file, DownloadMode mode, String
                         }
                     }
                     Logger.getRootLogger().debug("End reading from input stream");
+                    this.averageDownloadSpeed[0] = (this.averageDownloadSpeed[0] * this.averageDownloadSpeed[1] +
+                            (remoteFileLength / 1024 / (System.currentTimeMillis() - start) * 1000)) / (this.averageDownloadSpeed[1] + 1);
+                    this.averageDownloadSpeed[1] += 1;
                     getLogger().debug(completeMessage, currentProduct, currentStep, file.getFileName(), (System.currentTimeMillis() - start) / 1000);
                 } finally {
                     if (outputStream != null) outputStream.close();

src/main/java/ro/cs/products/base/AbstractSearch.java

@@ -84,6 +84,8 @@ public void copyFiltersFrom(AbstractSearch<T> anotherSearch) {
 
     public abstract AbstractSearch<T> start(int value);
 
+    public AbstractSearch<T> filter(List<ProductDescriptor> productNames) { return this; }
+
     public void setSensingStart(String sensingStart) {
         this.sensingStart = sensingStart;
     }

src/main/java/ro/cs/products/sentinel2/SentinelProductDownloader.java

@@ -224,17 +224,30 @@ public void copyOptionsFrom(SentinelProductDownloader anotherDownloader) {
     @Override
     protected Path download(SentinelProductDescriptor product) throws IOException {
         String tileId = product.getTileIdentifier();
-        if (tileId != null && this.filteredTiles != null && !this.filteredTiles.contains(tileId)) {
+        if (tileId != null && this.filteredTiles != null && this.filteredTiles.size() > 0 && !this.filteredTiles.contains(tileId)) {
             getLogger().warn("(" + currentProduct + ") The product %s did not contain any tiles from the tile list", product.getName());
             return null;
         }
+        resetCounter();
+        Path result = null;
         switch (store) {
             case AWS:
-                return downloadFromAWS(product);
+                result = downloadFromAWS(product);
+                break;
             case SCIHUB:
             default:
-                return downloadFromSciHub(product);
+                String uuid = product.getId();
+                if (uuid == null || uuid.isEmpty()) {
+                    getLogger().warn("Product identifier for %s missing from record", product.getName());
+                } else {
+                    result = downloadFromSciHub(product);
+                }
+                break;
+        }
+        if (result != null) {
+            getLogger().info("(" + currentProduct + ") Average download speed: %.2f kB/s", getAverageSpeed());
         }
+        return result;
     }
 
     @Override
@@ -301,7 +314,7 @@ private Path downloadFromSciHub(SentinelProductDescriptor productDescriptor) thr
                 List<String> allLines = Files.readAllLines(metadataFile);
                 List<String> metaTileNames = Utilities.filter(allLines, "<Granule" + (Constants.PSD_13.equals(productDescriptor.getVersion()) ? "s" : " "));
                 boolean hasTiles = updateMedatata(metadataFile, allLines) != null;
-                if (hasTiles) {
+                if (!shouldFilterTiles || hasTiles) {
                     Path tilesFolder = Utilities.ensureExists(rootPath.resolve(Constants.FOLDER_GRANULE));
                     Utilities.ensureExists(rootPath.resolve(Constants.FOLDER_AUXDATA));
                     Path dataStripFolder = Utilities.ensureExists(rootPath.resolve(Constants.FOLDER_DATASTRIP));

src/main/java/ro/cs/products/sentinel2/scihub/PreOpsSciHubSearch.java

@@ -65,12 +65,13 @@ public PreOpsSciHubSearch filter(String key, String value) {
         return this;
     }
 
-    public PreOpsSciHubSearch filter(List<String> productNames) {
-        if (productNames != null) {
+    @Override
+    public PreOpsSciHubSearch filter(List<ProductDescriptor> products) {
+        if (products != null) {
             StringBuilder list = new StringBuilder("(");
-            boolean more = productNames.size() > 1;
-            for (String productName : productNames) {
-                list.append(productName);
+            boolean more = products.size() > 1;
+            for (ProductDescriptor product : products) {
+                list.append(product.getName());
                 if (more) {
                     list.append(" OR ");
                 }

src/main/java/ro/cs/products/sentinel2/scihub/SciHubSearch.java

@@ -61,19 +61,21 @@ public SciHubSearch filter(String key, String value) {
         return this;
     }
 
-    public SciHubSearch filter(List<String> productNames) {
-        if (productNames != null) {
+    @Override
+    public SciHubSearch filter(List<ProductDescriptor> products) {
+        if (products != null && products.size() > 0) {
             String list = "(";
-            boolean more = productNames.size() > 1;
-            for (String productName : productNames) {
-                list += productName;
+            boolean more = products.size() > 1;
+            for (ProductDescriptor product : products) {
+                list += product.getName();
                 if (more) {
                     list += " OR ";
                 }
             }
             if (more) {
-                list = list.substring(0, list.length() - 4) + ")";
+                list = list.substring(0, list.length() - 4);
             }
+            list += ")";
             this.filter += " AND " + list;
         }
         return this;

src/main/resources/ro/cs/products/parameters.def

@@ -2,14 +2,14 @@
 
 # Common command line arguments
 n/a ; o ; out ; 1 ; output folder ; ; false ; The folder in which the products will be downloaded
-n/a ; i ; input ; 1 ; input folder ; ; false ; The folder in which the products are to be inspected
+1 ; i ; input ; 1 ; input folder ; ; false ; The folder in which the products are to be inspected
 1 ; a ; area ; n ; lon1,lat1 lon2,lat2 ... ; ' ' ; true ; A closed polygon whose vertices are given in <lon,lat> pairs, space-separated
 1 ; af ; areafile ; 1 ; file path and name ; ; true ; The file containing a closed polygon whose vertices are given in <lon lat> pairs, comma-separated
 1 ; ts ; shapetiles ; 1 ; file path and name ; ; true ; The KML file containing Sentinel-2 tile extents
+1 ; p ; products ; n ; product1 product2 ... ; ' ' ; true ; A list of S2/L8 product names, space-separated
+1 ; pf ; productfile ; 1 ; file path and name ; ; true ; A file containing a list of S2/L8 products, one product name per line
 2 ; t ; tiles ; n ; tileId1 tileId2 ... ; ' ' ; true ; A list of S2 tile IDs, space-separated
 2 ; tf ; tilefile ; 1 ; file path and name ; ; true ; A file containing a list of S2 tile IDs, one tile id per line
-3 ; p ; products ; n ; product1 product2 ... ; ' ' ; true ; A list of S2/L8 product names, space-separated
-3 ; pf ; productfile ; 1 ; file path and name ; ; true ; A file containing a list of S2/L8 products, one product name per line
 n/a ; b ; bands ; n ; band1 band2 ... ; ' ' ; true ; The list of S2/L8 band names, space-separated, to be downloaded. If not specified, all bands are downloaded
 n/a ; start ; startdate ; 1 ; yyyy-MM-dd ; ; true ; Look for products from a specific date (formatted as yyyy-MM-dd). Default is current date -7 days
 n/a ; end ; enddate ; 1 ; yyyy-MM-dd ; ; true ; Look for products up to (and including) a specific date (formatted as yyyy-MM-dd). Default is current date
@@ -22,7 +22,7 @@ n/a ; z ; zip ; 0 ; ; ; true ; Compress the product into a zip archive
 n/a ; d ; delete ; 0 ; ; ; true ; Delete the product files after compression
 n/a ; v ; verbose ; 0 ; ; ; true ; Produce verbose output/logs
 n/a ; q ; query ; 0 ; ; ; true ; Only perform query and return product names
-n.a ; all ; all ; 0 ; ; ; true ; Retrieve all pages of search results
+n/a ; all ; all ; 0 ; ; ; true ; Retrieve all pages of search results
 
 # Proxy parameters
 n/a ; ptype ; proxy.type ; 1 ; http|socks ; ; true ; Proxy type. Supported values are http or socks

src/test/java/SearchTests.java

@@ -2,6 +2,7 @@
 import org.junit.Test;
 import ro.cs.products.Executor;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -18,9 +19,37 @@ public void sciHubSearchTest() {
         Path expectedFile = path.resolve("results.txt");
         try {
             Executor.execute(cmd.split(" "));
-            Assert.assertTrue(Files.exists(expectedFile));
+            Assert.assertTrue(Files.exists(expectedFile) && Files.lines(expectedFile).count() > 0);
+        } catch (Exception e) {
+            Assert.fail(e.getMessage());
+        } finally {
+            try {
+                Files.deleteIfExists(expectedFile);
+            } catch (IOException e) {
+            }
+        }
+    }
+
+    @Test
+    public void sciHubSearchTest2() {
+        Path path = getOutputFolder();
+        String cmd = String.format("--out %s --products %s --tiles %s --startdate %s --enddate %s --user %s --password %s --query",
+                                   path.toString(),
+                                   "S2A_OPER_PRD_MSIL1C_PDMC_20160710T153652_R050_V20160710T091027_20160710T091027.SAFE",
+                                   "35TMM",
+                                   "2017-07-01", "2017-07-14",
+                                   getSciHubUser(), getSciHubPwd());
+        Path expectedFile = path.resolve("results.txt");
+        try {
+            Executor.execute(cmd.split(" "));
+            Assert.assertTrue(Files.exists(expectedFile) && Files.lines(expectedFile).count() > 0);
         } catch (Exception e) {
             Assert.fail(e.getMessage());
+        } finally {
+            try {
+                Files.deleteIfExists(expectedFile);
+            } catch (IOException e) {
+            }
         }
     }
 
@@ -32,9 +61,14 @@ public void awsSentinel2SearchTest() {
         Path expectedFile = path.resolve("results.txt");
         try {
             Executor.execute(cmd.split(" "));
-            Assert.assertTrue(Files.exists(expectedFile));
+            Assert.assertTrue(Files.exists(expectedFile) && Files.lines(expectedFile).count() > 0);
         } catch (Exception e) {
             Assert.fail(e.getMessage());
+        } finally {
+            try {
+                Files.deleteIfExists(expectedFile);
+            } catch (IOException e) {
+            }
         }
     }
 
@@ -50,6 +84,11 @@ public void awsLandsat8SearchTest() {
             Assert.assertTrue(Files.lines(expectedFile).anyMatch("LC08_L1GT_139045_20170710_20170710_01_RT"::equals));
         } catch (Exception e) {
             Assert.fail(e.getMessage());
+        } finally {
+            try {
+                Files.deleteIfExists(expectedFile);
+            } catch (IOException e) {
+            }
         }
     }