Code cleanups

* Add example of keep: URL
* make ramMin optional to suppress a spurious checker warning
* rename quote() function to sanitize() (although it still also adds quotes)
* don't generate empty batches when len(urls) < count

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

README.md

@@ -53,9 +53,11 @@ Here is what the input.yml file should look like:
 # List of File objects to upload
 files:
   - class: File
+    # local file
     location: file1.txt
   - class: File
-    location: file2.txt
+    # file in keep
+    location: keep:3c1ffefa04e684405dcb7e0c77830038+129/file2.txt
 
 # The target s3:// URL to upload to
 # This is a string prefix to which the file name is appended

tools/aws-s3-download.cwl

@@ -11,7 +11,7 @@ inputs:
   aws_secret_access_key: string
   endpoint: string?
   ramMin:
-    type: int
+    type: int?
     default: 1000
 
 requirements:
@@ -34,15 +34,15 @@ requirements:
         entry: |
           ${
           // cwltool expression scanner has trouble with unbalanced quotes, the workaround
-          // is adding the comment on the next line
+          // is adding trailing comment on the next line
           var rx = /['\\]/g; // '
-          var quote = function(s) { return "'"+s.replace(rx, "")+"'"; }
+          var sanitize = function(s) { return "'"+s.replace(rx, "")+"'"; }
           var endpoint = "";
           if (inputs.endpoint) {
-            endpoint = "--endpoint "+quote(inputs.endpoint);
+            endpoint = "--endpoint "+sanitize(inputs.endpoint);
           }
           var commands = inputs.s3urls.map(function(url) {
-            return "aws s3 cp "+endpoint+" --no-progress "+quote(url)+" "+quote(url.split('/').pop());
+            return "aws s3 cp "+endpoint+" --no-progress "+sanitize(url)+" "+sanitize(url.split('/').pop());
           });
           commands.unshift("set -e");
           commands.push("");

tools/aws-s3-upload.cwl

@@ -12,7 +12,7 @@ inputs:
   aws_secret_access_key: string
   endpoint: string?
   ramMin:
-    type: int
+    type: int?
     default: 1000
 
 requirements:
@@ -37,15 +37,15 @@ requirements:
         entry: |
           ${
           // cwltool expression scanner has trouble with unbalanced quotes, the workaround
-          // is adding the comment on the next line
+          // is adding the trailing comment on the next line
           var rx = /['\\]/g; // '
-          var quote = function(s) { return "'"+s.replace(rx, "")+"'"; }
+          var sanitize = function(s) { return "'"+s.replace(rx, "")+"'"; }
           var endpoint = "";
           if (inputs.endpoint) {
-            endpoint = "--endpoint "+quote(inputs.endpoint);
+            endpoint = "--endpoint "+sanitize(inputs.endpoint);
           }
           var commands = inputs.files.map(function(file) {
-            return "aws s3 cp "+endpoint+" --no-progress "+quote(file.path)+" "+quote(inputs.s3target+file.basename);
+            return "aws s3 cp "+endpoint+" --no-progress "+sanitize(file.path)+" "+sanitize(inputs.s3target+file.basename);
           });
           commands.unshift("set -e");
           commands.push("");

tools/batch.cwl

@@ -18,11 +18,12 @@ outputs:
 expression: |
   ${
   var batches = [];
-  for (var i = 0; i < inputs.count; i++) {
+  var batchcount = Math.min(inputs.count, inputs.urls.length);
+  for (var i = 0; i < batchcount; i++) {
     batches.push([]);
   }
   for (var n = 0; n < inputs.urls.length; n++) {
-    batches[n % inputs.count].push(inputs.urls[n]);
+    batches[n % batchcount].push(inputs.urls[n]);
   }
   return {"batches": batches};
   }