Fixed memory allocation for SGE executor

This commit fixes the memory allocation made by the SGE
executor so that physical memory request and limit are
submitted instead of virtual memory.

This is required to make the mem requriment for the SGE
executor uniform with other batch schedulers and cloud
platforms.

It also needed to correctly show the memory allocation
in the execution report.

The CRG executor continues to use virtual memory for
compatibility with legacy CRG environment.

modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy

@@ -39,7 +39,42 @@ class CrgExecutor extends SgeExecutor {
             task.config.penv = 'smp'
         }
 
-        super.getDirectives(task, result)
+        result << '-wd' << quote(task.workDir)
+        result << '-N' << getJobNameFor(task)
+        result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
+        result << '-j' << 'y'
+        result << '-terse' << ''    // note: directive need to be returned as pairs
+
+        /*
+         * By using command line option -notify SIGUSR1 will be sent to your script prior to SIGSTOP
+         * and SIGUSR2 will be sent to your script prior to SIGKILL
+         */
+        result << '-notify' << ''
+
+        // the requested queue name
+        if( task.config.queue ) {
+            result << '-q' << (task.config.queue as String)
+        }
+
+        //number of cpus for multiprocessing/multi-threading
+        if ( task.config.penv ) {
+            result << "-pe" << "${task.config.penv} ${task.config.cpus}"
+        }
+        else if( task.config.cpus>1 ) {
+            result << "-l" << "slots=${task.config.cpus}"
+        }
+
+        // max task duration
+        if( task.config.time ) {
+            final time = task.config.getTime()
+            result << "-l" << "h_rt=${time.format('HH:mm:ss')}"
+        }
+
+        // task max memory
+        if( task.config.getMemory() ) {
+            final mem = "${task.config.getMemory().mega}M"
+            result << "-l" << "h_vmem=$mem,virtual_free=$mem"
+        }
 
         if( task.config.getDisk() ) {
             result << "-l" << "disk=${task.config.getDisk().toMega()}M"
@@ -58,6 +93,11 @@ class CrgExecutor extends SgeExecutor {
             result << '-soft' << "-l docker_images=*;${task.container};*"
         }
 
+        // -- at the end append the command script wrapped file name
+        if( task.config.clusterOptions ) {
+            result << task.config.clusterOptions.toString() << ''
+        }
+
         return result
     }
 

modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy

@@ -67,7 +67,8 @@ class SgeExecutor extends AbstractGridExecutor {
 
         // task max memory
         if( task.config.memory ) {
-            result << "-l" << "virtual_free=${task.config.memory.toString().replaceAll(/[\sB]/,'')}"
+            final mem = "${task.config.getMemory().mega}M"
+            result << "-l" << "h_rss=$mem,mem_free=$mem"
         }
 
         // -- at the end append the command script wrapped file name

modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy

@@ -166,7 +166,7 @@ class CrgExecutorTest extends Specification {
         // config
         task.config = new TaskConfig(
                         queue: 'short',
-                        memory: '4 GB',
+                        memory: '200 MB',
                         time: '1d',
                         disk: '2G'
                     )
@@ -181,7 +181,7 @@ class CrgExecutorTest extends Specification {
                     #$ -notify
                     #$ -q short
                     #$ -l h_rt=24:00:00
-                    #$ -l virtual_free=4G
+                    #$ -l h_vmem=200M,virtual_free=200M
                     #$ -l disk=2048M
                     '''
                     .stripIndent().leftTrim()
@@ -205,7 +205,7 @@ class CrgExecutorTest extends Specification {
                     #$ -notify
                     #$ -q short
                     #$ -l h_rt=24:00:00
-                    #$ -l virtual_free=4G
+                    #$ -l h_vmem=4096M,virtual_free=4096M
                     '''
                 .stripIndent().leftTrim()
 
@@ -230,7 +230,7 @@ class CrgExecutorTest extends Specification {
                     #$ -notify
                     #$ -q short
                     #$ -l h_rt=24:00:00
-                    #$ -l virtual_free=4G
+                    #$ -l h_vmem=4096M,virtual_free=4096M
                     #$ -binding env linear:1
                     #$ -soft -l docker_images=*;ubuntu;*
                     '''
@@ -259,7 +259,7 @@ class CrgExecutorTest extends Specification {
                     #$ -q long
                     #$ -pe mpi 2
                     #$ -l h_rt=72:00:00
-                    #$ -l virtual_free=3G
+                    #$ -l h_vmem=3072M,virtual_free=3072M
                     #$ -binding env linear:2
                     #$ -R y
                     #$ -soft -l docker_images=*;busybox;*
@@ -292,7 +292,7 @@ class CrgExecutorTest extends Specification {
                     #$ -q long
                     #$ -pe mpi 4
                     #$ -l h_rt=72:00:00
-                    #$ -l virtual_free=3G
+                    #$ -l h_vmem=3072M,virtual_free=3072M
                     #$ -binding env linear:4
                     #$ -R y
                     #$ -soft -l docker_images=*;busybox;*

modules/nextflow/src/test/groovy/nextflow/executor/SgeExecutorTest.groovy

@@ -126,7 +126,7 @@ class SgeExecutorTest extends Specification {
                 #$ -notify
                 #$ -q my-queue
                 #$ -l h_rt=00:10:00
-                #$ -l virtual_free=1M
+                #$ -l h_rss=1M,mem_free=1M
                 '''
                 .stripIndent().leftTrim()
 
@@ -151,7 +151,7 @@ class SgeExecutorTest extends Specification {
                 #$ -q my-queue
                 #$ -pe smp 1
                 #$ -l h_rt=00:02:00
-                #$ -l virtual_free=2M
+                #$ -l h_rss=2M,mem_free=2M
                 '''
                 .stripIndent().leftTrim()
 
@@ -174,7 +174,7 @@ class SgeExecutorTest extends Specification {
                 #$ -q my-queue
                 #$ -pe mpi 2
                 #$ -l h_rt=72:00:00
-                #$ -l virtual_free=3G
+                #$ -l h_rss=3072M,mem_free=3072M
                 '''
                 .stripIndent().leftTrim()
 
@@ -197,7 +197,7 @@ class SgeExecutorTest extends Specification {
                 #$ -q my-queue
                 #$ -pe orte 4
                 #$ -l h_rt=27:00:00
-                #$ -l virtual_free=4G
+                #$ -l h_rss=4096M,mem_free=4096M
                 '''
                 .stripIndent().leftTrim()