Allow secrets to be used in pipeline script (nextflow-io#4171)

Signed-off-by: Ben Sherman <bentshermann@gmail.com>
Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
Co-authored-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
Signed-off-by: Niklas Schandry <niklas@bio.lmu.de>

docs/script.md

@@ -533,6 +533,11 @@ The following variables are implicitly defined in the script global execution sc
   :::
 : The directory where the main script is located.
 
+`secrets`
+: :::{versionadded} 24.02.0-edge
+  :::
+: Dictionary like object holding workflow secrets. Read the {ref}`secrets-page` page for more information.
+
 `workDir`
 : The directory where tasks temporary files are created.
 

docs/secrets.md

@@ -47,7 +47,7 @@ The above snippet access the secrets `MY_ACCESS_KEY` and `MY_SECRET_KEY` previou
 Secrets **cannot** be assigned to pipeline parameters.
 :::
 
-## Process secrets
+## Process directive
 
 Secrets can be access by pipeline processes by using the `secret` directive. For example:
 
@@ -71,3 +71,20 @@ The secrets are made available in the process context running the command script
 :::{note}
 This feature is only available when using the local or grid executors (Slurm, Grid Engine, etc). The AWS Batch executor allows the use of secrets when deploying the pipeline execution via [Seqera Platform](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/).
 :::
+
+## Pipeline script
+
+:::{versionadded} 23.09.0-edge
+:::
+
+Secrets can be accessed in the pipeline script using the `secrets` variable. For example:
+
+```groovy
+workflow.onComplete {
+    println("The secret is: ${secrets.MY_SECRET}")
+}
+```
+
+:::{note}
+This feature is only available when using the local or grid executors (Slurm, Grid Engine, etc). The AWS Batch executor allows the use of secrets when deploying the pipeline execution via [Nextflow Tower](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/).
+:::

modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy

@@ -23,6 +23,8 @@ import groovy.util.logging.Slf4j
 import nextflow.NextflowMeta
 import nextflow.Session
 import nextflow.exception.AbortOperationException
+import nextflow.secret.SecretsLoader
+import nextflow.secret.SecretsProvider
 /**
  * Any user defined script will extends this class, it provides the base execution context
  *
@@ -78,14 +80,27 @@ abstract class BaseScript extends Script implements ExecutionContext {
         binding.owner = this
         session = binding.getSession()
         processFactory = session.newProcessFactory(this)
+        final secretsProvider = SecretsLoader.isEnabled() ? SecretsLoader.instance.load() : null
 
         binding.setVariable( 'baseDir', session.baseDir )
         binding.setVariable( 'projectDir', session.baseDir )
         binding.setVariable( 'workDir', session.workDir )
         binding.setVariable( 'workflow', session.workflowMetadata )
         binding.setVariable( 'nextflow', NextflowMeta.instance )
-        binding.setVariable('launchDir', Paths.get('./').toRealPath())
-        binding.setVariable('moduleDir', meta.moduleDir )
+        binding.setVariable( 'launchDir', Paths.get('./').toRealPath() )
+        binding.setVariable( 'moduleDir', meta.moduleDir )
+        binding.setVariable( 'secrets', makeSecretsContext(secretsProvider) )
+    }
+
+    protected makeSecretsContext(SecretsProvider provider) {
+
+        return new Object() {
+            def getProperty(String name) {
+                if( !provider )
+                    throw new AbortOperationException("Unable to resolve secrets.$name - no secret provider is available")
+                provider.getSecret(name)?.value
+            }
+        }
     }
 
     protected process( String name, Closure<BodyDef> body ) {

modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy

@@ -17,12 +17,11 @@
 
 package nextflow.secret
 
-import groovy.transform.Memoized
+
 import groovy.util.logging.Slf4j
 import nextflow.SysEnv
 import nextflow.exception.AbortOperationException
 import nextflow.plugin.Plugins
-
 /**
  * Implements dynamic secret providing loading strategy
  *
@@ -32,12 +31,23 @@ import nextflow.plugin.Plugins
 @Singleton
 class SecretsLoader {
 
+    private SecretsProvider provider
+
     static boolean isEnabled() {
         SysEnv.get('NXF_ENABLE_SECRETS', 'true') == 'true'
     }
 
-    @Memoized
     SecretsProvider load() {
+        if( provider )
+            return provider
+        synchronized (this) {
+            if( provider )
+                return provider
+            return provider = load0()
+        }
+    }
+
+    private SecretsProvider load0() {
         // discover all available secrets provider
         final all = Plugins.getPriorityExtensions(SecretsProvider)
         // find first activable in the current environment
@@ -49,4 +59,7 @@ class SecretsLoader {
             throw new AbortOperationException("Unable to load secrets provider")
     }
 
+    void reset() {
+        provider=null
+    }
 }

modules/nextflow/src/test/groovy/nextflow/cli/CmdSecretTest.groovy

@@ -51,8 +51,7 @@ class CmdSecretTest extends Specification {
         secretFile = new File("$tempDir/store.json")
         SysEnv.push([NXF_SECRETS_FILE: secretFile.toString()])
         //required to run all test due collisions with others
-        def memoized = SecretsLoader.instance.memoizedMethodClosure$load
-        memoized.@cache.clear()
+        SecretsLoader.instance.reset()
     }
 
     def cleanupSpec() {

modules/nextflow/src/test/groovy/nextflow/script/BaseScriptTest.groovy

@@ -21,6 +21,11 @@ import java.nio.file.Paths
 
 import nextflow.NextflowMeta
 import nextflow.Session
+import nextflow.SysEnv
+import nextflow.extension.FilesEx
+import nextflow.secret.Secret
+import nextflow.secret.SecretsLoader
+import nextflow.secret.SecretsProvider
 import test.Dsl2Spec
 import test.TestHelper
 /**
@@ -141,4 +146,60 @@ class BaseScriptTest extends Dsl2Spec {
         folder?.delete()
     }
 
+    def 'should create secret context' () {
+        given:
+        def script = Spy(BaseScript)
+        def provider = Mock(SecretsProvider)
+        and:
+        def ctx = script.makeSecretsContext(provider)
+        when:
+        def result = ctx.'MY_SECRET'
+        then:
+        provider.getSecret('MY_SECRET') >> Mock(Secret) { getValue()>>'123' }
+        result == '123'
+    }
+
+    def 'should resolve secret in a script' () {
+        given:
+        SecretsLoader.instance.reset()
+        and:
+        def folder = Files.createTempDirectory('test')
+        def script = folder.resolve('main.nf')
+        def secrets  = folder.resolve('store.json')
+        and:
+        secrets.text = '''
+            [
+              {
+                "name": "FOO",
+                "value": "ciao"
+              }
+            ]
+            '''
+        and:
+        FilesEx.setPermissions(secrets, 'rw-------')
+        SysEnv.push(NXF_SECRETS_FILE:secrets.toAbsolutePath().toString())
+        and:
+        def session = Mock(Session)
+        def binding = new ScriptBinding([:])
+        def parser = new ScriptParser(session)
+
+        when:
+        script.text = '''
+                return secrets.FOO
+                '''
+
+        def result = parser
+            .setBinding(binding)
+            .runScript(script)
+            .getResult()
+
+        then:
+        result == 'ciao'
+
+        cleanup:
+        folder?.deleteDir()
+        and:
+        SysEnv.pop()
+    }
+
 }