Fix and improve environment variable resolution in repositories

NEWS.adoc

@@ -1,5 +1,12 @@
 = Changes
 
+== [Future] - ?
+
+=== Fixed
+
+* (*Breaking*) Since migrating to a Kotlin Scripting backend, environment variables for repository authentication now use the `$FOO` syntax vs the `{{FOO}}` syntax. This has been broken since 4.2.0. (fixed thanks to https://github.com/rocketraman[rocketraman])
+* Packaging scripts via `--package` did not work with repository authentication (thanks to https://github.com/rocketraman[rocketraman])
+
 == [4.2.2] - 2023-04-29
 
 === Added

README.adoc

@@ -139,7 +139,7 @@ Two workarounds for ArchLinux exists, which can be used to make 'kscript' workin
 [source,bash]
 ----
 sudo mkdir /usr/share/kotlin/bin
-sudo ln -s /usr/bin/kotlin /usr/share/kotlin/bin/kotlin   
+sudo ln -s /usr/bin/kotlin /usr/share/kotlin/bin/kotlin
 sudo ln -s /usr/bin/kotlinc /usr/share/kotlin/bin/kotlinc
 ----
 
@@ -243,7 +243,7 @@ EOF
 
 [source,bash]
 ----
-kscript <(echo 'println("k-onliner")') arg1 arg2 arg3 
+kscript <(echo 'println("k-onliner")') arg1 arg2 arg3
 ----
 
 Inlined _kscripts_ are also cached based on `md5` checksum, so running the same snippet again will use a cached jar (
@@ -259,7 +259,7 @@ script for better readability)
 
 [source,bash]
 ----
-kscript https://git.io/v1cG6 my argu ments 
+kscript https://git.io/v1cG6 my argu ments
 ----
 
 To streamline the usage, the first part could be even aliased:
@@ -414,12 +414,12 @@ The latter is the default for `kt` files and could be omitted
 @file:DependsOnMaven("net.clearvolume:cleargl:2.0.1")
 // Note that for compatibility reasons, only one locator argument is allowed for @DependsOnMaven
 
-// also protected artifact repositories are supported, see <https://github.com/kscripting/kscript/blob/master/test/TestsReadme.md#manual-testing>
+// also protected repositories are supported, see <https://github.com/kscripting/kscript/blob/master/test/TestsReadme.md#manual-testing>
 // @file:Repository("my-art", "http://localhost:8081/artifactory/authenticated_repo", user="auth_user", password="password")
-// You can use environment variables for user and password when string surrounded by double {} brackets 
-// @file:Repository("my-art", "http://localhost:8081/artifactory/authenticated_repo", user="{{ARTIFACTORY_USER}}", password="{{ARTIFACTORY_PASSWORD}}")
+// You can use environment variables for user and password
+// @file:Repository("my-art", "http://localhost:8081/artifactory/authenticated_repo", user="$ARTIFACTORY_USER", password="$ARTIFACTORY_PASSWORD")
 // will be use 'ARTIFACTORY_USER' and 'ARTIFACTORY_PASSWORD' environment variables
-// if the value doesn't found in the script environment  will fail
+// if a referenced environment variable is not found, the script will fail
 
 // Include helper scripts without deployment or prior compilation
 @file:Import("util.kt")
@@ -516,7 +516,7 @@ which will bring up the classpath-enhanced REPL:
 Creating REPL from CountRecords.kts
 Welcome to Kotlin version 1.1.51 (JRE 1.8.0_151-b12)
 >>> import de.mpicbg.scicomp.bioinfo.openFasta
->>> 
+>>>
 ----
 
 == Boostrap IDEA from a scriptlet
@@ -582,7 +582,7 @@ KScript follows XDG directory standard, so the file should be created in (paths
 existing path is used):
 
 |===
-|OS |PATH 
+|OS |PATH
 
 |*Windows* | %LOCALAPPDATA%\kscript\kscript.properties; %USERPROFILE%.config\kscript\kscript.properties
 |*MacOs*   | ~/Library/Application Support/kscript/kscript.properties;
@@ -623,6 +623,43 @@ scripting.repository.user=user
 scripting.repository.password=password
 ----
 
+Various environment variables, if set, may override these properties or configure other behavior. See
+https://github.com/kscripting/kscript/blob/1c97c2197ddd08382834c014096ffe8b26727a4f/src/main/kotlin/io/github/kscripting/kscript/model/ConfigBuilder.kt#L42[ConfigBuilder]
+for all the possible environment overrides. The most common ones corresponding to the properties above will likely be:
+
+* `KSCRIPT_KOTLIN_OPTS`
+* `KSCRIPT_REPOSITORY_URL`
+* `KSCRIPT_REPOSITORY_USER`
+* `KSCRIPT_REPOSITORY_PASSWORD`
+
+In addition, environment lookups may be made dynamically in a script specifically for repository url, username, and
+password by prefixing the lookup with a literal `$`.
+
+NOTE: Unlike in Kotlin code, the '$' in a kscript is not escaped.
+
+In addition, ``KSCRIPT_REPOSITORY_*` values may be included in an override by using the double-brace syntax
+`{{KSCRIPT_REPOSITORY_URL}}`. For example, a repository declaration like:
+
+[source,kotlin]
+----
+#!/usr/bin/env kscript
+@file:Repository("$REPO", user="$USER", password="$PASS")
+----
+
+would obtain the user from the `USER` environment variable.
+If the `USER` environment variable contained a value like `foo-{{KSCRIPT_REPOSITORY_URL}}` and the value of
+`KSCRIPT_REPOSITORY_URL` (either also from the environment or from the kscript properties), was `bar`, then the resolved
+value for `user` would be `foo-bar`.
+
+The placeholder values `{{KSCRIPT_REPOSITORY_URL}}`, `{{KSCRIPT_REPOSITORY_USER}}`, and
+`{{KSCRIPT_REPOSITORY_PASSWORD}}` may also be used directly with optional prefixes or suffixes. For example:
+
+[source,kotlin]
+----
+#!/usr/bin/env kscript
+@file:Repository("{{KSCRIPT_REPOSITORY_URL}}", user="prefix-{{KSCRIPT_REPOSITORY_USER}}", password="{{KSCRIPT_REPOSITORY_PASSWORD}}-postfix")
+----
+
 == FAQ
 
 === How to edit kscript in VS Code?

src/main/kotlin/io/github/kscripting/kscript/KscriptHandler.kt

@@ -50,7 +50,7 @@ class KscriptHandler(
         }
 
         val inputOutputResolver = InputOutputResolver(config.osConfig, cache)
-        val sectionResolver = SectionResolver(inputOutputResolver, Parser(), config.scriptingConfig)
+        val sectionResolver = SectionResolver(inputOutputResolver, Parser(), config.scriptingConfig, config.osConfig)
         val scriptResolver = ScriptResolver(inputOutputResolver, sectionResolver, config.scriptingConfig)
 
         if (options.containsKey("add-bootstrap-header")) {

src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt

@@ -61,9 +61,9 @@ object GradleTemplates {
             |}
             |
             |repositories {
-            |    mavenLocal()
-            |    mavenCentral()
             |${createGradleRepositoriesSection(script.repositories).prependIndent()}
+            |    mavenCentral()
+            |    mavenLocal()
             |}
             |
             |tasks.jar {
@@ -124,6 +124,7 @@ object GradleTemplates {
     private fun createGradleRepositoriesSection(repositories: Set<Repository>) = repositories.joinToString("\n") {
         """|maven {
            |    url = uri("${it.url}")
+           |    isAllowInsecureProtocol = true
            |${createGradleRepositoryCredentials(it).prependIndent()}
            |}
         """.trimMargin()

src/main/kotlin/io/github/kscripting/kscript/model/Config.kt

@@ -33,6 +33,7 @@ data class OsConfig(
     val configFile: OsPath,
     val cacheDir: OsPath,
     val kotlinHomeDir: OsPath,
+    val environment: ProcessEnvironment,
 ) {
     override fun toString(): String {
         return """|OsConfig {
@@ -44,6 +45,7 @@ data class OsConfig(
                   |  configFile:            $configFile
                   |  cacheDir:              $cacheDir
                   |  kotlinHomeDir:         $kotlinHomeDir
+                  |  environment:           ${environment.size} entries
                   |}
                """.trimMargin()
     }

src/main/kotlin/io/github/kscripting/kscript/model/ConfigBuilder.kt

@@ -87,6 +87,7 @@ class ConfigBuilder(
             configFile,
             cacheDir,
             kotlinHomeDir,
+            environment,
         )
 
         val configProperties = Properties().apply {

src/main/kotlin/io/github/kscripting/kscript/model/ProcessEnvironment.kt

@@ -0,0 +1,3 @@
+package io.github.kscripting.kscript.model
+
+typealias ProcessEnvironment = Map<String, String?>

src/main/kotlin/io/github/kscripting/kscript/resolver/SectionResolver.kt

@@ -2,6 +2,7 @@ package io.github.kscripting.kscript.resolver
 
 import io.github.kscripting.kscript.model.*
 import io.github.kscripting.kscript.parser.Parser
+import io.github.kscripting.kscript.util.ScriptUtils.resolveRepositoryOption
 import io.github.kscripting.kscript.util.UriUtils
 import io.github.kscripting.shell.model.ScriptLocation
 import io.github.kscripting.shell.model.ScriptSource
@@ -10,14 +11,15 @@ import java.net.URI
 class SectionResolver(
     private val inputOutputResolver: InputOutputResolver,
     private val parser: Parser,
-    private val scriptingConfig: ScriptingConfig
+    private val scriptingConfig: ScriptingConfig,
+    private val osConfig: OsConfig,
 ) {
     fun resolve(
         scriptLocation: ScriptLocation,
         scriptText: String,
         allowLocalReferences: Boolean,
         maxResolutionLevel: Int,
-        resolutionContext: ResolutionContext
+        resolutionContext: ResolutionContext,
     ): List<Section> {
         val sections = parser.parse(scriptLocation, scriptText)
         val resultingSections = mutableListOf<Section>()
@@ -32,7 +34,7 @@ class SectionResolver(
                     allowLocalReferences,
                     scriptLocation.level,
                     maxResolutionLevel,
-                    resolutionContext
+                    resolutionContext,
                 )
             }
 
@@ -48,7 +50,7 @@ class SectionResolver(
         allowLocalReferences: Boolean,
         currentLevel: Int,
         maxResolutionLevel: Int,
-        resolutionContext: ResolutionContext
+        resolutionContext: ResolutionContext,
     ): List<ScriptAnnotation> {
         val resolvedScriptAnnotations = mutableListOf<ScriptAnnotation>()
 
@@ -82,7 +84,7 @@ class SectionResolver(
                         content.text,
                         allowLocalReferences && scriptSource == ScriptSource.FILE,
                         maxResolutionLevel,
-                        resolutionContext
+                        resolutionContext,
                     )
 
                     val scriptNode = ScriptNode(scriptLocation, newSections)
@@ -132,14 +134,30 @@ class SectionResolver(
             }
 
             is Repository -> {
+                val environment = osConfig.environment
                 val repository = Repository(
-                    scriptAnnotation.id, scriptAnnotation.url.replace(
-                        "{{KSCRIPT_REPOSITORY_URL}}", scriptingConfig.providedRepositoryUrl
-                    ), scriptAnnotation.user.replace(
-                        "{{KSCRIPT_REPOSITORY_USER}}", scriptingConfig.providedRepositoryUser
-                    ), scriptAnnotation.password.replace(
-                        "{{KSCRIPT_REPOSITORY_PASSWORD}}", scriptingConfig.providedRepositoryPassword
-                    )
+                    id = scriptAnnotation.id,
+                    url = resolveRepositoryOption(
+                        scriptAnnotation.url,
+                        "url",
+                        "{{KSCRIPT_REPOSITORY_URL}}",
+                        scriptingConfig.providedRepositoryUrl,
+                        environment,
+                    ),
+                    user = resolveRepositoryOption(
+                        scriptAnnotation.user,
+                        "user",
+                        "{{KSCRIPT_REPOSITORY_USER}}",
+                        scriptingConfig.providedRepositoryUser,
+                        environment,
+                    ),
+                    password = resolveRepositoryOption(
+                        scriptAnnotation.password,
+                        "password",
+                        "{{KSCRIPT_REPOSITORY_PASSWORD}}",
+                        scriptingConfig.providedRepositoryPassword,
+                        environment,
+                    ),
                 )
 
                 resolutionContext.repositories.add(repository)

src/main/kotlin/io/github/kscripting/kscript/util/ScriptUtils.kt

@@ -3,6 +3,7 @@ package io.github.kscripting.kscript.util
 import io.github.kscripting.kscript.model.Code
 import io.github.kscripting.kscript.model.ImportName
 import io.github.kscripting.kscript.model.PackageName
+import io.github.kscripting.kscript.model.ProcessEnvironment
 import io.github.kscripting.kscript.model.ScriptNode
 import io.github.kscripting.kscript.resolver.ResolutionContext
 import io.github.kscripting.shell.model.ScriptType
@@ -57,6 +58,35 @@ object ScriptUtils {
         return sb.toString()
     }
 
+    fun resolveRepositoryOption(
+        str: String?,
+        optionName: String,
+        placeholder: String,
+        property: String,
+        environment: ProcessEnvironment,
+    ): String = tryResolveEnvironmentVariable(str, optionName, environment)
+        ?.replace(placeholder, property)
+        ?: error("Failed to resolve value for option '$optionName'")
+
+    /**
+     * This is a variant of [kotlin.script.experimental.dependencies.maven.MavenDependenciesResolver.tryResolveEnvironmentVariable].
+     */
+    private fun tryResolveEnvironmentVariable(
+        str: String?,
+        optionName: String,
+        environment: ProcessEnvironment,
+    ): String? {
+        if (str == null) return null
+        if (!str.startsWith("$")) return str
+        val envName = str.substring(1)
+        val envValue: String? = environment[envName]
+        if (envValue.isNullOrEmpty()) {
+            Logger.errorMsg("Environment variable '$envName' is not defined for option '$optionName'")
+            return null
+        }
+        return envValue
+    }
+
     private fun resolveSimpleCode(sb: StringBuilder, scriptNode: ScriptNode, lastLineIsEmpty: Boolean = false) {
         var isLastLineEmpty = lastLineIsEmpty
 

src/test/kotlin/io/github/kscripting/kscript/code/GradleTemplatesTest.kt

@@ -95,13 +95,15 @@ class GradleTemplatesTest {
             |    mavenCentral()
             |    maven {
             |        url = uri("https://url1")
+            |        isAllowInsecureProtocol = true
             |        credentials {
             |            username = "user1"
             |            password = "pass1"
             |        }
             |    }
             |    maven {
             |        url = uri("https://url2")
+            |        isAllowInsecureProtocol = true
             |        credentials {
             |            username = "user2"
             |            password = "pass2"

src/test/kotlin/io/github/kscripting/kscript/resolver/CommandResolverTest.kt

@@ -106,6 +106,7 @@ class CommandResolverTest {
             homeDir.resolve("./.config/"),
             homeDir.resolve("./.cache/"),
             kotlinDir,
+            emptyMap(),
         )
 
         val scriptingConfig = ScriptingConfig("", "", "", "", "", null)