Merge pull request #2533 from markvandertol/coursier-headers

Collect HTTP Headers from Coursier for authentication

README.md

@@ -115,6 +115,7 @@ Thanks goes to these wonderful people for contributing to Scala Steward:
 * [Manuel Cueto](https://github.com/manuelcueto)
 * [Marco Zühlke](https://github.com/mzuehlke) 
 * [Mark Canlas](https://github.com/mcanlas)
+* [Mark van der Tol](https://github.com/markvandertol)
 * [MaT1g3R](https://github.com/MaT1g3R)
 * [Mat Mannion](https://github.com/matmannion)
 * [Michael Wizner](https://github.com/mwz)

docs/running.md

@@ -199,7 +199,7 @@ There is multiple articles on how to run Scala Steward on-premise:
 
 #### GitLab
 
-The following describes a setup using GitLab Docker runner, which you have to setup seperately.
+The following describes a setup using GitLab Docker runner, which you have to set up separately.
 
 1. create a "scalasteward" user in GitLab
 2. assign that user "Developer" permissions in every project that should be managed by Scala Steward
@@ -260,6 +260,33 @@ echo "${SCALA_STEWARD_TOKEN}"
 7. add the `repos.md` file 
 8. (*optional*) create a new schedule to trigger the pipeline on a daily/weekly basis
 
+Scala Steward is compatible with Coursier authentication using headers. To authenticate
+using the [Gitlab CI/CD job token](https://docs.gitlab.com/ee/ci/jobs/ci_job_token.html), while also supporting your own private token when performing
+local development, use the following snippet:
+```scala
+import lmcoursier.CoursierConfiguration
+import lmcoursier.definitions.Authentication
+
+lazy val gitlabToken: Option[(String, String)] = {
+  //The Gitlab runner sets CI_JOB_TOKEN automatically as part of running inside a build job
+  val jobToken = sys.env.get("CI_JOB_TOKEN").map(t => ("Job-Token", t)) 
+  //When running on your local machine, set the environment variable GITLAB_PRIVATE_TOKEN
+  val privateToken = sys.env.get("GITLAB_PRIVATE_TOKEN").map(t => ("Private-Token", t))
+
+  jobToken.orElse(privateToken)
+}
+
+def addGitlabToken(current: CoursierConfiguration): CoursierConfiguration = {
+  gitlabToken.fold(current) { token =>
+    current.addRepositoryAuthentication("gitlab-repo", Authentication(Seq(token)))
+  }
+}
+
+resolvers += "gitlab-repo" at s"https://gitlab.example.com/api/v4/groups/1/-/packages/maven"
+csrConfiguration ~= addGitlabToken
+updateClassifiers / csrConfiguration ~= addGitlabToken
+updateSbtClassifiers / csrConfiguration ~= addGitlabToken
+```
 
 [scalafixmigrations]: https://github.com/scala-steward-org/scala-steward/blob/main/docs/scalafix-migrations.md
 [artifactmigrations]: https://github.com/scala-steward-org/scala-steward/blob/main/docs/artifact-migrations.md

modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala

@@ -253,7 +253,7 @@ object Cli {
   private val defaultMavenRepo: Opts[Resolver] = {
     val default = Resolver.mavenCentral
     option[String]("default-maven-repo", s"default: ${default.location}")
-      .map(location => Resolver.MavenRepository("default", location, None))
+      .map(location => Resolver.MavenRepository("default", location, None, Nil))
       .withDefault(default)
   }
 

modules/core/src/main/scala/org/scalasteward/core/buildtool/maven/parser.scala

@@ -72,7 +72,7 @@ object parser {
       id <- stringNoSpace
       _ <- wsp.rep0 ~ Parser.string("url:") ~ wsp
       url <- stringNoSpace
-    } yield Resolver.MavenRepository(id, url, None)
+    } yield Resolver.MavenRepository(id, url, None, Nil)
 
   def parseResolvers(input: List[String]): List[Resolver] =
     input.mkString.split("""\[INFO]""").toList.flatMap { line =>

modules/core/src/main/scala/org/scalasteward/core/buildtool/mill/parser.scala

@@ -59,12 +59,14 @@ object MillModule {
         for {
           url <- c.downField("url").as[String]
           creds <- c.downField("auth").as[Option[Resolver.Credentials]]
-        } yield Resolver.MavenRepository(url, url, creds)
+          headers <- c.downField("headers").as[Option[List[Resolver.Header]]]
+        } yield Resolver.MavenRepository(url, url, creds, headers.getOrElse(Nil))
       case "ivy" =>
         for {
           url <- c.downField("pattern").as[String]
           creds <- c.downField("auth").as[Option[Resolver.Credentials]]
-        } yield Resolver.IvyRepository(url, url, creds)
+          headers <- c.downField("headers").as[Option[List[Resolver.Header]]]
+        } yield Resolver.IvyRepository(url, url, creds, headers.getOrElse(Nil))
       case typ => Left(DecodingFailure(s"Not a matching resolver type, $typ", c.history))
     }
   }

modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala

@@ -126,15 +126,35 @@ object CoursierAlg {
 
   private def toCoursierRepository(resolver: Resolver): Either[String, coursier.Repository] =
     resolver match {
-      case Resolver.MavenRepository(_, location, creds) =>
-        Right(coursier.maven.MavenRepository.apply(location, creds.map(toCoursierAuthentication)))
-      case Resolver.IvyRepository(_, pattern, creds) =>
+      case Resolver.MavenRepository(_, location, creds, headers) =>
+        Right(
+          coursier.maven.MavenRepository
+            .apply(location, toCoursierAuthentication(creds, headers))
+        )
+      case Resolver.IvyRepository(_, pattern, creds, headers) =>
         coursier.ivy.IvyRepository
-          .parse(pattern, authentication = creds.map(toCoursierAuthentication))
+          .parse(pattern, authentication = toCoursierAuthentication(creds, headers))
     }
 
-  private def toCoursierAuthentication(credentials: Credentials): Authentication =
-    Authentication(credentials.user, credentials.pass)
+  private def toCoursierAuthentication(
+      credentials: Option[Credentials],
+      headers: List[Resolver.Header]
+  ): Option[Authentication] =
+    if (credentials.isEmpty && headers.isEmpty) {
+      None
+    } else {
+      Some(
+        new Authentication(
+          credentials.fold("")(_.user),
+          credentials.map(_.pass),
+          headers.map(h => (h.key, h.value)),
+          optional = false,
+          None,
+          httpsOnly = true,
+          passOnRedirect = false
+        )
+      )
+    }
 
   private def getParentDependency(project: Project): Option[coursier.Dependency] =
     project.parent.map { case (module, version) =>

modules/core/src/main/scala/org/scalasteward/core/data/Resolver.scala

@@ -18,14 +18,16 @@ package org.scalasteward.core.data
 
 import cats.Order
 import io.circe.Codec
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.deriveConfiguredCodec
 import io.circe.generic.semiauto._
 import org.scalasteward.core.data.Resolver._
 
 sealed trait Resolver extends Product with Serializable {
   val path: String = {
     val url = this match {
-      case MavenRepository(_, location, _) => location
-      case IvyRepository(_, pattern, _)    => pattern.takeWhile(!Set('[', '(')(_))
+      case MavenRepository(_, location, _, _) => location
+      case IvyRepository(_, pattern, _, _)    => pattern.takeWhile(!Set('[', '(')(_))
     }
     url.replace(":", "")
   }
@@ -37,20 +39,35 @@ object Resolver {
     implicit val credentialsCodec: Codec[Credentials] =
       deriveCodec
   }
-  final case class MavenRepository(name: String, location: String, credentials: Option[Credentials])
-      extends Resolver
-  final case class IvyRepository(name: String, pattern: String, credentials: Option[Credentials])
-      extends Resolver
+  final case class Header(key: String, value: String)
+  object Header {
+    implicit val headerCodec: Codec[Header] = deriveCodec
+  }
+  final case class MavenRepository(
+      name: String,
+      location: String,
+      credentials: Option[Credentials],
+      headers: List[Header] = Nil
+  ) extends Resolver
+  final case class IvyRepository(
+      name: String,
+      pattern: String,
+      credentials: Option[Credentials],
+      headers: List[Header] = Nil
+  ) extends Resolver
 
   val mavenCentral: MavenRepository =
-    MavenRepository("public", "https://repo1.maven.org/maven2/", None)
+    MavenRepository("public", "https://repo1.maven.org/maven2/", None, Nil)
 
-  implicit val resolverCodec: Codec[Resolver] =
-    deriveCodec
+  @annotation.nowarn("cat=unused-locals")
+  implicit val resolverCodec: Codec[Resolver] = {
+    implicit val customConfig: Configuration = Configuration.default.withDefaults
+    deriveConfiguredCodec
+  }
 
   implicit val resolverOrder: Order[Resolver] =
     Order.by {
-      case MavenRepository(name, location, _) => (1, name, location)
-      case IvyRepository(name, pattern, _)    => (2, name, pattern)
+      case MavenRepository(name, location, _, _) => (1, name, location)
+      case IvyRepository(name, pattern, _, _)    => (2, name, pattern)
     }
 }

modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala

@@ -13,7 +13,8 @@ object TestSyntax {
       val sbtPluginReleases = IvyRepository(
         "sbt-plugin-releases",
         "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[defaultPattern]",
-        None
+        None,
+        Nil
       )
       Scope(self, List(sbtPluginReleases))
     }

modules/core/src/test/scala/org/scalasteward/core/buildtool/maven/parserTest.scala

@@ -63,10 +63,11 @@ class parserTest extends FunSuite {
       MavenRepository(
         "sonatype-nexus-snapshots",
         "https://oss.sonatype.org/content/repositories/snapshots",
-        None
+        None,
+        Nil
       ),
-      MavenRepository("bintrayakkamaven", "https://dl.bintray.com/akka/maven/", None),
-      MavenRepository("apache.snapshots", "http://repository.apache.org/snapshots", None)
+      MavenRepository("bintrayakkamaven", "https://dl.bintray.com/akka/maven/", None, Nil),
+      MavenRepository("apache.snapshots", "http://repository.apache.org/snapshots", None, Nil)
     )
     assertEquals(resolvers, expected)
   }

modules/core/src/test/scala/org/scalasteward/core/buildtool/sbt/parserTest.scala

@@ -5,7 +5,7 @@ import org.scalasteward.core.TestSyntax._
 import org.scalasteward.core.buildtool.sbt.data.{SbtVersion, ScalaVersion}
 import org.scalasteward.core.buildtool.sbt.parser._
 import org.scalasteward.core.data.Resolver.{Credentials, IvyRepository, MavenRepository}
-import org.scalasteward.core.data.Scope
+import org.scalasteward.core.data.{Resolver, Scope}
 
 class parserTest extends FunSuite {
   test("parseBuildProperties: with whitespace") {
@@ -19,16 +19,17 @@ class parserTest extends FunSuite {
          |[info] { "groupId": "org.scala-lang", "artifactId": { "name": "scala-library", "maybeCrossName": null }, "version": "2.12.7" }
          |[info] { "groupId": "com.github.pathikrit", "artifactId": { "name": "better-files", "maybeCrossName": "better-files_2.12" }, "version": "3.6.0" }
          |[info] { "groupId": "org.typelevel", "artifactId": { "name": "cats-effect", "maybeCrossName": "cats-effect_2.12" }, "version": "1.0.0" }
-         |[info] { "MavenRepository": { "name": "confluent-release", "location": "http://packages.confluent.io/maven/", "credentials": { "user": "donny", "pass": "brasc0" } } }
-         |[info] { "MavenRepository": { "name": "bintray-ovotech-maven", "location": "https://dl.bintray.com/ovotech/maven/" } }
+         |[info] { "MavenRepository": { "name": "confluent-release", "location": "http://packages.confluent.io/maven/", "credentials": { "user": "donny", "pass": "brasc0" }, "headers": [] } }
+         |[info] { "MavenRepository": { "name": "gitlab-internal", "location": "http://gitlab.example.com/maven/", "headers": [{ "key": "private-token", "value": "token123" }] } }
+         |[info] { "MavenRepository": { "name": "bintray-ovotech-maven", "location": "https://dl.bintray.com/ovotech/maven/", "headers": [] } }
          |[info] --- snip ---
          |sbt:project> stewardDependencies
          |[info] { "groupId": "org.scala-lang", "artifactId": { "name": "scala-library", "maybeCrossName": null }, "version": "2.12.6" }
          |[info] { "groupId": "com.dwijnand", "artifactId": { "name": "sbt-travisci", "maybeCrossName": null }, "version": "1.1.3",  "sbtVersion": "1.0" }
          |[info] { "groupId": "com.eed3si9n", "artifactId": { "name": "sbt-assembly", "maybeCrossName": null }, "version": "0.14.8", "sbtVersion": "1.0", "configurations": "foo" }
          |{ "groupId": "org.scalameta", "artifactId": { "name": "sbt-scalafmt", "maybeCrossName": null }, "version": "2.4.6", "sbtVersion": "1.0", "scalaVersion": "2.12", "configurations": null }
-         |[info] { "IvyRepository" : { "name": "sbt-plugin-releases", "pattern": "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)([branch]/)[revision]/[type]s/[artifact](-[classifier]).[ext]" } }
-         |[info] { "IvyRepository" : { "name": "sbt-plugin-releases-with-creds", "pattern": "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)([branch]/)[revision]/[type]s/[artifact](-[classifier]).[ext]", "credentials": { "user": "tony", "pass": "m0ntana" } } }
+         |[info] { "IvyRepository" : { "name": "sbt-plugin-releases", "pattern": "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)([branch]/)[revision]/[type]s/[artifact](-[classifier]).[ext]", "headers": [] } }
+         |[info] { "IvyRepository" : { "name": "sbt-plugin-releases-with-creds", "pattern": "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)([branch]/)[revision]/[type]s/[artifact](-[classifier]).[ext]", "credentials": { "user": "tony", "pass": "m0ntana" }, "headers": [] } }
          |[info] --- snip ---
          |""".stripMargin.linesIterator.toList
     val scopes = parseDependencies(lines)
@@ -40,11 +41,23 @@ class parserTest extends FunSuite {
           "org.typelevel".g % ("cats-effect", "cats-effect_2.12").a % "1.0.0"
         ),
         List(
-          MavenRepository("bintray-ovotech-maven", "https://dl.bintray.com/ovotech/maven/", None),
+          MavenRepository(
+            "bintray-ovotech-maven",
+            "https://dl.bintray.com/ovotech/maven/",
+            None,
+            Nil
+          ),
           MavenRepository(
             "confluent-release",
             "http://packages.confluent.io/maven/",
-            Some(Credentials("donny", "brasc0"))
+            Some(Credentials("donny", "brasc0")),
+            Nil
+          ),
+          MavenRepository(
+            "gitlab-internal",
+            "http://gitlab.example.com/maven/",
+            None,
+            List(Resolver.Header("private-token", "token123"))
           )
         )
       ),
@@ -62,16 +75,19 @@ class parserTest extends FunSuite {
           IvyRepository(
             "sbt-plugin-releases",
             "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)([branch]/)[revision]/[type]s/[artifact](-[classifier]).[ext]",
-            None
+            None,
+            Nil
           ),
           IvyRepository(
             "sbt-plugin-releases-with-creds",
             "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)([branch]/)[revision]/[type]s/[artifact](-[classifier]).[ext]",
-            Some(Credentials("tony", "m0ntana"))
+            Some(Credentials("tony", "m0ntana")),
+            Nil
           )
         )
       )
     )
+
     assertEquals(scopes, expected)
   }
 }

modules/core/src/test/scala/org/scalasteward/core/update/PruningAlgTest.scala

@@ -105,7 +105,8 @@ class PruningAlgTest extends FunSuite {
           |        {
           |          "MavenRepository" : {
           |            "name" : "public",
-          |            "location" : "https://foobar.org/maven2/"
+          |            "location" : "https://foobar.org/maven2/",
+          |            "headers" : []
           |          }
           |        }
           |      ]
@@ -192,7 +193,7 @@ class PruningAlgTest extends FunSuite {
             DependencyInfo("org.scala-lang".g % "scala-library".a % "2.12.14", List("build.sbt")),
             DependencyInfo("org.scala-lang".g % "scala-library".a % "2.13.5", List("build.sbt"))
           ),
-          List(MavenRepository("public", "https://repo5.org/maven/", None))
+          List(MavenRepository("public", "https://repo5.org/maven/", None, Nil))
         )
       )
     )
@@ -243,7 +244,8 @@ class PruningAlgTest extends FunSuite {
           |        {
           |          "MavenRepository" : {
           |            "name" : "public",
-          |            "location" : "https://foobar.org/maven2/"
+          |            "location" : "https://foobar.org/maven2/",
+          |            "headers" : []
           |          }
           |        }
           |      ]
@@ -354,7 +356,8 @@ class PruningAlgTest extends FunSuite {
           |        {
           |          "MavenRepository" : {
           |            "name" : "public",
-          |            "location" : "https://foobar.org/maven2/"
+          |            "location" : "https://foobar.org/maven2/",
+          |            "headers" : []
           |          }
           |        }
           |      ]
@@ -464,7 +467,8 @@ class PruningAlgTest extends FunSuite {
           |        {
           |          "MavenRepository" : {
           |            "name" : "public",
-          |            "location" : "https://foobar.org/maven2/"
+          |            "location" : "https://foobar.org/maven2/",
+          |            "headers" : []
           |          }
           |        }
           |      ]

modules/docs/mdoc/running.md

@@ -199,7 +199,7 @@ There is multiple articles on how to run Scala Steward on-premise:
 
 #### GitLab
 
-The following describes a setup using GitLab Docker runner, which you have to setup seperately.
+The following describes a setup using GitLab Docker runner, which you have to set up separately.
 
 1. create a "scalasteward" user in GitLab
 2. assign that user "Developer" permissions in every project that should be managed by Scala Steward
@@ -260,6 +260,33 @@ echo "${SCALA_STEWARD_TOKEN}"
 7. add the `repos.md` file 
 8. (*optional*) create a new schedule to trigger the pipeline on a daily/weekly basis
 
+Scala Steward is compatible with Coursier authentication using headers. To authenticate
+using the [Gitlab CI/CD job token](https://docs.gitlab.com/ee/ci/jobs/ci_job_token.html), while also supporting your own private token when performing
+local development, use the following snippet:
+```scala
+import lmcoursier.CoursierConfiguration
+import lmcoursier.definitions.Authentication
+
+lazy val gitlabToken: Option[(String, String)] = {
+  //The Gitlab runner sets CI_JOB_TOKEN automatically as part of running inside a build job
+  val jobToken = sys.env.get("CI_JOB_TOKEN").map(t => ("Job-Token", t)) 
+  //When running on your local machine, set the environment variable GITLAB_PRIVATE_TOKEN
+  val privateToken = sys.env.get("GITLAB_PRIVATE_TOKEN").map(t => ("Private-Token", t))
+
+  jobToken.orElse(privateToken)
+}
+
+def addGitlabToken(current: CoursierConfiguration): CoursierConfiguration = {
+  gitlabToken.fold(current) { token =>
+    current.addRepositoryAuthentication("gitlab-repo", Authentication(Seq(token)))
+  }
+}
+
+resolvers += "gitlab-repo" at s"https://gitlab.example.com/api/v4/groups/1/-/packages/maven"
+csrConfiguration ~= addGitlabToken
+updateClassifiers / csrConfiguration ~= addGitlabToken
+updateSbtClassifiers / csrConfiguration ~= addGitlabToken
+```
 
 [scalafixmigrations]: @GITHUB_URL@/blob/@MAIN_BRANCH@/docs/scalafix-migrations.md
 [artifactmigrations]: @GITHUB_URL@/blob/@MAIN_BRANCH@/docs/artifact-migrations.md