Skip to content
Permalink
Browse files

Do not store versions in VersionsCacheAlg (#1241)

This renames VersionsCacheAlg to VersionsCacheFacade and changes it to
not store versions anymore. Versions are now always obtained from
Coursier. VersionsCacheFacade's job now is to keep track of when
versions where updated by Coursier and to rate limit calls to Coursier
that hit the network (because of #1218).

The big advantage of this change is that we don't need to build and
maintain our own versions cache (which would always be inferior to
Coursier's cache). For example, until now we've neglected the
repositories where dependencies are available in our cache which would
have lead to incorrect PRs with #1209.

VersionsCacheFacade still ignores resolvers when storing the timestamp
of the last update, but this will only lead to unlimited calls to
Coursier that might hit the network but not to incorrect data and
behaviour.
  • Loading branch information
fthomas committed Jan 14, 2020
1 parent 556def6 commit 6824d0df186280b588793fa829343366c20bd1a9
@@ -22,7 +22,7 @@ import io.chrisdavenport.log4cats.Logger
import io.chrisdavenport.log4cats.slf4j.Slf4jLogger
import org.http4s.client.Client
import org.http4s.client.asynchttpclient.AsyncHttpClient
import org.scalasteward.core.coursier.CoursierAlg
import org.scalasteward.core.coursier.{CoursierAlg, VersionsCacheFacade}
import org.scalasteward.core.edit.EditAlg
import org.scalasteward.core.git.GitAlg
import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg}
@@ -33,7 +33,7 @@ import org.scalasteward.core.repoconfig.RepoConfigAlg
import org.scalasteward.core.sbt.SbtAlg
import org.scalasteward.core.scalafix.MigrationAlg
import org.scalasteward.core.scalafmt.ScalafmtAlg
import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg, VersionsCacheAlg}
import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg}
import org.scalasteward.core.util._
import org.scalasteward.core.vcs.data.AuthenticatedUser
import org.scalasteward.core.vcs.{VCSApiAlg, VCSExtraAlg, VCSRepoAlg, VCSSelection}
@@ -70,9 +70,13 @@ object Context {
implicit val pullRequestRepository: PullRequestRepository[F] =
new PullRequestRepository[F](new JsonKeyValueStore("pull_requests", "1"))
implicit val scalafmtAlg: ScalafmtAlg[F] = ScalafmtAlg.create[F]
implicit val coursierAlg: CoursierAlg[F] = CoursierAlg.create
implicit val versionsCacheAlg: VersionsCacheAlg[F] =
new VersionsCacheAlg[F](new JsonKeyValueStore("versions", "1"), rateLimiter)
implicit val coursierAlg: CoursierAlg[F] = CoursierAlg.create(config.cacheTtl)
implicit val versionsCacheAlg: VersionsCacheFacade[F] =
new VersionsCacheFacade[F](
config.cacheTtl,
new JsonKeyValueStore("versions", "2"),
rateLimiter
)
implicit val updateAlg: UpdateAlg[F] = new UpdateAlg[F]
implicit val sbtAlg: SbtAlg[F] = SbtAlg.create[F]
implicit val refreshErrorAlg: RefreshErrorAlg[F] =
@@ -19,15 +19,16 @@ package org.scalasteward.core.coursier
import cats.effect._
import cats.implicits._
import cats.{Applicative, Parallel}
import coursier.cache.FileCache
import coursier.core.Project
import coursier.interop.cats._
import coursier.maven.MavenRepository
import coursier.util.StringInterpolators.SafeIvyRepository
import coursier.{Info, Module, ModuleName, Organization}
import coursier.{Fetch, Info, Module, ModuleName, Organization, Versions}
import io.chrisdavenport.log4cats.Logger
import org.http4s.Uri
import org.scalasteward.core.application.Config
import org.scalasteward.core.data.{Dependency, Resolver, Version}
import scala.concurrent.duration.FiniteDuration

/** An interface to [[https://get-coursier.io Coursier]] used for
* fetching dependency versions and metadata.
@@ -37,6 +38,8 @@ trait CoursierAlg[F[_]] {

def getVersions(dependency: Dependency, resolvers: List[Resolver]): F[List[Version]]

def getVersionsFresh(dependency: Dependency, resolvers: List[Resolver]): F[List[Version]]

final def getArtifactIdUrlMapping(dependencies: List[Dependency], resolvers: List[Resolver])(
implicit F: Applicative[F]
): F[Map[String, Uri]] =
@@ -46,19 +49,24 @@ trait CoursierAlg[F[_]] {
}

object CoursierAlg {
def create[F[_]](
def create[F[_]](cacheTtl: FiniteDuration)(
implicit
config: Config,
contextShift: ContextShift[F],
logger: Logger[F],
F: Sync[F]
): CoursierAlg[F] = {
implicit val parallel: Parallel.Aux[F, F] = Parallel.identity[F]
val cache = coursier.cache.FileCache[F]().withTtl(config.cacheTtl)

val sbtPluginReleases =
ivy"https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[defaultPattern]"
val fetch = coursier.Fetch[F](cache).withRepositories(List(sbtPluginReleases))
val versions = coursier.Versions[F](cache).withRepositories(List(sbtPluginReleases))

val cache: FileCache[F] = FileCache[F]().withTtl(cacheTtl)
val cacheNoTtl: FileCache[F] = cache.withTtl(None)

val fetch: Fetch[F] = Fetch[F](cache).withRepositories(List(sbtPluginReleases))

val versions: Versions[F] = Versions[F](cache).withRepositories(List(sbtPluginReleases))
val versionsNoTtl: Versions[F] = versions.withCache(cacheNoTtl)

new CoursierAlg[F] {
override def getArtifactUrl(
@@ -95,12 +103,28 @@ object CoursierAlg {
override def getVersions(
dependency: Dependency,
resolvers: List[Resolver]
): F[List[Version]] =
getVersionsImpl(versions, dependency, resolvers)

override def getVersionsFresh(
dependency: Dependency,
resolvers: List[Resolver]
): F[List[Version]] =
getVersionsImpl(versionsNoTtl, dependency, resolvers)

private def getVersionsImpl(
versions: Versions[F],
dependency: Dependency,
resolvers: List[Resolver]
): F[List[Version]] =
versions
.addRepositories(resolvers.map(toCoursierRepository): _*)
.withModule(toCoursierModule(dependency))
.versions()
.map(_.available.map(Version.apply).sorted)
.handleErrorWith { throwable =>
logger.debug(throwable)(s"Failed to get versions of $dependency").as(List.empty)
}
}
}

@@ -0,0 +1,93 @@
/*
* Copyright 2018-2020 Scala Steward contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.scalasteward.core.coursier

import cats.FlatMap
import cats.implicits._
import io.circe.generic.semiauto.deriveCodec
import io.circe.{Codec, KeyEncoder}
import java.util.concurrent.TimeUnit
import org.scalasteward.core.coursier.VersionsCacheFacade.{Key, Value}
import org.scalasteward.core.data.{Dependency, Resolver, Version}
import org.scalasteward.core.persistence.KeyValueStore
import org.scalasteward.core.util.{DateTimeAlg, RateLimiter}
import scala.concurrent.duration.FiniteDuration

/** Facade of Coursier's versions cache that keeps track of the instant
* when versions of a dependency are updated. This information is used
* to rate limit calls to Coursier that require a network call to
* populate or refresh its cache. Calls that hit the cache are
* unlimited.
*
* Note that resolvers are ignored when storing the instant of the
* last updates which could lead to unlimited calls to Coursier that
* hit the network. This will only happen for dependencies that are
* available in multiple repositories.
*/
final class VersionsCacheFacade[F[_]](
cacheTtl: FiniteDuration,
store: KeyValueStore[F, Key, Value],
rateLimiter: RateLimiter[F]
)(
implicit
coursierAlg: CoursierAlg[F],
dateTimeAlg: DateTimeAlg[F],
F: FlatMap[F]
) {
def getVersions(dependency: Dependency, resolvers: List[Resolver]): F[List[Version]] =
dateTimeAlg.currentTimeMillis.flatMap { now =>
store.get(Key(dependency)).flatMap {
case Some(value) if value.age(now) <= cacheTtl =>
coursierAlg.getVersions(dependency, resolvers)
case _ =>
getVersionsFresh(dependency, resolvers)
}
}

def getVersionsFresh(dependency: Dependency, resolvers: List[Resolver]): F[List[Version]] =
rateLimiter.limit {
dateTimeAlg.currentTimeMillis.flatMap { now =>
coursierAlg.getVersionsFresh(dependency, resolvers) <*
store.put(Key(dependency), Value(now))
}
}
}

object VersionsCacheFacade {
final case class Key(dependency: Dependency) {
override def toString: String =
dependency.groupId.value.replace('.', '/') + "/" +
dependency.artifactId.crossName +
dependency.scalaVersion.fold("")("_" + _.value) +
dependency.sbtVersion.fold("")("_" + _.value)
}

object Key {
implicit val keyKeyEncoder: KeyEncoder[Key] =
KeyEncoder.instance(_.toString)
}

final case class Value(updatedAt: Long) {
def age(now: Long): FiniteDuration =
FiniteDuration(now - updatedAt, TimeUnit.MILLISECONDS)
}

object Value {
implicit val valueCodec: Codec[Value] =
deriveCodec
}
}
@@ -19,6 +19,7 @@ package org.scalasteward.core.update
import cats.implicits._
import cats.{Monad, Parallel}
import io.chrisdavenport.log4cats.Logger
import org.scalasteward.core.coursier.VersionsCacheFacade
import org.scalasteward.core.data._
import org.scalasteward.core.repoconfig.RepoConfig
import org.scalasteward.core.util
@@ -29,14 +30,16 @@ final class UpdateAlg[F[_]](
filterAlg: FilterAlg[F],
logger: Logger[F],
parallel: Parallel[F],
versionsCacheAlg: VersionsCacheAlg[F],
versionsCache: VersionsCacheFacade[F],
F: Monad[F]
) {
def findUpdate(dependency: Dependency, resolvers: List[Resolver]): F[Option[Update.Single]] =
for {
newerVersions0 <- versionsCacheAlg.getNewerVersions(dependency, resolvers)
maybeUpdate0 = Nel.fromList(newerVersions0).map { newerVersions1 =>
Update.Single(CrossDependency(dependency), newerVersions1.map(_.value))
versions <- versionsCache.getVersions(dependency, resolvers)
current = Version(dependency.version)
maybeNewerVersions = Nel.fromList(versions.filter(_ > current))
maybeUpdate0 = maybeNewerVersions.map { newerVersions =>
Update.Single(CrossDependency(dependency), newerVersions.map(_.value))
}
maybeUpdate1 = maybeUpdate0.orElse(UpdateAlg.findUpdateWithNewerGroupId(dependency))
} yield maybeUpdate1

This file was deleted.

@@ -7,7 +7,7 @@ import org.http4s.Uri
import org.scalasteward.core.TestInstances.ioContextShift
import org.scalasteward.core.application.Cli.EnvVar
import org.scalasteward.core.application.{Config, SupportedVCS}
import org.scalasteward.core.coursier.CoursierAlg
import org.scalasteward.core.coursier.{CoursierAlg, VersionsCacheFacade}
import org.scalasteward.core.edit.EditAlg
import org.scalasteward.core.git.{Author, GitAlg}
import org.scalasteward.core.io.{MockFileAlg, MockProcessAlg, MockWorkspaceAlg}
@@ -18,7 +18,7 @@ import org.scalasteward.core.repoconfig.RepoConfigAlg
import org.scalasteward.core.sbt.SbtAlg
import org.scalasteward.core.scalafix.MigrationAlg
import org.scalasteward.core.scalafmt.ScalafmtAlg
import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg, VersionsCacheAlg}
import org.scalasteward.core.update.{FilterAlg, PruningAlg, UpdateAlg}
import org.scalasteward.core.util.{BracketThrowable, DateTimeAlg, RateLimiter}
import org.scalasteward.core.vcs.VCSRepoAlg
import org.scalasteward.core.vcs.data.AuthenticatedUser
@@ -61,7 +61,7 @@ object MockContext {
implicit val processAlg: MockProcessAlg = new MockProcessAlg
implicit val workspaceAlg: MockWorkspaceAlg = new MockWorkspaceAlg

implicit val coursierAlg: CoursierAlg[MockEff] = CoursierAlg.create
implicit val coursierAlg: CoursierAlg[MockEff] = CoursierAlg.create(config.cacheTtl)
implicit val dateTimeAlg: DateTimeAlg[MockEff] = DateTimeAlg.create
implicit val gitAlg: GitAlg[MockEff] = GitAlg.create
implicit val user: AuthenticatedUser = AuthenticatedUser("scala-steward", "token")
@@ -71,8 +71,12 @@ object MockContext {
implicit val cacheRepository: RepoCacheRepository[MockEff] =
new RepoCacheRepository[MockEff](new JsonKeyValueStore("repo_cache", "1"))
implicit val filterAlg: FilterAlg[MockEff] = new FilterAlg[MockEff]
implicit val versionsCacheAlg: VersionsCacheAlg[MockEff] =
new VersionsCacheAlg[MockEff](new JsonKeyValueStore("versions", "1"), nopLimiter)
implicit val versionsCacheAlg: VersionsCacheFacade[MockEff] =
new VersionsCacheFacade[MockEff](
config.cacheTtl,
new JsonKeyValueStore("versions", "1"),
nopLimiter
)
implicit val updateAlg: UpdateAlg[MockEff] = new UpdateAlg[MockEff]
implicit val sbtAlg: SbtAlg[MockEff] = SbtAlg.create
implicit val editAlg: EditAlg[MockEff] = new EditAlg[MockEff]

0 comments on commit 6824d0d

Please sign in to comment.
You can’t perform that action at this time.