Skip to content
Permalink
Browse files

Add in-memory cache to HttpExistenceClient

This adds an in-memory cache to HttpExistenceClient which caches the
status codes of HEAD requests it does. The effect of this cache is that
repeated calls to `VCSExtraAlg.getReleaseNoteUrl` for example now only
take a few milliseconds:

```scala
scala> dateTimeAlg.timed(vcsExtraAlg.getReleaseNoteUrl("https://github.com/fthomas/refined", data.Update.Single(data.GroupId("eu.timepit"), "refined", "0.9.9", Nel.of("0.9.10")))).unsafeRunSync
res0: (Option[String], scala.concurrent.duration.FiniteDuration) =
  (Some(https://github.com/fthomas/refined/releases/tag/v0.9.10),7997 milliseconds)

scala> dateTimeAlg.timed(vcsExtraAlg.getReleaseNoteUrl("https://github.com/fthomas/refined", data.Update.Single(data.GroupId("eu.timepit"), "refined", "0.9.9", Nel.of("0.9.10")))).unsafeRunSync
res1: (Option[String], scala.concurrent.duration.FiniteDuration) =
  (Some(https://github.com/fthomas/refined/releases/tag/v0.9.10),8 milliseconds)
```

Closes #941.
  • Loading branch information...
fthomas committed Nov 6, 2019
1 parent c3c1b26 commit 77a952d9d9d289dacee27770a051b40c10e2b6d6
@@ -45,6 +45,8 @@ lazy val core = myCrossProject("core")
Dependencies.monocleCore,
Dependencies.refined,
Dependencies.refinedCats,
Dependencies.scalacacheCaffeine,
Dependencies.scalacacheCatsEffect,
Dependencies.logbackClassic % Runtime,
Dependencies.catsKernelLaws % Test,
Dependencies.circeLiteral % Test,
@@ -73,16 +75,23 @@ lazy val core = myCrossProject("core")
buildInfoPackage := moduleRootPkg.value,
initialCommands += s"""
import ${moduleRootPkg.value}._
import ${moduleRootPkg.value}.data._
import ${moduleRootPkg.value}.util.Nel
import ${moduleRootPkg.value}.vcs.data._
import better.files.File
import cats.effect.ContextShift
import cats.effect.IO
import cats.effect.Timer
import _root_.io.chrisdavenport.log4cats.Logger
import _root_.io.chrisdavenport.log4cats.slf4j.Slf4jLogger
import org.http4s.client.Client
import org.http4s.client.asynchttpclient.AsyncHttpClient
import scala.concurrent.ExecutionContext

implicit val ioContextShift: ContextShift[IO] = IO.contextShift(ExecutionContext.global)
implicit val ioTimer: Timer[IO] = IO.timer(ExecutionContext.global)
implicit val logger: Logger[IO] = Slf4jLogger.getLogger[IO]
implicit val client: Client[IO] = AsyncHttpClient.allocate[IO]().map(_._1).unsafeRunSync
""",
fork in run := true,
fork in Test := true
@@ -46,6 +46,7 @@ object Context {
implicit0(config: Config) <- Resource.liftF(Config.create[F](cliArgs_))
implicit0(client: Client[F]) <- AsyncHttpClient.resource[F]()
implicit0(logger: Logger[F]) <- Resource.liftF(Slf4jLogger.create[F])
implicit0(httpExistenceClient: HttpExistenceClient[F]) <- HttpExistenceClient.create[F]
implicit0(user: AuthenticatedUser) <- Resource.liftF(config.vcsUser[F])
} yield {
implicit val dateTimeAlg: DateTimeAlg[F] = DateTimeAlg.create[F]
@@ -56,7 +57,6 @@ object Context {
implicit val filterAlg: FilterAlg[F] = new FilterAlg[F]
implicit val gitAlg: GitAlg[F] = GitAlg.create[F]
implicit val httpJsonClient: HttpJsonClient[F] = new HttpJsonClient[F]
implicit val httpExistenceClient: HttpExistenceClient[F] = new HttpExistenceClient[F]
implicit val repoCacheRepository: RepoCacheRepository[F] =
new RepoCacheRepository[F](new JsonKeyValueStore("repos", "6"))
val vcsSelection = new VCSSelection[F]
@@ -16,23 +16,46 @@

package org.scalasteward.core.util

import cats.effect.{Async, Resource}
import cats.implicits._
import com.github.benmanes.caffeine.cache.Caffeine
import io.chrisdavenport.log4cats.Logger
import org.http4s.client.Client
import org.http4s.{Method, Request, Status, Uri}
import scalacache.CatsEffect.modes._
import scalacache.caffeine.CaffeineCache
import scalacache.{Async => _, _}

final class HttpExistenceClient[F[_]](
final class HttpExistenceClient[F[_]](statusCache: Cache[Status])(
implicit
client: Client[F],
logger: Logger[F],
mode: Mode[F],
F: MonadThrowable[F]
) {
def exists(uri: String): F[Boolean] = F.fromEither(Uri.fromString(uri)).flatMap(exists)

def exists(uri: Uri): F[Boolean] = {
val req = Request[F](method = Method.HEAD, uri = uri)
client.status(req).map(_ === Status.Ok).handleErrorWith { throwable =>
def exists(uri: Uri): F[Boolean] =
status(uri).map(_ === Status.Ok).handleErrorWith { throwable =>
logger.debug(throwable)(s"Failed to check if $uri exists").as(false)
}

private def status(uri: Uri): F[Status] =
statusCache.cachingForMemoizeF(uri.renderString)(None) {
client.status(Request[F](method = Method.HEAD, uri = uri))
}
}

object HttpExistenceClient {
def create[F[_]](
implicit
client: Client[F],
logger: Logger[F],
F: Async[F]
): Resource[F, HttpExistenceClient[F]] = {
val buildCache = F.delay {
CaffeineCache(Caffeine.newBuilder().maximumSize(16384L).build[String, Entry[Status]]())
}
Resource.make(buildCache)(_.close().void).map(new HttpExistenceClient[F](_))
}
}
@@ -20,7 +20,8 @@ class VCSExtraAlgTest extends AnyFunSuite with Matchers {
}

implicit val client = Client.fromHttpApp[IO](routes.orNotFound)
implicit val httpExistenceClient = new HttpExistenceClient[IO]
implicit val httpExistenceClient =
HttpExistenceClient.create[IO].allocated.map(_._1).unsafeRunSync()

val vcsExtraAlg = VCSExtraAlg.create[IO]
val updateFoo = Update.Single(GroupId("com.example"), "foo", "0.1.0", Nel.of("0.2.0"))
@@ -27,6 +27,8 @@ object Dependencies {
val refined = "eu.timepit" %% "refined" % "0.9.10"
val refinedCats = "eu.timepit" %% "refined-cats" % refined.revision
val refinedScalacheck = "eu.timepit" %% "refined-scalacheck" % refined.revision
val scalacacheCaffeine = "com.github.cb372" %% "scalacache-caffeine" % "0.28.0"
val scalacacheCatsEffect = "com.github.cb372" %% "scalacache-cats-effect" % scalacacheCaffeine.revision
val scalacheck = "org.scalacheck" %% "scalacheck" % "1.14.2"
val scalaTest = "org.scalatest" %% "scalatest" % "3.1.0-RC3"
}

0 comments on commit 77a952d

Please sign in to comment.
You can’t perform that action at this time.