diff --git a/CHANGELOG b/CHANGELOG index f6ef5127..4c896a06 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,11 @@ +Version 3.3.0 (2023-05-30) +-------------------------- +Embedded and in-memory lookups should be common across http modules (#244) +Take superseding schema into account during validation (#231) +Apps should explicitly import java.net.http.HttpClient instances of RegistryLookup (#241) +Resolver caches suffers from races and http server overload during the cold start (#227) +Ignore `$ref` keyword referencing HTTP resources (#238) + Version 2.2.1 (2023-01-24) -------------------------- Update links in Readme (#205) diff --git a/README.md b/README.md index fcd443be..2cf0f7cf 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ Iglu Scala Client is used extensively in **[Snowplow][snowplow-repo]** to valida ## Installation -The latest version of Iglu Scala Client is 2.2.1, which works with Scala 2.12, 2.13, and 3. +The latest version of Iglu Scala Client is 3.0.0, which works with Scala 2.12, 2.13, and 3. If you're using SBT, add the following lines to your build file: ```scala -val igluClient = "com.snowplowanalytics" %% "iglu-scala-client" % "2.2.1" +val igluClient = "com.snowplowanalytics" %% "iglu-scala-client" % "3.0.0" ``` ## API @@ -65,8 +65,7 @@ import cats.syntax.show._ import com.snowplowanalytics.iglu.client.Client import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} - -implicit val clockIoInstance: Clock[IO] = Clock.create[IO] // Usually provided by IOApp +import com.snowplowanalytics.iglu.client.resolver.registries.JavaNetRegistryLookup._ val resolverConfig: Json = json"""{ "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-1", diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/Client.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/Client.scala index 5ba0f831..980624dc 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/Client.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/Client.scala @@ -17,7 +17,7 @@ import cats.data.EitherT import cats.effect.{Clock, IO} import io.circe.{DecodingFailure, Json} import com.snowplowanalytics.iglu.core.SelfDescribingData -import resolver.{InitListCache, InitSchemaCache} +import resolver.CreateResolverCache import resolver.registries.{Registry, RegistryLookup} /** @@ -39,9 +39,9 @@ final case class Client[F[_], A](resolver: Resolver[F], validator: Validator[A]) for { schema <- EitherT(resolver.lookupSchema(instance.schema)) schemaValidation = validator.validateSchema(schema) - _ <- EitherT.fromEither[F](schemaValidation).leftMap(_.toClientError) + _ <- EitherT.fromEither[F](schemaValidation).leftMap(_.toClientError(None)) validation = validator.validate(instance.data, schema) - _ <- EitherT.fromEither[F](validation).leftMap(_.toClientError) + _ <- EitherT.fromEither[F](validation).leftMap(_.toClientError(None)) } yield () } @@ -51,7 +51,7 @@ object Client { val IgluCentral: Client[IO, Json] = Client[IO, Json](Resolver(List(Registry.IgluCentral), None), CirceValidator) - def parseDefault[F[_]: Monad: InitSchemaCache: InitListCache]( + def parseDefault[F[_]: Monad: CreateResolverCache]( json: Json ): EitherT[F, DecodingFailure, Client[F, Json]] = EitherT(Resolver.parse[F](json)).map { resolver => diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/IgluCirceClient.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/IgluCirceClient.scala index 73ecd5bf..3d39cacf 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/IgluCirceClient.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/IgluCirceClient.scala @@ -17,7 +17,8 @@ import cats.data.EitherT import cats.effect.Clock import cats.implicits._ import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup -import com.snowplowanalytics.iglu.client.resolver.{InitListCache, InitSchemaCache} +import com.snowplowanalytics.iglu.client.resolver.CreateResolverCache +import com.snowplowanalytics.iglu.client.resolver.Resolver.SupersededBy import com.snowplowanalytics.iglu.client.validator.CirceValidator.WithCaching.{ InitValidatorCache, SchemaEvaluationCache, @@ -43,18 +44,24 @@ final class IgluCirceClient[F[_]] private ( M: Monad[F], L: RegistryLookup[F], C: Clock[F] - ): EitherT[F, ClientError, Unit] = + ): EitherT[F, ClientError, SupersededBy] = for { - resolverResult <- EitherT(resolver.lookupSchemaResult(instance.schema)) + resolverResult <- EitherT( + resolver.lookupSchemaResult(instance.schema, resolveSupersedingSchema = true) + ) validation = CirceValidator.WithCaching.validate(schemaEvaluationCache)(instance.data, resolverResult) - _ <- EitherT(validation).leftMap(_.toClientError) - } yield () + _ <- EitherT(validation).leftMap(e => + e.toClientError(resolverResult.value.supersededBy.map(_.asString)) + ) + // Returning superseding schema info as well since we want to inform caller that sdj is validated + // against superseding schema if it is superseded by another schema. + } yield resolverResult.value.supersededBy } object IgluCirceClient { - def parseDefault[F[_]: Monad: InitSchemaCache: InitListCache: InitValidatorCache]( + def parseDefault[F[_]: Monad: CreateResolverCache: InitValidatorCache]( json: Json ): EitherT[F, DecodingFailure, IgluCirceClient[F]] = for { diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/CreateResolverCache.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/CreateResolverCache.scala new file mode 100644 index 00000000..caf7543b --- /dev/null +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/CreateResolverCache.scala @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2018-2023 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.iglu.client.resolver + +import cats.Id +import cats.effect.Async +import cats.effect.std.Mutex +import cats.implicits._ +import com.snowplowanalytics.lrumap.{CreateLruMap, LruMap} +import com.snowplowanalytics.iglu.core.SchemaKey + +trait CreateResolverCache[F[_]] { + + def createSchemaCache(size: Int): F[LruMap[F, SchemaKey, SchemaCacheEntry]] + + def createSchemaListCache(size: Int): F[LruMap[F, ListCacheKey, ListCacheEntry]] + + def createMutex[K]: F[ResolverMutex[F, K]] + +} + +object CreateResolverCache { + + def apply[F[_]](implicit instance: CreateResolverCache[F]): CreateResolverCache[F] = instance + + private trait SimpleCreateResolverCache[F[_]] extends CreateResolverCache[F] { + + def createLruMap[K, V](size: Int): F[LruMap[F, K, V]] + + override def createSchemaCache(size: Int): F[LruMap[F, SchemaKey, SchemaCacheEntry]] = + createLruMap(size) + + override def createSchemaListCache(size: Int): F[LruMap[F, ListCacheKey, ListCacheEntry]] = + createLruMap(size) + + } + + implicit def idCreateResolverCache: CreateResolverCache[Id] = + new SimpleCreateResolverCache[Id] { + def createLruMap[K, V](size: Int): LruMap[Id, K, V] = + CreateLruMap[Id, K, V].create(size) + + def createMutex[K]: ResolverMutex[Id, K] = + ResolverMutex.idResolverMutex(new Object, createLruMap[K, Object](1000)) + } + + implicit def asyncCreateResolverCache[F[_]: Async]: CreateResolverCache[F] = + new SimpleCreateResolverCache[F] { + + def createLruMap[K, V](size: Int): F[LruMap[F, K, V]] = + CreateLruMap[F, K, V].create(size) + + def createMutex[K]: F[ResolverMutex[F, K]] = + for { + mutex <- Mutex[F] + lrumap <- createLruMap[K, Mutex[F]](1000) + } yield ResolverMutex.asyncResolverMutex(mutex, lrumap) + } +} diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/Resolver.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/Resolver.scala index 6a65d677..cf70bb39 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/Resolver.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/Resolver.scala @@ -18,7 +18,6 @@ import cats.effect.Clock import cats.implicits._ import cats.{Applicative, Id, Monad} import com.snowplowanalytics.iglu.client.ClientError.ResolutionError -import com.snowplowanalytics.iglu.client.resolver.registries.Registry.Get import com.snowplowanalytics.iglu.client.resolver.ResolverCache.TimestampedItem import com.snowplowanalytics.iglu.client.resolver.registries.{ Registry, @@ -27,7 +26,7 @@ import com.snowplowanalytics.iglu.client.resolver.registries.{ } import com.snowplowanalytics.iglu.core.circe.CirceIgluCodecs._ import com.snowplowanalytics.iglu.core._ -import io.circe.{Decoder, DecodingFailure, HCursor, Json} +import io.circe.{Decoder, DecodingFailure, FailedCursor, HCursor, Json} import java.time.Instant import scala.collection.immutable.SortedMap @@ -44,19 +43,69 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac * If any of repositories gives non-non-found error, lookup will retried * * @param schemaKey The SchemaKey uniquely identifying the schema in Iglu + * @param resolveSupersedingSchema Specify whether superseding schema version should be taken into account * @return a [[Resolver.ResolverResult]] boxing the schema Json on success, or a ResolutionError on failure */ def lookupSchemaResult( - schemaKey: SchemaKey + schemaKey: SchemaKey, + resolveSupersedingSchema: Boolean = false )(implicit F: Monad[F], L: RegistryLookup[F], C: Clock[F] ): F[Either[ResolutionError, SchemaLookupResult]] = { - val get: Registry => F[Either[RegistryError, Json]] = r => L.lookup(r, schemaKey) + def extractSupersededBy(schema: Json): Either[RegistryError, SupersededBy] = + schema.hcursor.downField("$supersededBy") match { + case _: FailedCursor => None.asRight + case c => + c.as[SchemaVer.Full] + .bimap( + e => + RegistryError.ClientFailure( + s"Error while trying to decode superseding version: ${e.toString()}" + ), + _.some + ) + } + + def checkSupersedingVersion( + schemaKey: SchemaKey, + supersededBy: SupersededBy + ): Either[RegistryError, Unit] = + supersededBy match { + case None => ().asRight + case Some(superseding) => + if (Ordering[SchemaVer.Full].gt(superseding, schemaKey.version)) ().asRight + else + RegistryError + .ClientFailure( + s"Superseding version ${superseding.asString} isn't greater than the version of schema ${schemaKey.toPath}" + ) + .asLeft + } + + val get: Registry => F[Either[RegistryError, SchemaItem]] = { + if (resolveSupersedingSchema) + r => + (for { + schema <- EitherT(L.lookup(r, schemaKey)) + supersededByOpt <- EitherT.fromEither[F](extractSupersededBy(schema)) + _ <- EitherT.fromEither[F](checkSupersedingVersion(schemaKey, supersededByOpt)) + res <- supersededByOpt match { + case None => + EitherT.rightT[F, RegistryError](SchemaItem(schema, Option.empty[SchemaVer.Full])) + case Some(supersededBy) => + val supersedingSchemaKey = schemaKey.copy(version = supersededBy) + EitherT(L.lookup(r, supersedingSchemaKey)) + .map(supersedingSchema => SchemaItem(supersedingSchema, supersededBy.some)) + } + } yield res).value + else + r => EitherT(L.lookup(r, schemaKey)).map(s => SchemaItem(s, Option.empty)).value + } def handleAfterFetch( - result: Either[LookupFailureMap, Json] + result: Either[LookupFailureMap, SchemaItem] ): F[Either[ResolutionError, SchemaLookupResult]] = cache match { case Some(c) => @@ -74,15 +123,43 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac .pure[F] } + def lockAndLookup: F[Either[ResolutionError, SchemaLookupResult]] = + withLockOnSchemaKey(schemaKey) { + getSchemaFromCache(schemaKey).flatMap { + case Some(TimestampedItem(Right(schema), timestamp)) => + Monad[F].pure(Right(ResolverResult.Cached(schemaKey, schema, timestamp))) + case Some(TimestampedItem(Left(failures), _)) => + for { + toBeRetried <- reposForRetry(failures) + result <- traverseRepos[F, SchemaItem]( + get, + prioritize(schemaKey.vendor, toBeRetried), + failures + ) + fixed <- handleAfterFetch(result) + } yield fixed + case None => + traverseRepos[F, SchemaItem]( + get, + prioritize(schemaKey.vendor, allRepos.toList), + Map.empty + ) + .flatMap(handleAfterFetch) + } + } + getSchemaFromCache(schemaKey).flatMap { case Some(TimestampedItem(Right(schema), timestamp)) => Monad[F].pure(Right(ResolverResult.Cached(schemaKey, schema, timestamp))) case Some(TimestampedItem(Left(failures), _)) => - retryCached[F, Json](get, schemaKey.vendor)(failures) - .flatMap(handleAfterFetch) + reposForRetry(failures).flatMap { + case Nil => + Monad[F].pure(Left(resolutionError(failures))) + case _ => + lockAndLookup + } case None => - traverseRepos[F, Json](get, prioritize(schemaKey.vendor, allRepos.toList), Map.empty) - .flatMap(handleAfterFetch) + lockAndLookup } } @@ -102,7 +179,7 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac L: RegistryLookup[F], C: Clock[F] ): F[Either[ResolutionError, Json]] = - lookupSchemaResult(schemaKey).map(_.map(_.value)) + lookupSchemaResult(schemaKey).map(_.map(_.value.schema)) /** * Get list of available schemas for particular vendor and name part @@ -164,19 +241,44 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac .pure[F] } + def lockAndLookup: F[Either[ResolutionError, SchemaListLookupResult]] = + withLockOnSchemaModel(vendor, name, model) { + getSchemaListFromCache(vendor, name, model).flatMap { + case Some(TimestampedItem(Right(schemaList), timestamp)) => + if (mustIncludeKey.forall(schemaList.schemas.contains)) + Monad[F].pure( + Right(ResolverResult.Cached((vendor, name, model), schemaList, timestamp)) + ) + else + traverseRepos[F, SchemaList](get, prioritize(vendor, allRepos.toList), Map.empty) + .flatMap(handleAfterFetch) + case Some(TimestampedItem(Left(failures), _)) => + for { + toBeRetried <- reposForRetry(failures) + result <- traverseRepos[F, SchemaList](get, prioritize(vendor, toBeRetried), failures) + fixed <- handleAfterFetch(result) + } yield fixed + case None => + traverseRepos[F, SchemaList](get, prioritize(vendor, allRepos.toList), Map.empty) + .flatMap(handleAfterFetch) + } + } + getSchemaListFromCache(vendor, name, model).flatMap { case Some(TimestampedItem(Right(schemaList), timestamp)) => if (mustIncludeKey.forall(schemaList.schemas.contains)) Monad[F].pure(Right(ResolverResult.Cached((vendor, name, model), schemaList, timestamp))) else - traverseRepos[F, SchemaList](get, prioritize(vendor, allRepos.toList), Map.empty) - .flatMap(handleAfterFetch) + lockAndLookup case Some(TimestampedItem(Left(failures), _)) => - retryCached[F, SchemaList](get, vendor)(failures) - .flatMap(handleAfterFetch) + reposForRetry(failures).flatMap { + case Nil => + Monad[F].pure(Left(resolutionError(failures))) + case _ => + lockAndLookup + } case None => - traverseRepos[F, SchemaList](get, prioritize(vendor, allRepos.toList), Map.empty) - .flatMap(handleAfterFetch) + lockAndLookup } } @@ -233,6 +335,18 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac case None => Monad[F].pure(None) } + private def withLockOnSchemaKey[A](schemaKey: SchemaKey)(f: => F[A]): F[A] = + cache match { + case Some(c) => c.withLockOnSchemaKey(schemaKey)(f) + case None => f + } + + private def withLockOnSchemaModel[A](vendor: Vendor, name: Name, model: Model)(f: => F[A]): F[A] = + cache match { + case Some(c) => c.withLockOnSchemaModel(vendor, name, model)(f) + case None => f + } + private def getSchemaListFromCache( vendor: Vendor, name: Name, @@ -252,8 +366,18 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac object Resolver { type SchemaListKey = (Vendor, Name, Model) - type SchemaLookupResult = ResolverResult[SchemaKey, Json] + type SchemaLookupResult = ResolverResult[SchemaKey, SchemaItem] type SchemaListLookupResult = ResolverResult[SchemaListKey, SchemaList] + type SupersededBy = Option[SchemaVer.Full] + + /** + * The result of doing schema lookup + * + * @param schema Schema json + * @param supersededBy Superseding schema version if the schema is superseded by another schema. + * Otherwise, it is None. + */ + case class SchemaItem(schema: Json, supersededBy: SupersededBy) /** The result of doing a lookup with the resolver, carrying information on whether the cache was used */ sealed trait ResolverResult[+K, +A] { @@ -278,17 +402,12 @@ object Resolver { /** The result of a lookup when the resolver is not configured to use a cache */ case class NotCached[A](value: A) extends ResolverResult[Nothing, A] } - def retryCached[F[_]: Clock: Monad: RegistryLookup, A]( - get: Get[F, A], - vendor: Vendor - )( - cachedErrors: LookupFailureMap - ): F[Either[LookupFailureMap, A]] = - for { - now <- Clock[F].realTimeInstant - reposForRetry = getReposForRetry(cachedErrors, now) - result <- traverseRepos[F, A](get, prioritize(vendor, reposForRetry), cachedErrors) - } yield result + + private def reposForRetry[F[_]: Clock: Monad](cachedErrors: LookupFailureMap): F[List[Registry]] = + Clock[F].realTimeInstant + .map { now => + getReposForRetry(cachedErrors, now) + } /** * Tail-recursive function to find our schema in one of our repositories @@ -353,7 +472,7 @@ object Resolver { * @param refs Any RepositoryRef to add to this resolver * @return a configured Resolver instance */ - def init[F[_]: Monad: InitSchemaCache: InitListCache]( + def init[F[_]: Monad: CreateResolverCache]( cacheSize: Int, cacheTtl: Option[TTL], refs: Registry* @@ -370,7 +489,7 @@ object Resolver { private[client] val EmbeddedSchemaCount = 4 /** A Resolver which only looks at our embedded repo */ - def bootstrap[F[_]: Monad: InitSchemaCache: InitListCache]: F[Resolver[F]] = + def bootstrap[F[_]: Monad: CreateResolverCache]: F[Resolver[F]] = Resolver.init[F](EmbeddedSchemaCount, None, Registry.EmbeddedRegistry) final case class ResolverConfig( @@ -398,7 +517,7 @@ object Resolver { * for this resolver * @return a configured Resolver instance */ - def parse[F[_]: Monad: InitSchemaCache: InitListCache]( + def parse[F[_]: Monad: CreateResolverCache]( config: Json ): F[Either[DecodingFailure, Resolver[F]]] = { val result: EitherT[F, DecodingFailure, Resolver[F]] = for { @@ -419,7 +538,7 @@ object Resolver { } yield config } - def fromConfig[F[_]: Monad: InitSchemaCache: InitListCache]( + def fromConfig[F[_]: Monad: CreateResolverCache]( config: ResolverConfig ): EitherT[F, DecodingFailure, Resolver[F]] = { for { @@ -498,6 +617,7 @@ object Resolver { private val MinBackoff = 500 // ms // Count how many milliseconds the Resolver needs to wait before retrying + // TODO: This should not exceed TTL private def backoff(retryCount: Int): Long = retryCount match { case c if c > 20 => 1200000L + (retryCount * 100L) diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/ResolverCache.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/ResolverCache.scala index faa4c0a9..c0933451 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/ResolverCache.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/ResolverCache.scala @@ -17,17 +17,16 @@ import cats.{Applicative, Monad} import cats.data.OptionT import cats.effect.Clock import cats.implicits._ -import com.snowplowanalytics.iglu.core.SchemaList -// circe -import io.circe.Json import scala.concurrent.duration.{DurationInt, FiniteDuration} // LruMap -import com.snowplowanalytics.lrumap.{CreateLruMap, LruMap} +import com.snowplowanalytics.lrumap.LruMap // Iglu core -import com.snowplowanalytics.iglu.core.SchemaKey +import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaList} + +import Resolver.SchemaItem /** * Resolver cache and associated logic to (in)validate entities, @@ -39,6 +38,8 @@ import com.snowplowanalytics.iglu.core.SchemaKey class ResolverCache[F[_]] private ( schemas: LruMap[F, SchemaKey, SchemaCacheEntry], schemaLists: LruMap[F, ListCacheKey, ListCacheEntry], + schemaMutex: ResolverMutex[F, SchemaKey], + schemaListMutex: ResolverMutex[F, ListCacheKey], val ttl: Option[TTL] ) { @@ -94,7 +95,7 @@ class ResolverCache[F[_]] private ( )(implicit F: Monad[F], C: Clock[F] - ): F[Either[LookupFailureMap, TimestampedItem[Json]]] = + ): F[Either[LookupFailureMap, TimestampedItem[SchemaItem]]] = putItemResult(schemas, schemaKey, freshResult) /** Lookup a `SchemaList`, no TTL is available */ @@ -133,6 +134,14 @@ class ResolverCache[F[_]] private ( freshResult: ListLookup )(implicit F: Monad[F], C: Clock[F]): F[Either[LookupFailureMap, TimestampedItem[SchemaList]]] = putItemResult(schemaLists, (vendor, name, model), freshResult) + + private[resolver] def withLockOnSchemaKey[A](key: SchemaKey)(f: => F[A]): F[A] = + schemaMutex.withLockOn(key)(f) + + private[resolver] def withLockOnSchemaModel[A](vendor: Vendor, name: Name, model: Model)( + f: => F[A] + ): F[A] = + schemaListMutex.withLockOn((vendor, name, model))(f) } object ResolverCache { @@ -144,14 +153,15 @@ object ResolverCache { size: Int, ttl: Option[TTL] )(implicit - C: CreateLruMap[F, SchemaKey, SchemaCacheEntry], - L: CreateLruMap[F, ListCacheKey, ListCacheEntry] + C: CreateResolverCache[F] ): F[Option[ResolverCache[F]]] = { if (shouldCreateResolverCache(size, ttl)) { for { - schemas <- CreateLruMap[F, SchemaKey, SchemaCacheEntry].create(size) - schemaLists <- CreateLruMap[F, ListCacheKey, ListCacheEntry].create(size) - } yield new ResolverCache[F](schemas, schemaLists, ttl).some + schemas <- C.createSchemaCache(size) + schemaLists <- C.createSchemaListCache(size) + schemaMutex <- C.createMutex[SchemaKey] + listMutex <- C.createMutex[ListCacheKey] + } yield new ResolverCache[F](schemas, schemaLists, schemaMutex, listMutex, ttl).some } else Applicative[F].pure(none) } diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/ResolverMutex.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/ResolverMutex.scala new file mode 100644 index 00000000..252add85 --- /dev/null +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/ResolverMutex.scala @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018-2023 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.iglu.client.resolver + +import cats.Id +import cats.effect.std.Mutex +import cats.effect.Async +import cats.implicits._ +import com.snowplowanalytics.lrumap.LruMap + +trait ResolverMutex[F[_], K] { + + def withLockOn[A](key: K)(f: => F[A]): F[A] + +} + +object ResolverMutex { + + private[resolver] def idResolverMutex[K]( + topMutex: Object, + keyedMutex: LruMap[Id, K, Object] + ): ResolverMutex[Id, K] = + new ResolverMutex[Id, K] { + def withLockOn[A](key: K)(f: => A): A = { + val mutexForKey = topMutex.synchronized { + val current = keyedMutex.get(key) + current match { + case Some(o) => + o + case None => + val o = new Object + keyedMutex.put(key, o) + o + } + } + + mutexForKey.synchronized(f) + } + } + + private[resolver] def asyncResolverMutex[F[_]: Async, K]( + topMutex: Mutex[F], + keyedMutex: LruMap[F, K, Mutex[F]] + ): ResolverMutex[F, K] = + new ResolverMutex[F, K] { + + def withLockOn[A](key: K)(f: => F[A]): F[A] = + topMutex.lock + .surround { + for { + current <- keyedMutex.get(key) + fixed <- current match { + case Some(m) => Async[F].pure(m) + case None => + for { + m <- Mutex[F] + _ <- keyedMutex.put(key, m) + } yield m + } + } yield fixed + } + .flatMap { mutexForKey => + mutexForKey.lock.surround(f) + } + } + +} diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/package.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/package.scala index 83bca016..b0127c60 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/package.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/package.scala @@ -12,19 +12,14 @@ */ package com.snowplowanalytics.iglu.client -// circe -import io.circe.Json - import scala.concurrent.duration.FiniteDuration -// LRU -import com.snowplowanalytics.lrumap.CreateLruMap - // Iglu Core -import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaList} +import com.snowplowanalytics.iglu.core.SchemaList // This project import resolver.registries.Registry +import resolver.Resolver.SchemaItem package object resolver { @@ -49,7 +44,7 @@ package object resolver { * Json in case of success or Map of all currently failed repositories * in case of failure */ - type SchemaLookup = Either[LookupFailureMap, Json] + type SchemaLookup = Either[LookupFailureMap, SchemaItem] /** * Validated schema list lookup result containing, cache result which is @@ -82,7 +77,4 @@ package object resolver { /** Cache entry for schema list lookup results */ type ListCacheEntry = CacheEntry[ListLookup] - /** Ability to initialize the cache */ - type InitSchemaCache[F[_]] = CreateLruMap[F, SchemaKey, SchemaCacheEntry] - type InitListCache[F[_]] = CreateLruMap[F, ListCacheKey, ListCacheEntry] } diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Embedded.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Embedded.scala new file mode 100644 index 00000000..b52dd5f1 --- /dev/null +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Embedded.scala @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2014-2023 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.iglu.client.resolver.registries + +// Java +import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} + +import java.io.{File, InputStream} +import scala.util.matching.Regex + +// Scala +import scala.io.Source +import scala.util.control.NonFatal + +// Cats +import cats.effect.Sync +import cats.effect.implicits._ +import cats.implicits._ +import cats.data.EitherT + +// circe +import io.circe.parser.parse +import io.circe.Json + +import com.snowplowanalytics.iglu.core.SchemaList + +private[registries] object Embedded { + + /** + * Retrieves an Iglu Schema from the Embedded Iglu Repo as a JSON + * + * @param base path on the local filesystem system + * @param key The SchemaKey uniquely identifying the schema in Iglu + * @return either a `Json` on success, or `RegistryError` in case of any failure + * (i.e. all exceptions should be swallowed by `RegistryError`) + */ + def lookup[F[_]: Sync]( + base: String, + key: SchemaKey + ): F[Either[RegistryError, Json]] = { + val path = RegistryLookup.toPath(base, key) + val is = readResource[F](path) + val schema = is.bracket(_.traverse(fromStream[F]))(_.traverse_(closeStream[F])) + val result = for { + stringOption <- schema.attemptT.leftMap(Utils.repoFailure) + string <- EitherT.fromOption[F](stringOption, RegistryError.NotFound: RegistryError) + json <- EitherT.fromEither[F](parse(string)).leftMap(Utils.invalidSchema) + } yield json + + result.value + } + + /** Not-RT analog of [[Embedded.lookup]] */ + def unsafeLookup(path: String): Either[RegistryError, Json] = + try { + val is = unsafeReadResource(path) + val schema = is.map(unsafeFromStream) + val result = schema + .toRight(RegistryError.NotFound: RegistryError) + .flatMap(x => parse(x).leftMap(Utils.invalidSchema)) + is.fold(())(unsafeCloseStream) + result + } catch { + case NonFatal(e) => + e match { + case _: NullPointerException => RegistryError.NotFound.asLeft + case _ => Utils.repoFailure(e).asLeft + } + } + + def unsafeList(path: String, modelMatch: Int): Either[RegistryError, SchemaList] = + try { + val d = + new File( + getClass.getResource(path).getPath + ) // this will throw NPE for missing entry in embedded repos + val schemaFileRegex: Regex = (".*/schemas/?" + // path to file + "([a-zA-Z0-9-_.]+)/" + // Vendor + "([a-zA-Z0-9-_]+)/" + // Name + "([a-zA-Z0-9-_]+)/" + // Format + "([1-9][0-9]*)-(\\d+)-(\\d+)$").r // MODEL, REVISION and ADDITION + + def getFolderContent(d: File): List[String] = { + d.listFiles + .filter(_.isFile) + .toList + .filter(_.getName.startsWith(s"${modelMatch.toString}-")) + .map(_.getAbsolutePath) + } + + val content = + if (d.exists & d.isDirectory) + getFolderContent(d) + else + List.empty[String] + + content + .traverse { + case schemaFileRegex(vendor, name, format, model, revision, addition) + if model == modelMatch.toString => + SchemaKey( + vendor = vendor, + name = name, + format = format, + version = SchemaVer + .Full(model = model.toInt, revision = revision.toInt, addition = addition.toInt) + ).asRight + case f => RegistryError.RepoFailure(s"Corrupted schema file name at $f").asLeft + } + .map(_.sortBy(_.version)) + .flatMap(s => + if (s.isEmpty) + RegistryError.NotFound.asLeft + else + s.asRight + ) + .map(SchemaList.parseUnsafe) + } catch { + case NonFatal(e) => + e match { + case _: NullPointerException => RegistryError.NotFound.asLeft + case _ => Utils.repoFailure(e).asLeft + } + } + + private def readResource[F[_]: Sync](path: String): F[Option[InputStream]] = + Sync[F].delay(unsafeReadResource(path)) + private def unsafeReadResource(path: String): Option[InputStream] = + Option(getClass.getResource(path)).map(_.openStream()) + + private def fromStream[F[_]: Sync](is: InputStream): F[String] = + Sync[F].delay(unsafeFromStream(is)) + private def unsafeFromStream(is: InputStream): String = + Source.fromInputStream(is).mkString + + private def closeStream[F[_]: Sync](is: InputStream): F[Unit] = + Sync[F].delay(unsafeCloseStream(is)) + private def unsafeCloseStream(is: InputStream): Unit = + is.close() + +} diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/JavaNetRegistryLookup.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/JavaNetRegistryLookup.scala new file mode 100644 index 00000000..3c0c0432 --- /dev/null +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/JavaNetRegistryLookup.scala @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2014-2023 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.iglu.client.resolver.registries + +import cats.effect.Sync +import cats.Id +import cats.data.OptionT +import cats.implicits._ +import io.circe.Json +import io.circe.parser.parse + +import com.snowplowanalytics.iglu.core.circe.implicits._ +import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaList} + +import java.net.UnknownHostException +import java.net.URI +import java.net.http.HttpResponse.BodyHandlers +import java.net.http.{HttpClient, HttpRequest, HttpResponse} +import java.time.Duration + +import scala.util.control.NonFatal + +object JavaNetRegistryLookup { + + private val ReadTimeoutMs = 4000L + + private lazy val httpClient = HttpClient + .newBuilder() + .connectTimeout(Duration.ofMillis(1000)) + .build() + + implicit def ioLookupInstance[F[_]](implicit F: Sync[F]): RegistryLookup[F] = + new RegistryLookup.StdRegistryLookup[F] { + def httpLookup( + registry: Registry.Http, + schemaKey: SchemaKey + ): F[Either[RegistryError, Json]] = + lookupImpl(registry.http, schemaKey) + + def httpList( + registry: Registry.Http, + vendor: String, + name: String, + model: Int + ): F[Either[RegistryError, SchemaList]] = + listImpl(registry.http, vendor, name, model) + } + + // Id instance also swallows all exceptions into `RegistryError` + implicit def idLookupInstance: RegistryLookup[Id] = + new RegistryLookup[Id] { + def lookup(repositoryRef: Registry, schemaKey: SchemaKey): Id[Either[RegistryError, Json]] = + repositoryRef match { + case Registry.Http(_, connection) => + Utils + .stringToUri(RegistryLookup.toPath(connection.uri.toString, schemaKey)) + .flatMap(uri => unsafeGetFromUri(uri, connection.apikey)) + case Registry.Embedded(_, base) => + val path = RegistryLookup.toPath(base, schemaKey) + Embedded.unsafeLookup(path) + case Registry.InMemory(_, schemas) => + RegistryLookup.inMemoryLookup(schemas, schemaKey) + } + + def list( + registry: Registry, + vendor: String, + name: String, + model: Int + ): Id[Either[RegistryError, SchemaList]] = + registry match { + case Registry.Http(_, connection) => + val subpath = RegistryLookup.toSubpath(connection.uri.toString, vendor, name, model) + Utils.stringToUri(subpath).flatMap(unsafeHttpList(_, connection.apikey)) + case Registry.Embedded(_, base) => + val path = RegistryLookup.toSubpath(base, vendor, name) + Embedded.unsafeList(path, model) + case _ => + RegistryError.NotFound.asLeft + } + } + + /** + * Retrieves an Iglu Schema from the HTTP Iglu Repo as a JSON + * + * @param http endpoint and optional apikey + * @param key The SchemaKey uniquely identifying the schema in Iglu + * @return either a `Json` on success, or `RegistryError` in case of any failure + * (i.e. all exceptions should be swallowed by `RegistryError`) + */ + private def lookupImpl[F[_]: Sync]( + http: Registry.HttpConnection, + key: SchemaKey + ): F[Either[RegistryError, Json]] = + Utils + .stringToUri(RegistryLookup.toPath(http.uri.toString, key)) + .traverse(uri => getFromUri(uri, http.apikey)) + .map { response => + val result = for { + body <- OptionT(response) + json = parse(body) + result <- OptionT.liftF[Either[RegistryError, *], Json]( + json.leftMap(e => RegistryError.RepoFailure(e.show)) + ) + } yield result + + result.getOrElseF[Json](RegistryError.NotFound.asLeft) + } + .recover { case uhe: UnknownHostException => + val error = s"Unknown host issue fetching: ${uhe.getMessage}" + RegistryError.RepoFailure(error).asLeft + } + + private def listImpl[F[_]: Sync]( + http: Registry.HttpConnection, + vendor: String, + name: String, + model: Int + ): F[Either[RegistryError, SchemaList]] = + Utils + .stringToUri(RegistryLookup.toSubpath(http.uri.toString, vendor, name, model)) + .traverse(uri => getFromUri(uri, http.apikey)) + .map { response => + for { + body <- response + text <- body.toRight(RegistryError.NotFound) + json <- parse(text).leftMap(e => RegistryError.RepoFailure(e.show)) + list <- json.as[SchemaList].leftMap(e => RegistryError.RepoFailure(e.show)) + } yield list + } + + /** + * Read a Json from an URI using optional apikey + * with added optional header, so it is unsafe as well and throws same exceptions + * + * @param uri the URL to fetch the JSON document from + * @param apikey optional apikey UUID to authenticate in Iglu Server + * @return The document at that URL if code is 2xx + */ + private def getFromUri[F[_]: Sync](uri: URI, apikey: Option[String]): F[Option[String]] = + Sync[F].blocking(executeCall(uri, apikey)) + + /** Non-RT analog of [[getFromUri]] */ + private def unsafeGetFromUri(uri: URI, apikey: Option[String]): Either[RegistryError, Json] = + try { + executeCall(uri, apikey) + .map(parse) + .map(_.leftMap(e => RegistryError.RepoFailure(e.show))) + .getOrElse(RegistryError.NotFound.asLeft) + } catch { + case NonFatal(e) => + Utils.repoFailure(e).asLeft + } + + /** Non-RT analog of [[JavaNetRegistryLookup.httpList]] */ + private def unsafeHttpList(uri: URI, apikey: Option[String]): Either[RegistryError, SchemaList] = + for { + json <- unsafeGetFromUri(uri, apikey) + list <- json.as[SchemaList].leftMap(e => RegistryError.RepoFailure(e.show)) + } yield list + + private def executeCall(uri: URI, apikey: Option[String]): Option[String] = { + val httpRequest = buildLookupRequest(uri, apikey) + val response = httpClient.send(httpRequest, BodyHandlers.ofString()) + if (is2xx(response)) response.body.some else None + } + + private def buildLookupRequest(uri: URI, apikey: Option[String]): HttpRequest = { + val baseRequest = HttpRequest + .newBuilder(uri) + .timeout(Duration.ofMillis(ReadTimeoutMs)) + + apikey + .fold(baseRequest)(key => baseRequest.header("apikey", key)) + .build() + } + + private def is2xx(response: HttpResponse[String]) = + response.statusCode() >= 200 && response.statusCode() <= 299 + +} diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Registry.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Registry.scala index 869de974..3bd1c1b2 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Registry.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Registry.scala @@ -15,8 +15,12 @@ package com.snowplowanalytics.iglu.client.resolver.registries // Java import java.net.URI +// Cats +import cats.syntax.either._ +import cats.syntax.show._ + // circe -import io.circe.{Decoder, HCursor, Json} +import io.circe.{Decoder, DecodingFailure, HCursor, Json} // Iglu Core import com.snowplowanalytics.iglu.core.SelfDescribingSchema @@ -35,9 +39,6 @@ sealed trait Registry extends Product with Serializable { } object Registry { - import Utils._ - - type Get[F[_], A] = Registry => F[Either[RegistryError, A]] /** * An embedded repository is one which is embedded inside the calling code, @@ -90,6 +91,14 @@ object Registry { /** Helper class to extract HTTP URI and api key from config JSON */ case class HttpConnection(uri: URI, apikey: Option[String]) + private implicit val uriCirceJsonDecoder: Decoder[URI] = + Decoder.instance { cursor => + for { + string <- cursor.as[String] + uri <- Utils.stringToUri(string).leftMap(e => DecodingFailure(e.show, cursor.history)) + } yield uri + } + implicit val httpConnectionDecoder: Decoder[HttpConnection] = new Decoder[HttpConnection] { def apply(c: HCursor): Decoder.Result[HttpConnection] = diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala index 39222ab7..c34fde32 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala @@ -13,28 +13,20 @@ package com.snowplowanalytics.iglu.client.resolver package registries -// Java -import cats.effect.Sync - -import java.net.UnknownHostException - -// Scala -import scala.util.control.NonFatal - // cats -import cats.Id -import cats.data.{EitherT, OptionT} -import cats.effect.implicits._ +import cats.effect.Sync import cats.implicits._ +import cats.ApplicativeThrow // circe import io.circe.Json -import io.circe.parser.parse // Iglu Core import com.snowplowanalytics.iglu.core.circe.implicits._ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaList, SelfDescribingSchema} +import scala.util.control.NonFatal + /** * A capability of `F` to communicate with Iglu registries, using `RepositoryRef` ADT, * in order to lookup for schemas or get schema lists @@ -88,65 +80,61 @@ object RegistryLookup { RegistryLookup[F].list(repositoryRef, vendor: String, name: String, model: Int) } - implicit def ioLookupInstance[F[_]](implicit F: Sync[F]): RegistryLookup[F] = - new RegistryLookup[F] { - def lookup(repositoryRef: Registry, schemaKey: SchemaKey): F[Either[RegistryError, Json]] = - repositoryRef match { - case Registry.Http(_, connection) => httpLookup(connection, schemaKey) - case Registry.Embedded(_, path) => embeddedLookup[F](path, schemaKey) - case Registry.InMemory(_, schemas) => F.pure(inMemoryLookup(schemas, schemaKey)) - } - - def list( - registry: Registry, - vendor: String, - name: String, - model: Int - ): F[Either[RegistryError, SchemaList]] = - registry match { - case Registry.Http(_, connection) => httpList(connection, vendor, name, model) - case Registry.Embedded(_, base) => - val path = toSubpath(base, vendor, name) - Sync[F].delay(Utils.unsafeEmbeddedList(path, model)) - case _ => F.pure(RegistryError.NotFound.asLeft) - } - } + /** + * An implementation of [[RegistryLookup]] with standard implementations of embedded/in-memory + * lookups and pluggable implementation of http lookups + */ + private[registries] abstract class StdRegistryLookup[F[_]: Sync] extends RegistryLookup[F] { - // Id instance also swallows all exceptions into `RegistryError` - implicit def idLookupInstance: RegistryLookup[Id] = - new RegistryLookup[Id] { - def lookup(repositoryRef: Registry, schemaKey: SchemaKey): Id[Either[RegistryError, Json]] = - repositoryRef match { - case Registry.Http(_, connection) => - Utils - .stringToUri(toPath(connection.uri.toString, schemaKey)) - .flatMap(uri => Utils.unsafeGetFromUri(uri, connection.apikey)) - case Registry.Embedded(_, base) => - val path = toPath(base, schemaKey) - Utils.unsafeEmbeddedLookup(path) - case Registry.InMemory(_, schemas) => - inMemoryLookup(schemas, schemaKey) - } - - def list( - registry: Registry, - vendor: String, - name: String, - model: Int - ): Id[Either[RegistryError, SchemaList]] = - registry match { - case Registry.Http(_, connection) => - val subpath = toSubpath(connection.uri.toString, vendor, name, model) - Utils.stringToUri(subpath).flatMap(Utils.unsafeHttpList(_, connection.apikey)) - case Registry.Embedded(_, base) => - val path = toSubpath(base, vendor, name) - Utils.unsafeEmbeddedList(path, model) - case _ => - RegistryError.NotFound.asLeft - } + /** Abstract methods to be provided by the implementation */ + def httpLookup(registry: Registry.Http, schemaKey: SchemaKey): F[Either[RegistryError, Json]] + def httpList( + registry: Registry.Http, + vendor: String, + name: String, + model: Int + ): F[Either[RegistryError, SchemaList]] + + /** Common functionality across all implementations */ + override def lookup(registry: Registry, schemaKey: SchemaKey): F[Either[RegistryError, Json]] = + registry match { + case http: Registry.Http => + withErrorHandling { + httpLookup(http, schemaKey) + } + case Registry.Embedded(_, path) => Embedded.lookup[F](path, schemaKey) + case Registry.InMemory(_, schemas) => + Sync[F].delay(RegistryLookup.inMemoryLookup(schemas, schemaKey)) + } + + override def list( + registry: Registry, + vendor: String, + name: String, + model: Int + ): F[Either[RegistryError, SchemaList]] = + registry match { + case http: Registry.Http => + withErrorHandling { + httpList(http, vendor, name, model) + } + case Registry.Embedded(_, base) => + val path = toSubpath(base, vendor, name) + Sync[F].delay(Embedded.unsafeList(path, model)) + case _ => Sync[F].pure(RegistryError.NotFound.asLeft) + } + + } + + private def withErrorHandling[F[_]: ApplicativeThrow, A]( + f: F[Either[RegistryError, A]] + ): F[Either[RegistryError, A]] = + f.recover { case NonFatal(nfe) => + val error = s"Unexpected exception fetching: $nfe" + RegistryError.RepoFailure(error).asLeft } - def inMemoryLookup( + private[registries] def inMemoryLookup( schemas: List[SelfDescribingSchema[Json]], key: SchemaKey ): Either[RegistryError, Json] = @@ -156,7 +144,7 @@ object RegistryLookup { private[registries] def toPath(prefix: String, key: SchemaKey): String = s"${prefix.stripSuffix("/")}/schemas/${key.toPath}" - private def toSubpath( + private[registries] def toSubpath( prefix: String, vendor: String, name: String, @@ -164,87 +152,11 @@ object RegistryLookup { ): String = s"${prefix.stripSuffix("/")}/schemas/$vendor/$name/jsonschema/$model" - private def toSubpath( + private[registries] def toSubpath( prefix: String, vendor: String, name: String ): String = s"${prefix.stripSuffix("/")}/schemas/$vendor/$name/jsonschema" - /** - * Retrieves an Iglu Schema from the Embedded Iglu Repo as a JSON - * - * @param base path on the local filesystem system - * @param key The SchemaKey uniquely identifying the schema in Iglu - * @return either a `Json` on success, or `RegistryError` in case of any failure - * (i.e. all exceptions should be swallowed by `RegistryError`) - */ - private[registries] def embeddedLookup[F[_]: Sync]( - base: String, - key: SchemaKey - ): F[Either[RegistryError, Json]] = { - val path = toPath(base, key) - val is = Utils.readResource[F](path) - val schema = is.bracket(_.traverse(Utils.fromStream[F]))(_.traverse_(Utils.closeStream[F])) - val result = for { - stringOption <- schema.attemptT.leftMap(Utils.repoFailure) - string <- EitherT.fromOption[F](stringOption, RegistryError.NotFound: RegistryError) - json <- EitherT.fromEither[F](parse(string)).leftMap(Utils.invalidSchema) - } yield json - - result.value - } - - /** - * Retrieves an Iglu Schema from the HTTP Iglu Repo as a JSON - * - * @param http endpoint and optional apikey - * @param key The SchemaKey uniquely identifying the schema in Iglu - * @return either a `Json` on success, or `RegistryError` in case of any failure - * (i.e. all exceptions should be swallowed by `RegistryError`) - */ - private[registries] def httpLookup[F[_]: Sync]( - http: Registry.HttpConnection, - key: SchemaKey - ): F[Either[RegistryError, Json]] = - Utils - .stringToUri(toPath(http.uri.toString, key)) - .traverse(uri => Utils.getFromUri(uri, http.apikey)) - .map { response => - val result = for { - body <- OptionT(response) - json = parse(body) - result <- OptionT.liftF[Either[RegistryError, *], Json]( - json.leftMap(e => RegistryError.RepoFailure(e.show)) - ) - } yield result - - result.getOrElseF[Json](RegistryError.NotFound.asLeft) - } - .recover { - case uhe: UnknownHostException => - val error = s"Unknown host issue fetching: ${uhe.getMessage}" - RegistryError.RepoFailure(error).asLeft - case NonFatal(nfe) => - val error = s"Unexpected exception fetching: $nfe" - RegistryError.RepoFailure(error).asLeft - } - - private[registries] def httpList[F[_]: Sync]( - http: Registry.HttpConnection, - vendor: String, - name: String, - model: Int - ): F[Either[RegistryError, SchemaList]] = - Utils - .stringToUri(toSubpath(http.uri.toString, vendor, name, model)) - .traverse(uri => Utils.getFromUri(uri, http.apikey)) - .map { response => - for { - body <- response - text <- body.toRight(RegistryError.NotFound) - json <- parse(text).leftMap(e => RegistryError.RepoFailure(e.show)) - list <- json.as[SchemaList].leftMap(e => RegistryError.RepoFailure(e.show)) - } yield list - } } diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala index 88368087..d125e5e6 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala @@ -14,147 +14,19 @@ package com.snowplowanalytics.iglu.client package resolver.registries // Java -import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} - -import java.io.{File, InputStream} import java.net.URI -import java.net.http.HttpResponse.BodyHandlers -import java.net.http.{HttpClient, HttpRequest, HttpResponse} -import java.time.Duration -import scala.util.matching.Regex - -// Scala -import scala.io.Source -import scala.util.control.NonFatal // Cats -import cats.effect.Sync import cats.syntax.either._ -import cats.syntax.option._ import cats.syntax.show._ -import cats.syntax.traverse._ // circe -import io.circe.parser.parse -import io.circe.{Decoder, DecodingFailure, Json, ParsingFailure} +import io.circe.ParsingFailure // Apache Commons import org.apache.commons.lang3.exception.ExceptionUtils -// scalaj -import com.snowplowanalytics.iglu.core.SchemaList -import com.snowplowanalytics.iglu.core.circe.CirceIgluCodecs._ - private[registries] object Utils { - private val ReadTimeoutMs = 4000L - - private lazy val httpClient = HttpClient - .newBuilder() - .connectTimeout(Duration.ofMillis(1000)) - .build() - - /** - * Read a Json from an URI using optional apikey - * with added optional header, so it is unsafe as well and throws same exceptions - * - * @param uri the URL to fetch the JSON document from - * @param apikey optional apikey UUID to authenticate in Iglu Server - * @return The document at that URL if code is 2xx - */ - def getFromUri[F[_]: Sync](uri: URI, apikey: Option[String]): F[Option[String]] = - Sync[F].blocking(executeCall(uri, apikey)) - - /** Non-RT analog of [[getFromUri]] */ - def unsafeGetFromUri(uri: URI, apikey: Option[String]): Either[RegistryError, Json] = - try { - executeCall(uri, apikey) - .map(parse) - .map(_.leftMap(e => RegistryError.RepoFailure(e.show))) - .getOrElse(RegistryError.NotFound.asLeft) - } catch { - case NonFatal(e) => - repoFailure(e).asLeft - } - - def unsafeEmbeddedList(path: String, modelMatch: Int): Either[RegistryError, SchemaList] = - try { - val d = - new File( - getClass.getResource(path).getPath - ) // this will throw NPE for missing entry in embedded repos - val schemaFileRegex: Regex = (".*/schemas/?" + // path to file - "([a-zA-Z0-9-_.]+)/" + // Vendor - "([a-zA-Z0-9-_]+)/" + // Name - "([a-zA-Z0-9-_]+)/" + // Format - "([1-9][0-9]*)-(\\d+)-(\\d+)$").r // MODEL, REVISION and ADDITION - - def getFolderContent(d: File): List[String] = { - d.listFiles - .filter(_.isFile) - .toList - .filter(_.getName.startsWith(s"${modelMatch.toString}-")) - .map(_.getAbsolutePath) - } - - val content = - if (d.exists & d.isDirectory) - getFolderContent(d) - else - List.empty[String] - - content - .traverse { - case schemaFileRegex(vendor, name, format, model, revision, addition) - if model == modelMatch.toString => - SchemaKey( - vendor = vendor, - name = name, - format = format, - version = SchemaVer - .Full(model = model.toInt, revision = revision.toInt, addition = addition.toInt) - ).asRight - case f => RegistryError.RepoFailure(s"Corrupted schema file name at $f").asLeft - } - .map(_.sortBy(_.version)) - .flatMap(s => - if (s.isEmpty) - RegistryError.NotFound.asLeft - else - s.asRight - ) - .map(SchemaList.parseUnsafe) - } catch { - case NonFatal(e) => - e match { - case _: NullPointerException => RegistryError.NotFound.asLeft - case _ => repoFailure(e).asLeft - } - } - - /** Not-RT analog of [[RegistryLookup.embeddedLookup]] */ - def unsafeEmbeddedLookup(path: String): Either[RegistryError, Json] = - try { - val is = Utils.unsafeReadResource(path) - val schema = is.map(unsafeFromStream) - val result = schema - .toRight(RegistryError.NotFound: RegistryError) - .flatMap(x => parse(x).leftMap(invalidSchema)) - is.fold(())(unsafeCloseStream) - result - } catch { - case NonFatal(e) => - e match { - case _: NullPointerException => RegistryError.NotFound.asLeft - case _ => repoFailure(e).asLeft - } - } - - /** Non-RT analog of [[RegistryLookup.httpList]] */ - def unsafeHttpList(uri: URI, apikey: Option[String]): Either[RegistryError, SchemaList] = - for { - json <- unsafeGetFromUri(uri, apikey) - list <- json.as[SchemaList].leftMap(e => RegistryError.RepoFailure(e.show)) - } yield list /** * A wrapper around Java's URI. @@ -172,53 +44,12 @@ private[registries] object Utils { RegistryError.ClientFailure(s"Provided URI string violates RFC 2396: [$error]").asLeft } - implicit val uriCirceJsonDecoder: Decoder[URI] = - Decoder.instance { cursor => - for { - string <- cursor.as[String] - uri <- stringToUri(string).leftMap(e => DecodingFailure(e.show, cursor.history)) - } yield uri - } - - private def executeCall(uri: URI, apikey: Option[String]): Option[String] = { - val httpRequest = buildLookupRequest(uri, apikey) - val response = httpClient.send(httpRequest, BodyHandlers.ofString()) - if (is2xx(response)) response.body.some else None - } - - private def buildLookupRequest(uri: URI, apikey: Option[String]): HttpRequest = { - val baseRequest = HttpRequest - .newBuilder(uri) - .timeout(Duration.ofMillis(ReadTimeoutMs)) - - apikey - .fold(baseRequest)(key => baseRequest.header("apikey", key)) - .build() - } - - private def is2xx(response: HttpResponse[String]) = - response.statusCode() >= 200 && response.statusCode() <= 299 - - private[resolver] def readResource[F[_]: Sync](path: String): F[Option[InputStream]] = - Sync[F].delay(unsafeReadResource(path)) - private[resolver] def unsafeReadResource(path: String): Option[InputStream] = - Option(getClass.getResource(path)).map(_.openStream()) - - private[resolver] def fromStream[F[_]: Sync](is: InputStream): F[String] = - Sync[F].delay(unsafeFromStream(is)) - private[resolver] def unsafeFromStream(is: InputStream): String = - Source.fromInputStream(is).mkString - - private[resolver] def closeStream[F[_]: Sync](is: InputStream): F[Unit] = - Sync[F].delay(unsafeCloseStream(is)) - private[resolver] def unsafeCloseStream(is: InputStream): Unit = - is.close() - - private[resolver] def invalidSchema(failure: ParsingFailure): RegistryError = + def invalidSchema(failure: ParsingFailure): RegistryError = RegistryError.RepoFailure(failure.show) - private[resolver] def repoFailure(failure: Throwable): RegistryError = + def repoFailure(failure: Throwable): RegistryError = RegistryError.RepoFailure( if (failure.getMessage != null) failure.getMessage else "Unhandled error" ) + } diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/package.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/package.scala deleted file mode 100644 index b3c13084..00000000 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/package.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2014-2023 Snowplow Analytics Ltd. All rights reserved. - * - * This program is licensed to you under the Apache License Version 2.0, - * and you may not use this file except in compliance with the Apache License Version 2.0. - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the Apache License Version 2.0 is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - */ -package com.snowplowanalytics.iglu.client.resolver - -import scala.util.{Either, Right} - -package object registries { - - /** Workaround for 2.12/2.13 compatibility */ - implicit class EitherOps[A, B](either: Either[A, B]) { - def orElse[A1 >: A, B1 >: B](or: => Either[A1, B1]): Either[A1, B1] = - either match { - case Right(_) => either - case _ => or - } - } -} diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/validator/CirceValidator.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/validator/CirceValidator.scala index 12e03cdd..be19e390 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/validator/CirceValidator.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/validator/CirceValidator.scala @@ -19,7 +19,11 @@ import com.snowplowanalytics.iglu.client.resolver.StorageTime import com.snowplowanalytics.iglu.core.circe.MetaSchemas // Scala import com.fasterxml.jackson.databind.JsonNode -import com.snowplowanalytics.iglu.client.resolver.Resolver.SchemaLookupResult +import com.networknt.schema.uri.URIFetcher +import com.snowplowanalytics.iglu.client.resolver.Resolver.{SchemaItem, SchemaLookupResult} +import java.io.{ByteArrayInputStream, InputStream} +import java.net.URI +import java.nio.charset.StandardCharsets import scala.jdk.CollectionConverters._ // Cats @@ -55,12 +59,21 @@ object CirceValidator extends Validator[Json] { private val V4SchemaInstance = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V4) + private val fakeUrlFetcher = new URIFetcher { + override def fetch(uri: URI): InputStream = { + // No effect on validation, because we return empty JSON Schema which matches any data. + val emptyJsonObject = Json.obj() + new ByteArrayInputStream(emptyJsonObject.toString().getBytes(StandardCharsets.UTF_8)) + } + } + private val IgluMetaschemaFactory = JsonSchemaFactory .builder(V4SchemaInstance) .addMetaSchema(IgluMetaschema) .forceHttps(false) .removeEmptyFragmentSuffix(false) + .uriFetcher(fakeUrlFetcher, "http", "https") .build() private val SchemaValidatorsConfig: SchemaValidatorsConfig = { @@ -155,7 +168,7 @@ object CirceValidator extends Validator[Json] { evaluationCache: SchemaEvaluationCache[F] )(result: SchemaLookupResult): F[Either[ValidatorError.InvalidSchema, JsonSchema]] = { result match { - case ResolverResult.Cached(key, schema, timestamp) => + case ResolverResult.Cached(key, SchemaItem(schema, _), timestamp) => evaluationCache.get((key, timestamp)).flatMap { case Some(alreadyEvaluatedSchema) => alreadyEvaluatedSchema.pure[F] @@ -164,7 +177,7 @@ object CirceValidator extends Validator[Json] { .pure[F] .flatTap(result => evaluationCache.put((key, timestamp), result)) } - case ResolverResult.NotCached(schema) => + case ResolverResult.NotCached(SchemaItem(schema, _)) => provideNewJsonSchema(schema).pure[F] } } diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-0 new file mode 100644 index 00000000..de9fcdc0 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-0 @@ -0,0 +1,20 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "$supersededBy": "1-0", + "description": "Test schema", + "self": { + "vendor": "com.snowplowanalytics.iglu-test", + "name": "invalid-superseded-schema", + "format": "jsonschema", + "version": "1-0-0" + }, + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + + "required": ["id"], + "additionalProperties": false +} diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-1 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-1 new file mode 100644 index 00000000..e9a74233 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-1 @@ -0,0 +1,24 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "$supersededBy": "1-0-3", + "description": "Test schema", + "self": { + "vendor": "com.snowplowanalytics.iglu-test", + "name": "invalid-superseded-schema", + "format": "jsonschema", + "version": "1-0-1" + }, + + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + + "required": ["id"], + "additionalProperties": false +} diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-2 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-2 new file mode 100644 index 00000000..c9349afc --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-2 @@ -0,0 +1,27 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "$supersededBy": "1-0-1", + "description": "Test schema", + "self": { + "vendor": "com.snowplowanalytics.iglu-test", + "name": "invalid-superseded-schema", + "format": "jsonschema", + "version": "1-0-2" + }, + + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "price": { + "type": "number" + } + }, + + "required": ["id"], + "additionalProperties": false +} diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-0 new file mode 100644 index 00000000..07acb641 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-0 @@ -0,0 +1,21 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "$supersededBy": "1-0-2", + "description": "Test schema", + "self": { + "vendor": "com.snowplowanalytics.iglu-test", + "name": "superseded-schema", + "format": "jsonschema", + "version": "1-0-0" + }, + + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + + "required": ["id"], + "additionalProperties": false +} diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-1 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-1 new file mode 100644 index 00000000..aed7db42 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-1 @@ -0,0 +1,23 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Test schema", + "self": { + "vendor": "com.snowplowanalytics.iglu-test", + "name": "superseded-schema", + "format": "jsonschema", + "version": "1-0-1" + }, + + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + + "required": ["id"], + "additionalProperties": false +} diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-2 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-2 new file mode 100644 index 00000000..1cf7cb35 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-2 @@ -0,0 +1,26 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Test schema", + "self": { + "vendor": "com.snowplowanalytics.iglu-test", + "name": "superseded-schema", + "format": "jsonschema", + "version": "1-0-2" + }, + + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "price": { + "type": "number" + } + }, + + "required": ["id"], + "additionalProperties": false +} diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/CachingClientSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/CachingClientSpec.scala index 43a3ee0c..6b06694b 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/CachingClientSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/CachingClientSpec.scala @@ -17,6 +17,7 @@ import cats.effect.testing.specs2.CatsEffect import io.circe.literal._ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} +import com.snowplowanalytics.iglu.client.resolver.registries.JavaNetRegistryLookup._ // Specs2 import org.specs2.mutable.Specification @@ -30,6 +31,9 @@ class CachingClientSpec extends Specification with CatsEffect { validating a correct self-desc JSON should return the JSON in a Success $e1 validating a correct self-desc JSON with JSON Schema with incorrect $$schema property should return Failure $e2 validating an incorrect self-desc JSON should return the validation errors in a Failure $e3 + validating a correct self-desc JSON with superseded schema should return the JSON in a Success $e4 + validating an incorrect self-desc JSON with superseded schema should return validation errors in a Failure $e5 + validating self-desc JSONs with invalid superseded schemas should return resolution errors $e6 """ @@ -66,6 +70,72 @@ class CachingClientSpec extends Specification with CatsEffect { json"""{"schema": "iglu://jsonschema/1-0-0", "data": { "id": 0 } }""" ) + val validJsonWithSupersededSchema1 = + SelfDescribingData( + SchemaKey( + "com.snowplowanalytics.iglu-test", + "superseded-schema", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ), + json"""{ "id": "test-id" }""" + ) + + val validJsonWithSupersededSchema2 = + SelfDescribingData( + SchemaKey( + "com.snowplowanalytics.iglu-test", + "superseded-schema", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ), + json"""{ "id": "test-id", "name": "test-name" }""" + ) + + val invalidJsonWithSupersededSchema = + SelfDescribingData( + SchemaKey( + "com.snowplowanalytics.iglu-test", + "superseded-schema", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ), + json"""{ "id": "test-id", "name": "test-name", "field_a": "value_a" }""" + ) + + val jsonWithInvalidSupersededSchema100 = + SelfDescribingData( + SchemaKey( + "com.snowplowanalytics.iglu-test", + "invalid-superseded-schema", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ), + json"""{ "id": "test-id" }""" + ) + + val jsonWithInvalidSupersededSchema101 = + SelfDescribingData( + SchemaKey( + "com.snowplowanalytics.iglu-test", + "invalid-superseded-schema", + "jsonschema", + SchemaVer.Full(1, 0, 1) + ), + json"""{ "id": "test-id" }""" + ) + + val jsonWithInvalidSupersededSchema102 = + SelfDescribingData( + SchemaKey( + "com.snowplowanalytics.iglu-test", + "invalid-superseded-schema", + "jsonschema", + SchemaVer.Full(1, 0, 2) + ), + json"""{ "id": "test-id" }""" + ) + def e1 = { for { client <- SpecHelpers.CachingTestClient @@ -87,4 +157,55 @@ class CachingClientSpec extends Specification with CatsEffect { } yield result must beLeft } + def e4 = { + val supersedingSchema = SchemaVer.Full(1, 0, 2) + for { + res1 <- for { + client <- SpecHelpers.CachingTestClient + result <- client.check(validJsonWithSupersededSchema1).value + } yield result + res2 <- for { + client <- SpecHelpers.CachingTestClient + result <- client.check(validJsonWithSupersededSchema2).value + } yield result + } yield (res1 must beRight(Some(supersedingSchema))) and + (res2 must beRight(Some(supersedingSchema))) + } + + def e5 = { + for { + client <- SpecHelpers.CachingTestClient + result <- client.check(invalidJsonWithSupersededSchema).value + } yield { + result must beLeft.like { + case ClientError.ValidationError(_, Some(supersededBy)) if supersededBy == "1-0-2" => ok + case _ => ko + } + } + } + + def e6 = { + for { + res1 <- for { + client <- SpecHelpers.CachingTestClient + result <- client.check(jsonWithInvalidSupersededSchema100).value + } yield result + res2 <- for { + client <- SpecHelpers.CachingTestClient + result <- client.check(jsonWithInvalidSupersededSchema101).value + } yield result + res3 <- for { + client <- SpecHelpers.CachingTestClient + result <- client.check(jsonWithInvalidSupersededSchema102).value + } yield result + } yield { + val match1 = res1.toString must contain("Invalid schema version: 1-0") + val match2 = res2.toString must contain("Iglu Test Embedded -> LookupHistory(Set(NotFound)") + val match3 = res3.toString must contain( + "ClientFailure(Superseding version 1-0-1 isn't greater than the version of schema com.snowplowanalytics.iglu-test/invalid-superseded-schema/jsonschema/1-0-2)" + ) + match1.and(match2).and(match3) + } + } + } diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/SpecHelpers.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/SpecHelpers.scala index 2320716a..a5d906d6 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/SpecHelpers.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/SpecHelpers.scala @@ -13,11 +13,19 @@ package com.snowplowanalytics.iglu.client import cats.Applicative +import com.snowplowanalytics.iglu.client.resolver.registries.{ + JavaNetRegistryLookup, + RegistryError, + RegistryLookup +} +import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaList} +import io.circe.Json import java.net.URI import java.time.Instant import java.util.concurrent.TimeUnit import scala.concurrent.duration.FiniteDuration +import java.util.concurrent.atomic.AtomicReference // Cats import cats.Id @@ -44,6 +52,41 @@ object SpecHelpers { Registry.HttpConnection(URI.create("http://iglucentral.com"), None) ) + case class TrackingRegistry( + lookupState: AtomicReference[List[String]], + listState: AtomicReference[List[String]] + ) extends RegistryLookup[IO] { + override def lookup( + registry: Registry, + schemaKey: SchemaKey + ): IO[Either[RegistryError, Json]] = { + IO( + lookupState.updateAndGet((l: List[String]) => + Seq(registry.config.name, schemaKey.toSchemaUri).mkString("-") :: l + ) + ) >> + JavaNetRegistryLookup.ioLookupInstance[IO].lookup(registry, schemaKey) + } + + override def list( + registry: Registry, + vendor: String, + name: String, + model: Int + ): IO[Either[RegistryError, SchemaList]] = { + IO( + listState.updateAndGet((l: List[String]) => + Seq(registry.config.name, vendor, name, model.toString).mkString("-") :: l + ) + ) >> + JavaNetRegistryLookup.ioLookupInstance[IO].list(registry, vendor, name, model) + } + } + def mkTrackingRegistry: TrackingRegistry = TrackingRegistry( + new AtomicReference[List[String]](List.empty[String]), + new AtomicReference[List[String]](List.empty[String]) + ) + val EmbeddedTest: Registry = Registry.Embedded( Registry.Config("Iglu Test Embedded", 0, List("com.snowplowanalytics")), diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverCacheSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverCacheSpec.scala index da988503..c7d062fe 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverCacheSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverCacheSpec.scala @@ -24,6 +24,8 @@ import org.specs2.matcher.MatchResult import java.time.Instant import scala.concurrent.duration.DurationInt +import Resolver.SchemaItem + class ResolverCacheSpec extends Specification { def is = s2""" Disallow overwriting successful request with failed one $e1 @@ -36,20 +38,26 @@ class ResolverCacheSpec extends Specification { val key = SchemaKey("com.acme", "schema", "jsonschema", SchemaVer.Full(1, 0, 0)) val schema = Json.Null - val lookupResult = schema.asRight[LookupFailureMap] + val lookupResult = SchemaItem(schema, None).asRight[LookupFailureMap] val expectedState = - RegistryState(Map.empty, 4.millis, List((key, (2.millis, Right(Json.Null)))), 5, List()) + RegistryState( + Map.empty, + 4.millis, + List((key, (2.millis, Right(SchemaItem(Json.Null, None))))), + 5, + List() + ) val test = for { cache <- ResolverCache.init[StaticLookup](5, Some(10.seconds)) cacheUnsafe = cache.getOrElse(throw new RuntimeException("Cache cannot be created")) _ <- cacheUnsafe.putSchema(key, lookupResult) - result <- cacheUnsafe.putSchema(key, Map.empty[Registry, LookupHistory].asLeft[Json]) + result <- cacheUnsafe.putSchema(key, Map.empty[Registry, LookupHistory].asLeft) } yield result val (state, result) = test.run(RegistryState.init).value - val schemaResult = result must beRight(schema) + val schemaResult = result must beRight(SchemaItem(schema, None)) val stateResult = state must beEqualTo(expectedState) schemaResult and stateResult @@ -95,8 +103,8 @@ class ResolverCacheSpec extends Specification { val test = for { cache <- ResolverCache.init[StaticLookup](5, Some(10.seconds)) cacheUnsafe = cache.getOrElse(throw new RuntimeException("Cache cannot be created")) - _ <- cacheUnsafe.putSchema(key, failure1.asLeft[Json]) - result <- cacheUnsafe.putSchema(key, failure2.asLeft[Json]) + _ <- cacheUnsafe.putSchema(key, failure1.asLeft) + result <- cacheUnsafe.putSchema(key, failure2.asLeft) } yield result val (_, result: SchemaLookup) = test.run(RegistryState.init).value @@ -116,7 +124,6 @@ class ResolverCacheSpec extends Specification { object ResolverCacheSpec { // No need to overwrite anything - implicit val a: InitSchemaCache[StaticLookup] = staticCache - implicit val c: InitListCache[StaticLookup] = staticCacheForList - implicit val b: Clock[StaticLookup] = staticClock + implicit val a: CreateResolverCache[StaticLookup] = staticResolverCache + implicit val b: Clock[StaticLookup] = staticClock } diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverResultSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverResultSpec.scala index 82338fd8..bc69fd1c 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverResultSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverResultSpec.scala @@ -24,6 +24,7 @@ import scala.concurrent.duration._ // Cats import cats.Id import cats.effect.IO +import cats.effect.implicits._ import cats.implicits._ // circe @@ -37,9 +38,13 @@ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} import com.snowplowanalytics.iglu.client.ClientError._ import com.snowplowanalytics.iglu.client.SpecHelpers import com.snowplowanalytics.iglu.client.resolver.ResolverSpecHelpers.StaticLookup -import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup._ -import com.snowplowanalytics.iglu.client.resolver.registries.{Registry, RegistryError} -import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup +import com.snowplowanalytics.iglu.client.resolver.registries.{ + JavaNetRegistryLookup, + Registry, + RegistryError, + RegistryLookup +} +import com.snowplowanalytics.iglu.client.resolver.Resolver.SchemaItem // Specs2 import com.snowplowanalytics.iglu.client.SpecHelpers._ @@ -66,6 +71,9 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE a Resolver should not cache schema if cache is disabled $e12 a Resolver should return cached schema when ttl not exceeded $e13 a Resolver should return cached schema when ttl exceeded $e14 + a Resolver should not spam the registry with similar requests $e15 + a Resolver should return superseding schema if resolveSupersedingSchema is true $e16 + a Resolver shouldn't return superseding schema if resolveSupersedingSchema is false $e17 """ import ResolverSpec._ @@ -132,6 +140,8 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE ) ) + implicit val lookup: RegistryLookup[IO] = JavaNetRegistryLookup.ioLookupInstance[IO] + SpecHelpers.TestResolver .flatMap(resolver => resolver.lookupSchema(schemaKey)) .map { result => @@ -158,6 +168,8 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE ) ) + implicit val lookup: RegistryLookup[IO] = JavaNetRegistryLookup.ioLookupInstance[IO] + SpecHelpers.TestResolver .flatMap(resolver => resolver.lookupSchema(schemaKey)) .map { result => @@ -174,6 +186,8 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE SchemaVer.Full(1, 0, 0) ) + implicit val lookup: RegistryLookup[IO] = JavaNetRegistryLookup.ioLookupInstance[IO] + SpecHelpers.TestResolver .flatMap(resolver => resolver.lookupSchema(schemaKey)) .map { result => @@ -195,15 +209,14 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE RegistryError.RepoFailure("shouldn't matter").asLeft[Json] val correctResult = Json.Null.asRight[RegistryError] - val time = Instant.ofEpochMilli(2L) + val time = Instant.ofEpochMilli(3L) val responses = List(timeoutError, correctResult) val httpRep = Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache = ResolverSpecHelpers.staticCache - implicit val cacheList = ResolverSpecHelpers.staticCacheForList - implicit val clock = ResolverSpecHelpers.staticClock + implicit val cache = ResolverSpecHelpers.staticResolverCache + implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) @@ -231,7 +244,7 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE } val thirdSucceeded = response3 must beRight[SchemaLookupResult].like { - case ResolverResult.Cached(key, value, _) => + case ResolverResult.Cached(key, SchemaItem(value, _), _) => key must beEqualTo(schemaKey) and (value must beEqualTo(Json.Null)) } val requestsNumber = state.req must beEqualTo(2) @@ -258,9 +271,8 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE val httpRep = Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache = ResolverSpecHelpers.staticCache - implicit val cacheList = ResolverSpecHelpers.staticCacheForList - implicit val clock = ResolverSpecHelpers.staticClock + implicit val cache = ResolverSpecHelpers.staticResolverCache + implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) @@ -285,7 +297,7 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE // Final response must not overwrite a successful one val finalResult = response must beRight[SchemaLookupResult].like { - case ResolverResult.Cached(key, value, _) => + case ResolverResult.Cached(key, SchemaItem(value, _), _) => key must beEqualTo(schemaKey) and (value must beEqualTo(Json.Null)) } @@ -318,9 +330,8 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE null ) - implicit val cache = ResolverSpecHelpers.staticCache - implicit val cacheList = ResolverSpecHelpers.staticCacheForList - implicit val clock = ResolverSpecHelpers.staticClock + implicit val cache = ResolverSpecHelpers.staticResolverCache + implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookupByRepo( Map( "Mock Repo 1" -> List(error1.asLeft, error2.asLeft), @@ -331,12 +342,12 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE val expected = ResolutionError( SortedMap( - "Mock Repo 1" -> LookupHistory(Set(error1, error2), 2, Instant.ofEpochMilli(2008L)), - "Mock Repo 2" -> LookupHistory(Set(error3, error4), 2, Instant.ofEpochMilli(2009L)), + "Mock Repo 1" -> LookupHistory(Set(error1, error2), 2, Instant.ofEpochMilli(2010L)), + "Mock Repo 2" -> LookupHistory(Set(error3, error4), 2, Instant.ofEpochMilli(2011L)), "Iglu Client Embedded" -> LookupHistory( Set(RegistryError.NotFound), 1, - Instant.ofEpochMilli(4L) + Instant.ofEpochMilli(5L) ) ) ) @@ -400,7 +411,8 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE .HttpConnection(URI.create("https://com-iglucentral-eu1-prod.iglu.snplow.net/api"), None) ) - val resolver = Resolver.init[Id](10, None, IgluCentralServer) + val resolver = Resolver.init[Id](10, None, IgluCentralServer) + implicit val lookup: RegistryLookup[Id] = JavaNetRegistryLookup.idLookupInstance val resultOne = resolver.listSchemasResult("com.sendgrid", "bounce", 2) val resultTwo = resolver.listSchemasResult("com.sendgrid", "bounce", 1) @@ -431,10 +443,12 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE SchemaVer.Full(1, 0, 0) ) + implicit val lookup: RegistryLookup[IO] = JavaNetRegistryLookup.ioLookupInstance[IO] + Resolver .init[IO](cacheSize = 0, cacheTtl = None, refs = EmbeddedTest) .flatMap(resolver => resolver.lookupSchemaResult(schemaKey)) - .map(_ must beRight(ResolverResult.NotCached(expectedSchema))) + .map(_ must beRight(ResolverResult.NotCached(SchemaItem(expectedSchema, None)))) } def e13 = { @@ -452,9 +466,8 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE val httpRep = Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache = ResolverSpecHelpers.staticCache - implicit val cacheList = ResolverSpecHelpers.staticCacheForList - implicit val clock = ResolverSpecHelpers.staticClock + implicit val cache = ResolverSpecHelpers.staticResolverCache + implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) @@ -491,9 +504,8 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE val httpRep = Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache = ResolverSpecHelpers.staticCache - implicit val cacheList = ResolverSpecHelpers.staticCacheForList - implicit val clock = ResolverSpecHelpers.staticClock + implicit val cache = ResolverSpecHelpers.staticResolverCache + implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) @@ -517,4 +529,113 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE } } } + + def e15 = { + + import cats.effect.unsafe.IORuntime.global + implicit val runtime = global + + val schemaKey = + SchemaKey( + "com.sendgrid", + "bounce", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + + val IgluCentralServer = Registry.Http( + Registry.Config("Iglu Central EU1", 0, List("com.snowplowanalytics")), + Registry + .HttpConnection(URI.create("https://com-iglucentral-eu1-prod.iglu.snplow.net/api"), None) + ) + + val trackingRegistry: TrackingRegistry = mkTrackingRegistry + implicit val reg: RegistryLookup[IO] = trackingRegistry.asInstanceOf[RegistryLookup[IO]] + val resolver = Resolver.init[IO](10, None, IgluCentralServer).unsafeRunSync() + + def listWorker = () => resolver.listSchemas("com.sendgrid", "bounce", 1) + def lookupWorker = () => resolver.lookupSchema(schemaKey) + (List.fill(200)(listWorker) zip List.fill(200)(lookupWorker)) + .flatMap(t => List(t._1, t._2)) + .parTraverseN(100)(f => f()) + .unsafeRunSync() + + ( + trackingRegistry.listState.get().mkString(", "), + trackingRegistry.lookupState.get().mkString(", ") + ) must equalTo( + ( + "Iglu Central EU1-com.sendgrid-bounce-1, Iglu Client Embedded-com.sendgrid-bounce-1", + "Iglu Central EU1-iglu:com.sendgrid/bounce/jsonschema/1-0-0, Iglu Client Embedded-iglu:com.sendgrid/bounce/jsonschema/1-0-0" + ) + ) + } + + def e16 = { + + val expectedSchema: Json = + parse( + scala.io.Source + .fromInputStream( + getClass.getResourceAsStream( + "/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-2" + ) + ) + .mkString + ) + .fold(e => throw new RuntimeException(s"Cannot parse superseded schema, $e"), identity) + + val schemaKey = SchemaKey( + "com.snowplowanalytics.iglu-test", + "superseded-schema", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + + val trackingRegistry: TrackingRegistry = mkTrackingRegistry + implicit val reg: RegistryLookup[IO] = trackingRegistry.asInstanceOf[RegistryLookup[IO]] + + Resolver + .init[IO](cacheSize = 0, cacheTtl = None, refs = EmbeddedTest) + .flatMap(resolver => resolver.lookupSchemaResult(schemaKey, resolveSupersedingSchema = true)) + .map { result => + result must beRight( + ResolverResult.NotCached(SchemaItem(expectedSchema, Some(SchemaVer.Full(1, 0, 2)))) + ) + } + } + + def e17 = { + + val expectedSchema: Json = + parse( + scala.io.Source + .fromInputStream( + getClass.getResourceAsStream( + "/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/superseded-schema/jsonschema/1-0-0" + ) + ) + .mkString + ) + .fold(e => throw new RuntimeException(s"Cannot parse superseded schema, $e"), identity) + + val schemaKey = SchemaKey( + "com.snowplowanalytics.iglu-test", + "superseded-schema", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + + val trackingRegistry: TrackingRegistry = mkTrackingRegistry + implicit val reg: RegistryLookup[IO] = trackingRegistry.asInstanceOf[RegistryLookup[IO]] + + Resolver + .init[IO](cacheSize = 0, cacheTtl = None, refs = EmbeddedTest) + .flatMap(resolver => resolver.lookupSchemaResult(schemaKey)) + .map { result => + result must beRight( + ResolverResult.NotCached(SchemaItem(expectedSchema, None)) + ) + } + } } diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpec.scala index 54ba8772..636ddd10 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpec.scala @@ -36,7 +36,7 @@ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} import com.snowplowanalytics.iglu.client.ClientError._ import com.snowplowanalytics.iglu.client.SpecHelpers import com.snowplowanalytics.iglu.client.resolver.ResolverSpecHelpers.StaticLookup -import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup._ +import com.snowplowanalytics.iglu.client.resolver.registries.JavaNetRegistryLookup._ import com.snowplowanalytics.iglu.client.resolver.registries.{Registry, RegistryError} import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup @@ -204,15 +204,14 @@ class ResolverSpec extends Specification with CatsEffect { RegistryError.RepoFailure("shouldn't matter").asLeft[Json] val correctSchema = Json.Null.asRight[RegistryError] - val time = Instant.ofEpochMilli(2L) + val time = Instant.ofEpochMilli(3L) val responses = List(timeoutError, correctSchema) val httpRep = Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache: InitSchemaCache[StaticLookup] = ResolverSpecHelpers.staticCache - implicit val cacheList: InitListCache[StaticLookup] = ResolverSpecHelpers.staticCacheForList - implicit val clock: Clock[StaticLookup] = ResolverSpecHelpers.staticClock + implicit val cache: CreateResolverCache[StaticLookup] = ResolverSpecHelpers.staticResolverCache + implicit val clock: Clock[StaticLookup] = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) @@ -264,9 +263,8 @@ class ResolverSpec extends Specification with CatsEffect { val httpRep = Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache: InitSchemaCache[StaticLookup] = ResolverSpecHelpers.staticCache - implicit val cacheList: InitListCache[StaticLookup] = ResolverSpecHelpers.staticCacheForList - implicit val clock: Clock[StaticLookup] = ResolverSpecHelpers.staticClock + implicit val cache: CreateResolverCache[StaticLookup] = ResolverSpecHelpers.staticResolverCache + implicit val clock: Clock[StaticLookup] = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) @@ -317,9 +315,8 @@ class ResolverSpec extends Specification with CatsEffect { null ) - implicit val cache: InitSchemaCache[StaticLookup] = ResolverSpecHelpers.staticCache - implicit val cacheList: InitListCache[StaticLookup] = ResolverSpecHelpers.staticCacheForList - implicit val clock: Clock[StaticLookup] = ResolverSpecHelpers.staticClock + implicit val cache: CreateResolverCache[StaticLookup] = ResolverSpecHelpers.staticResolverCache + implicit val clock: Clock[StaticLookup] = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookupByRepo( Map( "Mock Repo 1" -> List(error1.asLeft, error2.asLeft), @@ -330,12 +327,12 @@ class ResolverSpec extends Specification with CatsEffect { val expected = ResolutionError( SortedMap( - "Mock Repo 1" -> LookupHistory(Set(error1, error2), 2, Instant.ofEpochMilli(2008L)), - "Mock Repo 2" -> LookupHistory(Set(error3, error4), 2, Instant.ofEpochMilli(2009L)), + "Mock Repo 1" -> LookupHistory(Set(error1, error2), 2, Instant.ofEpochMilli(2010L)), + "Mock Repo 2" -> LookupHistory(Set(error3, error4), 2, Instant.ofEpochMilli(2011L)), "Iglu Client Embedded" -> LookupHistory( Set(RegistryError.NotFound), 1, - Instant.ofEpochMilli(4L) + Instant.ofEpochMilli(5L) ) ) ) diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpecHelpers.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpecHelpers.scala index 4cbfb9ad..202feac7 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpecHelpers.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpecHelpers.scala @@ -27,7 +27,7 @@ import io.circe.Json // LRU Map import com.snowplowanalytics.iglu.core.circe.implicits._ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaList} -import com.snowplowanalytics.lrumap.{CreateLruMap, LruMap} +import com.snowplowanalytics.lrumap.LruMap // This project import com.snowplowanalytics.iglu.client.resolver.registries.{ @@ -89,15 +89,14 @@ object ResolverSpecHelpers { State(s => (s.copy(time = s.time + delta), ())) } - val staticCache: InitSchemaCache[StaticLookup] = - new CreateLruMap[StaticLookup, SchemaKey, SchemaCacheEntry] { - def create(size: Int): StaticLookup[LruMap[StaticLookup, SchemaKey, SchemaCacheEntry]] = + val staticResolverCache: CreateResolverCache[StaticLookup] = + new CreateResolverCache[StaticLookup] { + def createSchemaCache( + size: Int + ): StaticLookup[LruMap[StaticLookup, SchemaKey, SchemaCacheEntry]] = State(s => (s.copy(cacheSize = size), StateCache)) - } - val staticCacheForList: InitListCache[StaticLookup] = - new CreateLruMap[StaticLookup, ListCacheKey, ListCacheEntry] { - def create( + def createSchemaListCache( size: Int ): StaticLookup[LruMap[StaticLookup, ListCacheKey, ListCacheEntry]] = State { s => @@ -105,6 +104,9 @@ object ResolverSpecHelpers { val state = s.copy(cacheSize = size) (state, cache) } + + def createMutex[K]: StaticLookup[ResolverMutex[StaticLookup, K]] = + State.pure(stateMutex[K]) } val staticClock: Clock[StaticLookup] = @@ -215,4 +217,10 @@ object ResolverSpecHelpers { (state.copy(schemaLists = value :: state.schemaLists).tick, ()) } } + + private def stateMutex[K]: ResolverMutex[StaticLookup, K] = + new ResolverMutex[StaticLookup, K] { + def withLockOn[A](key: K)(f: => StaticLookup[A]): StaticLookup[A] = + f + } } diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala index 55866628..bfac8701 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala @@ -25,6 +25,7 @@ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} // This project import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup._ +import com.snowplowanalytics.iglu.client.resolver.registries.JavaNetRegistryLookup._ // Specs2 import com.snowplowanalytics.iglu.client.SpecHelpers diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/HttpSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/HttpSpec.scala index ab811e60..a6071baa 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/HttpSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/HttpSpec.scala @@ -28,6 +28,7 @@ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} // This project import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup._ +import com.snowplowanalytics.iglu.client.resolver.registries.JavaNetRegistryLookup._ // Specs2 import com.snowplowanalytics.iglu.client.SpecHelpers diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/CachingValidationSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/CachingValidationSpec.scala index bd59ca5e..719378f2 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/CachingValidationSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/CachingValidationSpec.scala @@ -15,7 +15,7 @@ package com.snowplowanalytics.iglu.client.validator // Cats import cats.Id import cats.data.NonEmptyList -import com.snowplowanalytics.iglu.client.resolver.Resolver.ResolverResult +import com.snowplowanalytics.iglu.client.resolver.Resolver.{ResolverResult, SchemaItem} import com.snowplowanalytics.iglu.client.validator.CirceValidator.WithCaching.{ SchemaEvaluationKey, SchemaEvaluationResult @@ -85,7 +85,10 @@ class CachingValidationSpec extends Specification { ) { json => val result = CirceValidator.WithCaching - .validate(createCache())(json, ResolverResult.NotCached(simpleSchemaResult)) + .validate(createCache())( + json, + ResolverResult.NotCached(SchemaItem(simpleSchemaResult, None)) + ) result must beRight } @@ -147,7 +150,7 @@ class CachingValidationSpec extends Specification { CirceValidator.WithCaching .validate(createCache())( nonStringInput, - ResolverResult.NotCached(simpleSchemaResult) + ResolverResult.NotCached(SchemaItem(simpleSchemaResult, None)) ) must beLeft( nonStringExpected ) @@ -155,7 +158,7 @@ class CachingValidationSpec extends Specification { CirceValidator.WithCaching .validate(createCache())( missingKeyInput, - ResolverResult.NotCached(simpleSchemaResult) + ResolverResult.NotCached(SchemaItem(simpleSchemaResult, None)) ) must beLeft( missingKeyExpected ) @@ -163,7 +166,7 @@ class CachingValidationSpec extends Specification { CirceValidator.WithCaching .validate(createCache())( heterogeneusArrayInput, - ResolverResult.NotCached(simpleSchemaResult) + ResolverResult.NotCached(SchemaItem(simpleSchemaResult, None)) ) must beLeft( heterogeneusArrayExpected ) @@ -171,7 +174,7 @@ class CachingValidationSpec extends Specification { CirceValidator.WithCaching .validate(createCache())( doubleErrorInput, - ResolverResult.NotCached(simpleSchemaResult) + ResolverResult.NotCached(SchemaItem(simpleSchemaResult, None)) ) must beLeft( doubleErrorExpected ) @@ -200,7 +203,10 @@ class CachingValidationSpec extends Specification { ) CirceValidator.WithCaching - .validate(createCache())(input, ResolverResult.NotCached(schema)) must beLeft( + .validate(createCache())( + input, + ResolverResult.NotCached(SchemaItem(schema, None)) + ) must beLeft( expected ) } @@ -216,7 +222,10 @@ class CachingValidationSpec extends Specification { val input = json"""{"shortKey": 5 }""" CirceValidator.WithCaching - .validate(createCache())(input, ResolverResult.NotCached(schema)) must beRight + .validate(createCache())( + input, + ResolverResult.NotCached(SchemaItem(schema, None)) + ) must beRight } def e5 = { @@ -246,7 +255,10 @@ class CachingValidationSpec extends Specification { ) CirceValidator.WithCaching - .validate(createCache())(input, ResolverResult.NotCached(schema)) must beLeft( + .validate(createCache())( + input, + ResolverResult.NotCached(SchemaItem(schema, None)) + ) must beLeft( expected ) } @@ -275,7 +287,10 @@ class CachingValidationSpec extends Specification { ) CirceValidator.WithCaching - .validate(createCache())(input, ResolverResult.NotCached(schema)) must beLeft( + .validate(createCache())( + input, + ResolverResult.NotCached(SchemaItem(schema, None)) + ) must beLeft( expected ) } @@ -284,28 +299,40 @@ class CachingValidationSpec extends Specification { val schema = json"""{ "type": "integer" }""" val input = json"""9223372036854775809""" CirceValidator.WithCaching - .validate(createCache())(input, ResolverResult.NotCached(schema)) must beRight + .validate(createCache())( + input, + ResolverResult.NotCached(SchemaItem(schema, None)) + ) must beRight } def e8 = { val schema = json"""{ "type": ["array", "null"], "items": {"type": "object"} }""" val input = json"""null""" CirceValidator.WithCaching - .validate(createCache())(input, ResolverResult.NotCached(schema)) must beRight + .validate(createCache())( + input, + ResolverResult.NotCached(SchemaItem(schema, None)) + ) must beRight } def e9 = { val schema = json"""{ "type": "integer" }""" val input = json""""5"""" CirceValidator.WithCaching - .validate(createCache())(input, ResolverResult.NotCached(schema)) must beLeft + .validate(createCache())( + input, + ResolverResult.NotCached(SchemaItem(schema, None)) + ) must beLeft } def e10 = { val schema = json"""{ "type": "number" }""" val input = json"""5""" CirceValidator.WithCaching - .validate(createCache())(input, ResolverResult.NotCached(schema)) must beRight + .validate(createCache())( + input, + ResolverResult.NotCached(SchemaItem(schema, None)) + ) must beRight } @@ -322,7 +349,10 @@ class CachingValidationSpec extends Specification { ) ) CirceValidator.WithCaching - .validate(createCache())(input, ResolverResult.NotCached(schema)) must beLeft( + .validate(createCache())( + input, + ResolverResult.NotCached(SchemaItem(schema, None)) + ) must beLeft( expected ) @@ -336,7 +366,10 @@ class CachingValidationSpec extends Specification { val input = json"""5""" val result = CirceValidator.WithCaching - .validate(cache)(input, ResolverResult.Cached(schemaKey, schema, timestamp = 1.seconds)) + .validate(cache)( + input, + ResolverResult.Cached(schemaKey, SchemaItem(schema, None), timestamp = 1.seconds) + ) result must beRight(()) and (cache.get((schemaKey, 1.seconds)) must beSome) @@ -348,7 +381,8 @@ class CachingValidationSpec extends Specification { val schema = json"""{ "type": "number" }""" val input = json"""5""" - val result = CirceValidator.WithCaching.validate(cache)(input, ResolverResult.NotCached(schema)) + val result = CirceValidator.WithCaching + .validate(cache)(input, ResolverResult.NotCached(SchemaItem(schema, None))) result must beRight(()) and (cache.get((schemaKey, 1.seconds)) must beNone) diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/SchemaValidationSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/SchemaValidationSpec.scala index ec96dacc..015a5790 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/SchemaValidationSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/SchemaValidationSpec.scala @@ -18,6 +18,7 @@ import io.circe.literal._ // Specs2 import com.snowplowanalytics.iglu.client.SpecHelpers +import com.snowplowanalytics.iglu.client.resolver.registries.JavaNetRegistryLookup._ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} import org.specs2.Specification diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/SelfDescValidationSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/SelfDescValidationSpec.scala index 105182da..3d74eb1e 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/SelfDescValidationSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/SelfDescValidationSpec.scala @@ -19,6 +19,7 @@ import io.circe.literal._ // Iglu Core import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} +import com.snowplowanalytics.iglu.client.resolver.registries.JavaNetRegistryLookup._ // Specs2 import com.snowplowanalytics.iglu.client.SpecHelpers diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/ValidatingWithRefSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/ValidatingWithRefSpec.scala new file mode 100644 index 00000000..ea7ae5dc --- /dev/null +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/validator/ValidatingWithRefSpec.scala @@ -0,0 +1,36 @@ +package com.snowplowanalytics.iglu.client.validator + +import io.circe.literal._ +import org.specs2.mutable.Specification + +class ValidatingWithRefSpec extends Specification { + + val schema = + json""" + { + "$$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Test schema using $$ref with 'http'/'https' protocols", + "self": { + "vendor": "com.test", + "name": "test", + "format": "jsonschema", + "version": "1-0-0" + }, + "properties": { + "id": { + "allOf": [ + {"$$ref": "http://json-schema.org/draft-04/schema#"}, + {"$$ref": "https://json-schema.org/draft-04/schema"}, + {"$$ref": "http://anything"}, + {"$$ref": "https://anything"} + ] + } + } + } + """ + + "Validator should ignore '$ref' keyword" in { + val data = json"""{"id": "can_be_anything1234"}""" + CirceValidator.validate(data, schema) must beRight(()) + } +} diff --git a/modules/data/src/main/scala/com.snowplowanalytics.iglu/client/ClientError.scala b/modules/data/src/main/scala/com.snowplowanalytics.iglu/client/ClientError.scala index 4345fbda..1ca16354 100644 --- a/modules/data/src/main/scala/com.snowplowanalytics.iglu/client/ClientError.scala +++ b/modules/data/src/main/scala/com.snowplowanalytics.iglu/client/ClientError.scala @@ -17,7 +17,7 @@ import java.time.Instant import cats.Show import cats.syntax.show._ import cats.syntax.either._ -import io.circe.{Decoder, DecodingFailure, Encoder, Json} +import io.circe.{Decoder, DecodingFailure, Encoder, Json, JsonObject} import io.circe.syntax._ import validator.ValidatorError import resolver.LookupHistory @@ -33,6 +33,8 @@ sealed trait ClientError extends Product with Serializable { object ClientError { + val SupersededByField = "supersededBy" + /** Error happened during schema resolution step */ final case class ResolutionError(value: SortedMap[String, LookupHistory]) extends ClientError { def isNotFound: Boolean = @@ -40,7 +42,8 @@ object ClientError { } /** Error happened during schema/instance validation step */ - final case class ValidationError(error: ValidatorError) extends ClientError + final case class ValidationError(error: ValidatorError, supersededBy: Option[String]) + extends ClientError implicit val igluClientResolutionErrorCirceEncoder: Encoder[ClientError] = Encoder.instance { @@ -52,8 +55,16 @@ object ClientError { lookups.asJson.deepMerge(Json.obj("repository" := repo.asJson)) } ) - case ValidationError(error) => - error.asJson.deepMerge(Json.obj("error" := Json.fromString("ValidationError"))) + case ValidationError(error, supersededBy) => + val errorTypeJson = Json.obj("error" := Json.fromString("ValidationError")) + val supersededByJson = supersededBy + .map { v => + Json.obj(SupersededByField -> v.asJson) + } + .getOrElse(JsonObject.empty.asJson) + error.asJson + .deepMerge(errorTypeJson) + .deepMerge(supersededByJson) } implicit val igluClientResolutionErrorCirceDecoder: Decoder[ClientError] = @@ -69,10 +80,11 @@ object ClientError { ResolutionError(SortedMap[String, LookupHistory]() ++ history.map(_.toField).toMap) } case "ValidationError" => + val supersededBy = cursor.downField(SupersededByField).as[String].toOption cursor .as[ValidatorError] .map { error => - ValidationError(error) + ValidationError(error, supersededBy) } case _ => DecodingFailure( @@ -87,7 +99,7 @@ object ClientError { implicit val igluClientShowInstance: Show[ClientError] = Show.show { - case ClientError.ValidationError(ValidatorError.InvalidData(reports)) => + case ClientError.ValidationError(ValidatorError.InvalidData(reports), _) => val issues = reports.toList .groupBy(_.path) .map { case (path, messages) => @@ -97,7 +109,7 @@ object ClientError { .mkString("\n") } s"Instance is not valid against its schema:\n${issues.mkString("\n")}" - case ClientError.ValidationError(ValidatorError.InvalidSchema(reports)) => + case ClientError.ValidationError(ValidatorError.InvalidSchema(reports), _) => val r = reports.toList.map(i => s"* [${i.message}] (at ${i.path})").mkString(",\n") s"Resolved schema cannot be used to validate an instance. Following issues found:\n$r" case ClientError.ResolutionError(lookup) => diff --git a/modules/data/src/main/scala/com.snowplowanalytics.iglu/client/validator/ValidatorError.scala b/modules/data/src/main/scala/com.snowplowanalytics.iglu/client/validator/ValidatorError.scala index 549208f4..a636656c 100644 --- a/modules/data/src/main/scala/com.snowplowanalytics.iglu/client/validator/ValidatorError.scala +++ b/modules/data/src/main/scala/com.snowplowanalytics.iglu/client/validator/ValidatorError.scala @@ -20,7 +20,8 @@ import cats.data.NonEmptyList /** ADT describing issues that can be discovered by Validator */ sealed trait ValidatorError extends Product with Serializable { - def toClientError: ClientError = ClientError.ValidationError(this) + def toClientError(supersededBy: Option[String]): ClientError = + ClientError.ValidationError(this, supersededBy) } object ValidatorError { diff --git a/modules/data/src/test/scala/com.snowplowanalytics.iglu.client/ClientErrorSpec.scala b/modules/data/src/test/scala/com.snowplowanalytics.iglu.client/ClientErrorSpec.scala index 4e1dc95d..3b6c5a13 100644 --- a/modules/data/src/test/scala/com.snowplowanalytics.iglu.client/ClientErrorSpec.scala +++ b/modules/data/src/test/scala/com.snowplowanalytics.iglu.client/ClientErrorSpec.scala @@ -95,12 +95,14 @@ class ClientErrorSpec extends Specification { ValidatorReport("Something went wrong again", None, Nil, None), ValidatorReport("Something went wrong with targets", None, List("type", "property"), None) ) - ) + ), + Some("1-0-0") ) val json = json"""{ "error" : "ValidationError", + "supersededBy": "1-0-0", "dataReports" : [ { "message" : "Something went wrong", diff --git a/modules/http4s/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Http4sRegistryLookup.scala b/modules/http4s/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Http4sRegistryLookup.scala index 3c45fd8d..7f162d95 100644 --- a/modules/http4s/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Http4sRegistryLookup.scala +++ b/modules/http4s/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Http4sRegistryLookup.scala @@ -29,29 +29,23 @@ import scala.util.control.NonFatal object Http4sRegistryLookup { def apply[F[_]: Async](client: HttpClient[F]): RegistryLookup[F] = - new RegistryLookup[F] { - def lookup(repositoryRef: Registry, schemaKey: SchemaKey): F[Either[RegistryError, Json]] = - repositoryRef match { - case Registry.Http(_, connection) => httpLookup(client, connection, schemaKey).value - case Registry.Embedded(_, path) => RegistryLookup.embeddedLookup[F](path, schemaKey) - case Registry.InMemory(_, schemas) => - Async[F].pure(RegistryLookup.inMemoryLookup(schemas, schemaKey)) - } + new RegistryLookup.StdRegistryLookup[F] { + def httpLookup( + registry: Registry.Http, + schemaKey: SchemaKey + ): F[Either[RegistryError, Json]] = + lookupImpl(client, registry.http, schemaKey).value - def list( - registry: Registry, + def httpList( + registry: Registry.Http, vendor: String, name: String, model: Int ): F[Either[RegistryError, SchemaList]] = - registry match { - case Registry.Http(_, connection) => - httpList(client, connection, vendor, name, model).value - case _ => Async[F].pure(RegistryError.NotFound.asLeft) - } + listImpl(client, registry.http, vendor, name, model).value } - def httpLookup[F[_]: Concurrent]( + private def lookupImpl[F[_]: Concurrent]( client: HttpClient[F], http: Registry.HttpConnection, key: SchemaKey @@ -66,7 +60,7 @@ object Http4sRegistryLookup { result <- EitherT(response) } yield result - def httpList[F[_]: Concurrent]( + private def listImpl[F[_]: Concurrent]( client: HttpClient[F], http: Registry.HttpConnection, vendor: String, @@ -85,12 +79,12 @@ object Http4sRegistryLookup { result <- EitherT(response) } yield result - def toPath(cxn: Registry.HttpConnection, key: SchemaKey): Either[RegistryError, Uri] = + private def toPath(cxn: Registry.HttpConnection, key: SchemaKey): Either[RegistryError, Uri] = Uri .fromString(s"${cxn.uri.toString.stripSuffix("/")}/schemas/${key.toPath}") .leftMap(e => RegistryError.ClientFailure(e.message)) - def toSubpath( + private def toSubpath( cxn: Registry.HttpConnection, vendor: String, name: String, @@ -100,7 +94,7 @@ object Http4sRegistryLookup { .fromString(s"${cxn.uri.toString.stripSuffix("/")}/schemas/$vendor/$name/jsonschema/$model") .leftMap(e => RegistryError.ClientFailure(e.message)) - def runRequest[F[_]: Concurrent, A: EntityDecoder[F, *]]( + private def runRequest[F[_]: Concurrent, A: EntityDecoder[F, *]]( client: HttpClient[F], req: Request[F] ): F[Either[RegistryError, A]] = { @@ -137,6 +131,6 @@ object Http4sRegistryLookup { } } - implicit def schemaListDecoder[F[_]: Concurrent]: EntityDecoder[F, SchemaList] = + private implicit def schemaListDecoder[F[_]: Concurrent]: EntityDecoder[F, SchemaList] = jsonOf[F, SchemaList] } diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 45a4c18f..fdbfc033 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -68,9 +68,9 @@ object BuildSettings { // clear-out mimaBinaryIssueFilters and mimaPreviousVersions. // Otherwise, add previous version to set without // removing other versions. - val mimaPreviousVersionsData = Set("2.1.0", "2.2.0") - val mimaPreviousVersionsCore = Set("2.1.0", "2.2.0") - val mimaPreviousVersionsHttp4s = Set("2.0.0", "2.1.0", "2.2.0") + val mimaPreviousVersionsData = Set() + val mimaPreviousVersionsCore = Set() + val mimaPreviousVersionsHttp4s = Set() val mimaPreviousVersionsScala3 = Set() lazy val mimaSettings = Seq(