From 6e68bb60b9b532bd72b0fba666ee18f6637503a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Tue, 31 May 2022 14:02:21 +0200 Subject: [PATCH 1/6] Prepare for 2.1.0 release --- CHANGELOG | 13 +++++++++++++ README.md | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 6ae01149..d2483741 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,16 @@ +Version 2.1.0 (2022-05-31) +-------------------------- +Bump specs2 to 4.15.0 (#198) +Bump http4s to 0.23.12 (#197) +Bump scala-collection-compat to 2.7.0 (#196) +Bump circe to 0.14.2 (#195) +Bump cats-effect to 3.3.12 (#194) +Bump jackson-databind to 2.13.3 (#192) +Bump json-schema-validator to 1.0.70 (#193) +Replace deprecated scalaj http client (#190) +Use jsonschema v4 text from iglu-scala-core (#186) +Split out a iglu-scala-client-data module (#187) + Version 2.0.0 (2022-03-28) -------------------------- Bump jackson-databind to 2.13.1 (#185) diff --git a/README.md b/README.md index 5bbdf186..23fc437e 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,12 @@ Iglu Scala Client is used extensively in **[Snowplow][snowplow-repo]** to valida ## Installation -The latest version of Iglu Scala Client is 2.0.0, which currently works with Scala 2.12 and 2.13. +The latest version of Iglu Scala Client is 2.1.0, which currently works with Scala 2.12 and 2.13. If you're using SBT, add the following lines to your build file: ```scala -val igluClient = "com.snowplowanalytics" %% "iglu-scala-client" % "2.0.0" +val igluClient = "com.snowplowanalytics" %% "iglu-scala-client" % "2.1.0" ``` ## API From f221c0a978773c1cc050d41c69e572739a6bf16f Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Sat, 12 Nov 2022 00:29:42 +0000 Subject: [PATCH 2/6] Add list-schemas for the embedded repo (close #212) --- .../resolver/registries/RegistryLookup.scala | 22 +++++++- .../client/resolver/registries/Utils.scala | 53 ++++++++++++++++++- .../test-embedded-list/jsonschema/1-0-0 | 0 .../test-embedded-list/jsonschema/1-0-1 | 0 .../test-embedded-list/jsonschema/1-2-0 | 0 .../test-embedded-list/jsonschema/1-2-11 | 0 .../resolver/registries/EmbeddedSpec.scala | 13 +++++ 7 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-0-0 create mode 100644 modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-0-1 create mode 100644 modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-2-0 create mode 100644 modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-2-11 diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala index f59ec598..551738df 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala @@ -79,6 +79,13 @@ object RegistryLookup { implicit class LookupOps(val repositoryRef: Registry) extends AnyVal with Serializable { def lookupSchema[F[_]: RegistryLookup](schemaKey: SchemaKey): F[Either[RegistryError, Json]] = RegistryLookup[F].lookup(repositoryRef, schemaKey) + + def list[F[_]: RegistryLookup]( + vendor: String, + name: String, + model: Int + ): F[Either[RegistryError, SchemaList]] = + RegistryLookup[F].list(repositoryRef, vendor: String, name: String, model: Int) } implicit def ioLookupInstance[F[_]](implicit F: Sync[F]): RegistryLookup[F] = @@ -98,7 +105,10 @@ object RegistryLookup { ): F[Either[RegistryError, SchemaList]] = registry match { case Registry.Http(_, connection) => httpList(connection, vendor, name, model) - case _ => F.pure(RegistryError.NotFound.asLeft) + case Registry.Embedded(_, base) => + val path = toSubpath(base, vendor, name) + Utils.unsafeEmbeddedList(path, model).pure[F] + case _ => F.pure(RegistryError.NotFound.asLeft) } } @@ -128,6 +138,9 @@ object RegistryLookup { case Registry.Http(_, connection) => val subpath = toSubpath(connection.uri.toString, vendor, name, model) Utils.stringToUri(subpath).flatMap(Utils.unsafeHttpList(_, connection.apikey)) + case Registry.Embedded(_, base) => + val path = toSubpath(base, vendor, name) + Utils.unsafeEmbeddedList(path, model) case _ => RegistryError.NotFound.asLeft } @@ -151,6 +164,13 @@ object RegistryLookup { ): String = s"${prefix.stripSuffix("/")}/schemas/$vendor/$name/jsonschema/$model" + private def toSubpath( + prefix: String, + vendor: String, + name: String + ): String = + s"${prefix.stripSuffix("/")}/schemas/$vendor/$name/jsonschema" + /** * Retrieves an Iglu Schema from the Embedded Iglu Repo as a JSON * diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala index bfa7eaf7..f2f3a0e0 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala @@ -14,11 +14,14 @@ package com.snowplowanalytics.iglu.client package resolver.registries // Java -import java.io.InputStream +import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} + +import java.io.{File, InputStream} import java.net.URI import java.net.http.HttpResponse.BodyHandlers import java.net.http.{HttpClient, HttpRequest, HttpResponse} import java.time.Duration +import scala.util.matching.Regex // Scala import scala.io.Source @@ -29,6 +32,7 @@ import cats.effect.Sync import cats.syntax.either._ import cats.syntax.option._ import cats.syntax.show._ +import cats.syntax.traverse._ // circe import io.circe.parser.parse @@ -72,6 +76,40 @@ private[registries] object Utils { repoFailure(e).asLeft } + def unsafeEmbeddedList(path: String, modelMatch: Int): Either[RegistryError, SchemaList] = + try { + val d = new File(getClass.getResource(path).getPath) + val schemaFileRegex: Regex = (".*?" + // path to file + "([a-zA-Z0-9-_.]+)/" + // Vendor + "([a-zA-Z0-9-_]+)/" + // Name + "([a-zA-Z0-9-_]+)/" + // Format + "([1-9][0-9]*)-(\\d+)-(\\d+)$").r // MODEL, REVISION and ADDITION + if (d.exists && d.isDirectory) { + d.listFiles + .filter(_.isFile) + .toList + .filter(_.getName.startsWith(modelMatch.toString)) + .traverse(_.getAbsolutePath match { + case schemaFileRegex(vendor, name, format, model, revision, addition) => + SchemaKey( + vendor = vendor, + name = name, + format = format, + version = SchemaVer + .Full(model = model.toInt, revision = revision.toInt, addition = addition.toInt) + ).asRight + case f => RegistryError.RepoFailure(s"Corrupted schema file name at $f").asLeft + }) + .map(_.sortBy(_.version)) + .map(SchemaList.apply) + } else { + RegistryError.NotFound.asLeft + } + } catch { + case NonFatal(e) => + repoFailure(e).asLeft + } + /** Not-RT analog of [[RegistryLookup.embeddedLookup]] */ def unsafeEmbeddedLookup(path: String): Either[RegistryError, Json] = try { @@ -158,4 +196,17 @@ private[registries] object Utils { private[resolver] def repoFailure(failure: Throwable): RegistryError = RegistryError.RepoFailure(failure.getMessage) + implicit val orderingSchemaKey: Ordering[SchemaKey] = new Ordering[SchemaKey] { + override def compare(x: SchemaKey, y: SchemaKey): Int = + if (x == y) + 0 + else if (x.version.model < y.version.model) + -1 + else if (x.version.revision < y.version.revision) + -1 + else if (x.version.addition < y.version.addition) + -1 + else + 1 + } } diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-0-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-0-0 new file mode 100644 index 00000000..e69de29b diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-0-1 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-0-1 new file mode 100644 index 00000000..e69de29b diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-2-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-2-0 new file mode 100644 index 00000000..e69de29b diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-2-11 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/test-embedded-list/jsonschema/1-2-11 new file mode 100644 index 00000000..e69de29b diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala index 2bbba72a..5c16e937 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala @@ -15,6 +15,7 @@ package com.snowplowanalytics.iglu.client.resolver.registries // Cats import cats.effect.IO import cats.effect.testing.specs2.CatsEffect +import com.snowplowanalytics.iglu.core.SchemaList // circe import io.circe.literal._ @@ -38,6 +39,7 @@ class EmbeddedSpec extends Specification with CatsEffect { retrieving an existent JSON Schema from an embedded RepositoryRef should work $e3 requesting a non-existent JSON Schema from an embedded RepositoryRef should return None $e4 requesting a corrupted JSON Schema from an embedded RepositoryRef should return an appropriate Failure $e5 + Schema list should work for embedded repo $e6 """ val AcmeConfig = @@ -119,4 +121,15 @@ class EmbeddedSpec extends Specification with CatsEffect { .map(result => result must beLeft) } + def e6 = { + val schemaList = SchemaList(List( + SchemaKey("com.snowplowanalytics.iglu-test", "test-embedded-list", "jsonschema", SchemaVer.Full(1, 0, 0)), + SchemaKey("com.snowplowanalytics.iglu-test", "test-embedded-list", "jsonschema", SchemaVer.Full(1, 0, 1)), + SchemaKey("com.snowplowanalytics.iglu-test", "test-embedded-list", "jsonschema", SchemaVer.Full(1, 2, 0)), + SchemaKey("com.snowplowanalytics.iglu-test", "test-embedded-list", "jsonschema", SchemaVer.Full(1, 2, 11)), + )) + SpecHelpers.EmbeddedTest + .list[IO]("com.snowplowanalytics.iglu-test", "test-embedded-list", 1) + .map(result => result must beRight(schemaList)) + } } From 17376f84003ce50c96c52db168b2e11d4a331b56 Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Sun, 13 Nov 2022 22:57:16 +0000 Subject: [PATCH 3/6] ordering --- .../client/resolver/registries/Utils.scala | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala index f2f3a0e0..c74822f1 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala @@ -196,17 +196,5 @@ private[registries] object Utils { private[resolver] def repoFailure(failure: Throwable): RegistryError = RegistryError.RepoFailure(failure.getMessage) - implicit val orderingSchemaKey: Ordering[SchemaKey] = new Ordering[SchemaKey] { - override def compare(x: SchemaKey, y: SchemaKey): Int = - if (x == y) - 0 - else if (x.version.model < y.version.model) - -1 - else if (x.version.revision < y.version.revision) - -1 - else if (x.version.addition < y.version.addition) - -1 - else - 1 - } + implicit val orderingSchemaKey: Ordering[SchemaKey] = SchemaKey.ordering } From d1bd0770dce851769dc5ca60fb047afe72477e7a Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Tue, 15 Nov 2022 22:42:58 +0000 Subject: [PATCH 4/6] review feedback --- .../resolver/registries/RegistryLookup.scala | 2 +- .../client/resolver/registries/Utils.scala | 72 ++++++++++++------- .../resolver/registries/EmbeddedSpec.scala | 34 +++++++-- 3 files changed, 77 insertions(+), 31 deletions(-) diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala index 551738df..ee718fa2 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/RegistryLookup.scala @@ -107,7 +107,7 @@ object RegistryLookup { case Registry.Http(_, connection) => httpList(connection, vendor, name, model) case Registry.Embedded(_, base) => val path = toSubpath(base, vendor, name) - Utils.unsafeEmbeddedList(path, model).pure[F] + Sync[F].delay(Utils.unsafeEmbeddedList(path, model)) case _ => F.pure(RegistryError.NotFound.asLeft) } } diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala index c74822f1..34f4ee56 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala @@ -78,36 +78,57 @@ private[registries] object Utils { def unsafeEmbeddedList(path: String, modelMatch: Int): Either[RegistryError, SchemaList] = try { - val d = new File(getClass.getResource(path).getPath) - val schemaFileRegex: Regex = (".*?" + // path to file + val d = + new File( + getClass.getResource(path).getPath + ) // this will throw NPE for missing entry in embedded repos + val schemaFileRegex: Regex = (".*/schemas/?" + // path to file "([a-zA-Z0-9-_.]+)/" + // Vendor "([a-zA-Z0-9-_]+)/" + // Name "([a-zA-Z0-9-_]+)/" + // Format "([1-9][0-9]*)-(\\d+)-(\\d+)$").r // MODEL, REVISION and ADDITION - if (d.exists && d.isDirectory) { + + def getFolderContent(d: File): List[String] = { d.listFiles .filter(_.isFile) .toList - .filter(_.getName.startsWith(modelMatch.toString)) - .traverse(_.getAbsolutePath match { - case schemaFileRegex(vendor, name, format, model, revision, addition) => - SchemaKey( - vendor = vendor, - name = name, - format = format, - version = SchemaVer - .Full(model = model.toInt, revision = revision.toInt, addition = addition.toInt) - ).asRight - case f => RegistryError.RepoFailure(s"Corrupted schema file name at $f").asLeft - }) - .map(_.sortBy(_.version)) - .map(SchemaList.apply) - } else { - RegistryError.NotFound.asLeft + .filter(_.getName.startsWith(s"${modelMatch.toString}-")) + .map(_.getAbsolutePath) } + + val content = + if (d.exists & d.isDirectory) + getFolderContent(d) + else + List.empty[String] + + content + .traverse { + case schemaFileRegex(vendor, name, format, model, revision, addition) + if model == modelMatch.toString => + SchemaKey( + vendor = vendor, + name = name, + format = format, + version = SchemaVer + .Full(model = model.toInt, revision = revision.toInt, addition = addition.toInt) + ).asRight + case f => RegistryError.RepoFailure(s"Corrupted schema file name at $f").asLeft + } + .map(_.sortBy(_.version)) + .flatMap(s => + if (s.isEmpty) + RegistryError.NotFound.asLeft + else + s.asRight + ) + .map(SchemaList.apply) } catch { case NonFatal(e) => - repoFailure(e).asLeft + e match { + case NullPointerException => RegistryError.NotFound.asLeft + case _ => repoFailure(e).asLeft + } } /** Not-RT analog of [[RegistryLookup.embeddedLookup]] */ @@ -122,7 +143,10 @@ private[registries] object Utils { result } catch { case NonFatal(e) => - repoFailure(e).asLeft + e match { + case NullPointerException => RegistryError.NotFound.asLeft + case _ => repoFailure(e).asLeft + } } /** Non-RT analog of [[RegistryLookup.httpList]] */ @@ -194,7 +218,7 @@ private[registries] object Utils { RegistryError.RepoFailure(failure.show) private[resolver] def repoFailure(failure: Throwable): RegistryError = - RegistryError.RepoFailure(failure.getMessage) - - implicit val orderingSchemaKey: Ordering[SchemaKey] = SchemaKey.ordering + RegistryError.RepoFailure( + if (failure.getMessage != null) failure.getMessage else "Unhandled error" + ) } diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala index 5c16e937..1c8bb1e1 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/registries/EmbeddedSpec.scala @@ -122,12 +122,34 @@ class EmbeddedSpec extends Specification with CatsEffect { } def e6 = { - val schemaList = SchemaList(List( - SchemaKey("com.snowplowanalytics.iglu-test", "test-embedded-list", "jsonschema", SchemaVer.Full(1, 0, 0)), - SchemaKey("com.snowplowanalytics.iglu-test", "test-embedded-list", "jsonschema", SchemaVer.Full(1, 0, 1)), - SchemaKey("com.snowplowanalytics.iglu-test", "test-embedded-list", "jsonschema", SchemaVer.Full(1, 2, 0)), - SchemaKey("com.snowplowanalytics.iglu-test", "test-embedded-list", "jsonschema", SchemaVer.Full(1, 2, 11)), - )) + val schemaList = SchemaList( + List( + SchemaKey( + "com.snowplowanalytics.iglu-test", + "test-embedded-list", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ), + SchemaKey( + "com.snowplowanalytics.iglu-test", + "test-embedded-list", + "jsonschema", + SchemaVer.Full(1, 0, 1) + ), + SchemaKey( + "com.snowplowanalytics.iglu-test", + "test-embedded-list", + "jsonschema", + SchemaVer.Full(1, 2, 0) + ), + SchemaKey( + "com.snowplowanalytics.iglu-test", + "test-embedded-list", + "jsonschema", + SchemaVer.Full(1, 2, 11) + ) + ) + ) SpecHelpers.EmbeddedTest .list[IO]("com.snowplowanalytics.iglu-test", "test-embedded-list", 1) .map(result => result must beRight(schemaList)) From 1d818a78692f0e6c0399096803ca9d1065b099e1 Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Wed, 16 Nov 2022 00:52:56 +0000 Subject: [PATCH 5/6] NPE --- .../client/resolver/registries/Utils.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala index 34f4ee56..07fcc6b1 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala @@ -144,8 +144,8 @@ private[registries] object Utils { } catch { case NonFatal(e) => e match { - case NullPointerException => RegistryError.NotFound.asLeft - case _ => repoFailure(e).asLeft + case _: NullPointerException => RegistryError.NotFound.asLeft + case _ => repoFailure(e).asLeft } } From 3ecc8b83a4a4b42a27a1d8a1274261251e72b989 Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Wed, 16 Nov 2022 22:57:37 +0000 Subject: [PATCH 6/6] NPE --- .../client/resolver/registries/Utils.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala index 07fcc6b1..bbf62f20 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/registries/Utils.scala @@ -126,8 +126,8 @@ private[registries] object Utils { } catch { case NonFatal(e) => e match { - case NullPointerException => RegistryError.NotFound.asLeft - case _ => repoFailure(e).asLeft + case _: NullPointerException => RegistryError.NotFound.asLeft + case _ => repoFailure(e).asLeft } }