From 2f9d3500b8fcf3c8d0f253d02a75e75dd44a1677 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Tue, 14 Jun 2022 11:06:11 +0200 Subject: [PATCH 1/7] Migrate to magnolia1 --- build.sbt | 14 ++++++------ .../spotify/scio/coders/DerivedCoder.scala | 22 +++++++++---------- .../LowPrioritySchemaDerivation.scala | 10 ++++----- .../com/spotify/scio/MagnoliaMacros.scala | 3 ++- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/build.sbt b/build.sbt index 6a11cfeef6..5f46e50619 100644 --- a/build.sbt +++ b/build.sbt @@ -108,8 +108,8 @@ val junitVersion = "4.13.2" val kantanCodecsVersion = "0.5.1" val kantanCsvVersion = "0.6.2" val kryoVersion = "4.0.2" -val magnoliaVersion = "1.0.0-M4" -val magnolifyVersion = "0.4.8" +val magnoliaVersion = "1.1.2" +val magnolifyVersion = "0.4.8+10-0c6f9747-SNAPSHOT" val metricsVersion = "3.2.6" val parquetExtraVersion = "0.4.3" val parquetVersion = "1.12.3" @@ -219,7 +219,7 @@ val commonSettings = Def excludeDependencies ++= Seq( "org.apache.beam" % "beam-sdks-java-io-kafka" ), - resolvers += Resolver.sonatypeRepo("public"), + resolvers ++= Seq(Resolver.sonatypeRepo("public"), Resolver.sonatypeRepo("snapshots")), Test / javaOptions += "-Dscio.ignoreVersionWarning=true", Test / testOptions += Tests.Argument("-oD"), testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v", "-a"), @@ -512,7 +512,7 @@ lazy val `scio-core`: Project = project "org.slf4j" % "slf4j-api" % slf4jVersion, "org.typelevel" %% "algebra" % algebraVersion, "org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion, - "com.softwaremill.magnolia" %% "magnolia-core" % magnoliaVersion + "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaVersion ), buildInfoKeys := Seq[BuildInfoKey](scalaVersion, version, "beamVersion" -> beamVersion), buildInfoPackage := "com.spotify.scio" @@ -564,7 +564,7 @@ lazy val `scio-test`: Project = project "org.apache.beam" % "beam-sdks-java-core" % beamVersion, "org.hamcrest" % "hamcrest" % hamcrestVersion, "org.scalactic" %% "scalactic" % scalacticVersion, - "com.softwaremill.magnolia" %% "magnolia-core" % magnoliaVersion + "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaVersion ), Test / compileOrder := CompileOrder.JavaThenScala, Test / testGrouping := splitTests( @@ -592,7 +592,7 @@ lazy val `scio-macros`: Project = project "com.esotericsoftware" % "kryo-shaded" % kryoVersion, "org.apache.beam" % "beam-sdks-java-extensions-sql" % beamVersion, "org.apache.avro" % "avro" % avroVersion, - "com.softwaremill.magnolia" %% "magnolia-core" % magnoliaVersion + "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaVersion ) ) @@ -1019,7 +1019,7 @@ lazy val `scio-examples`: Project = project "org.apache.beam" % "beam-sdks-java-extensions-sql" % beamVersion, "org.apache.httpcomponents" % "httpcore" % httpCoreVersion, "org.elasticsearch" % "elasticsearch" % elasticsearch7Version, - "com.softwaremill.magnolia" %% "magnolia-core" % magnoliaVersion + "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaVersion ), // exclude problematic sources if we don't have GCP credentials unmanagedSources / excludeFilter := { diff --git a/scio-core/src/main/scala/com/spotify/scio/coders/DerivedCoder.scala b/scio-core/src/main/scala/com/spotify/scio/coders/DerivedCoder.scala index 588068f5da..6f19b6259a 100644 --- a/scio-core/src/main/scala/com/spotify/scio/coders/DerivedCoder.scala +++ b/scio-core/src/main/scala/com/spotify/scio/coders/DerivedCoder.scala @@ -18,7 +18,7 @@ package com.spotify.scio.coders private object Derived extends Serializable { - import magnolia._ + import magnolia1._ @inline private def catching[T](msg: => String)(v: => T): T = try { @@ -32,7 +32,7 @@ private object Derived extends Serializable { throw CoderStackTrace.append(e, msg) } - def combineCoder[T]( + def joinCoder[T]( typeName: TypeName, ps: Seq[Param[Coder, T]], rawConstruct: Seq[Any] => T @@ -72,23 +72,23 @@ private object Derived extends Serializable { } trait LowPriorityCoderDerivation { - import magnolia._ + import magnolia1._ type Typeclass[T] = Coder[T] - def combine[T](ctx: CaseClass[Coder, T]): Coder[T] = + def join[T](ctx: CaseClass[Coder, T]): Coder[T] = if (ctx.isValueClass) { - Coder.xmap(ctx.parameters(0).typeclass.asInstanceOf[Coder[Any]])( + Coder.xmap(ctx.parameters.head.typeclass.asInstanceOf[Coder[Any]])( a => ctx.rawConstruct(Seq(a)), - ctx.parameters(0).dereference + ctx.parameters.head.dereference ) } else { - Derived.combineCoder(ctx.typeName, ctx.parameters, ctx.rawConstruct) + Derived.joinCoder(ctx.typeName, ctx.parameters, ctx.rawConstruct) } - def dispatch[T](sealedTrait: SealedTrait[Coder, T]): Coder[T] = { + def split[T](sealedTrait: SealedTrait[Coder, T]): Coder[T] = { val typeName = sealedTrait.typeName.full - val idx: Map[magnolia.TypeName, Int] = + val idx: Map[TypeName, Int] = sealedTrait.subtypes.map(_.typeName).zipWithIndex.toMap val coders: Map[Int, Coder[T]] = sealedTrait.subtypes @@ -101,11 +101,11 @@ trait LowPriorityCoderDerivation { val booleanId: Int => Boolean = _ != 0 val cs = coders.map { case (key, v) => (booleanId(key), v) } Coder.disjunction[T, Boolean](typeName, cs) { t => - sealedTrait.dispatch(t)(subtype => booleanId(idx(subtype.typeName))) + sealedTrait.split(t)(subtype => booleanId(idx(subtype.typeName))) } } else { Coder.disjunction[T, Int](typeName, coders) { t => - sealedTrait.dispatch(t)(subtype => idx(subtype.typeName)) + sealedTrait.split(t)(subtype => idx(subtype.typeName)) } } } diff --git a/scio-core/src/main/scala/com/spotify/scio/schemas/instances/LowPrioritySchemaDerivation.scala b/scio-core/src/main/scala/com/spotify/scio/schemas/instances/LowPrioritySchemaDerivation.scala index c984ee735b..2cdaa7a643 100644 --- a/scio-core/src/main/scala/com/spotify/scio/schemas/instances/LowPrioritySchemaDerivation.scala +++ b/scio-core/src/main/scala/com/spotify/scio/schemas/instances/LowPrioritySchemaDerivation.scala @@ -20,8 +20,8 @@ package com.spotify.scio.schemas.instances import com.spotify.scio.schemas._ private object Derived extends Serializable { - import magnolia._ - def combineSchema[T](ps: Seq[Param[Schema, T]], rawConstruct: Seq[Any] => T): Record[T] = { + import magnolia1._ + def joinSchema[T](ps: Seq[Param[Schema, T]], rawConstruct: Seq[Any] => T): Record[T] = { @inline def destruct(v: T): Array[Any] = { val arr = new Array[Any](ps.length) var i = 0 @@ -39,11 +39,11 @@ private object Derived extends Serializable { } trait LowPrioritySchemaDerivation { - import magnolia._ + import magnolia1._ type Typeclass[T] = Schema[T] - def combine[T](ctx: CaseClass[Schema, T]): Record[T] = - Derived.combineSchema(ctx.parameters, ctx.rawConstruct) + def join[T](ctx: CaseClass[Schema, T]): Record[T] = + Derived.joinSchema(ctx.parameters, ctx.rawConstruct) import com.spotify.scio.MagnoliaMacros implicit def gen[T]: Schema[T] = macro MagnoliaMacros.genWithoutAnnotations[T] diff --git a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala index 64b89a2ce9..b596cbe2df 100644 --- a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala +++ b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala @@ -20,6 +20,7 @@ package com.spotify.scio import scala.reflect.macros._ private[scio] object MagnoliaMacros { + import magnolia1._ // Add a level of indirection to prevent the macro from capturing // $outer which would make the Coder serialization fail @@ -34,7 +35,7 @@ private[scio] object MagnoliaMacros { ) } - val magnoliaTree = magnolia.Magnolia.gen[T](c) + val magnoliaTree = Magnolia.gen[T](c) // Remove annotations from magnolia since they are // not serializable and we don't use them anyway From 150930d1f2b78f0af2421fa370f33134ad1231a7 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Tue, 14 Jun 2022 17:04:15 +0200 Subject: [PATCH 2/7] Update doc links --- site/src/main/paradox/Changelog.md | 2 +- site/src/main/paradox/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/site/src/main/paradox/Changelog.md b/site/src/main/paradox/Changelog.md index 839e59b622..d42f013d08 100644 --- a/site/src/main/paradox/Changelog.md +++ b/site/src/main/paradox/Changelog.md @@ -33,7 +33,7 @@ ## Breaking changes since Scio 0.7.0 (@ref:[v0.7.0 Migration Guide](migrations/v0.7.0-Migration-Guide.md)) -- New [Magnolia](https://github.com/propensive/magnolia) based @ref:[Coders](internals/Coders.md) derivation +- New [Magnolia](https://github.com/softwaremill/magnolia) based @ref:[Coders](internals/Coders.md) derivation - New @ref:[ScioIO](internals/ScioIO.md) replaces `TestIO[T]` to simplify IO implementation and stubbing in `JobTest` ## Breaking changes since Scio 0.6.0 diff --git a/site/src/main/paradox/index.md b/site/src/main/paradox/index.md index 448d2ee74d..a2a7ee5468 100644 --- a/site/src/main/paradox/index.md +++ b/site/src/main/paradox/index.md @@ -45,7 +45,7 @@ See @scaladoc[Scio Scaladocs](com.spotify.scio.index) for current API documenta - @ref:[ScioIO](internals/ScioIO.md) - new IO system to simplify implementation and stubbing in `JobTest` - @ref:[OverrideTypeProvider](internals/OverrideTypeProvider.md) - custom mappings for type-safe BigQuery - @ref:[Kryo](internals/Kryo.md) - [Kryo](https://github.com/EsotericSoftware/kryo) data serialization - - @ref:[Coders](internals/Coders.md) - new [Magnolia](https://github.com/propensive/magnolia) based Coders derivation + - @ref:[Coders](internals/Coders.md) - new [Magnolia](https://github.com/softwaremill/magnolia) based Coders derivation ## Further Readings - [Spotify Unwrapped: How We Brought You a Decade of Data](https://engineering.atspotify.com/2020/02/18/spotify-unwrapped-how-we-brought-you-a-decade-of-data/) From 96f0b3a622abb6dd1e0f90c6247c9745e0b3688c Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Tue, 14 Jun 2022 17:05:33 +0200 Subject: [PATCH 3/7] Remove magnolia1 annotations --- .../com/spotify/scio/MagnoliaMacros.scala | 23 ++++++++++--------- .../com/spotify/scio/coders/CoderTest.scala | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala index b596cbe2df..632cfb3d24 100644 --- a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala +++ b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala @@ -37,33 +37,34 @@ private[scio] object MagnoliaMacros { val magnoliaTree = Magnolia.gen[T](c) + // format:off // Remove annotations from magnolia since they are // not serializable and we don't use them anyway - val removeAnnotations = new Transformer { - override def transform(tree: Tree): c.universe.Tree = + override def transform(tree: Tree): c.universe.Tree = { tree match { - case Apply(tt: TypeTree, List(typeName, isObject, isValueClass, params, _, _)) - if tt.symbol.name == TypeName("CaseClass") => + case q"$caseClass($typeName, $isObject, $isValueClass, $parametersArray, $_, $_, $_)" if caseClass.symbol.name == TypeName("CaseClass") => super.transform( - Apply(tt, List(typeName, isObject, isValueClass, params, q"Array()", q"Array()")) + q"$caseClass($typeName, $isObject, $isValueClass, $parametersArray, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"Param.apply[$tc, $t, $p]($name, $tpn, $idx, $repeated, $tcParam, $defaultVal, $_, $_)" => + case q"Param.apply[$tc_, $t_, $p_]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, $_, $_, $_)" => super.transform( - q"_root_.magnolia.Param.apply[$tc, $t, $p]($name, $tpn, $idx, $repeated, $tcParam, $defaultVal, Array(), Array())" + q"_root_.magnolia1.Param[$tc_, $t_, $p_]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"new SealedTrait($typeName, $subtypes, $_, $_)" => + case q"new SealedTrait($typeName, $subtypesArray, $_, $_, $_)" => super.transform( - q"new _root_.magnolia.SealedTrait($typeName, $subtypes, Array(), Array())" + q"new _root_.magnolia1.SealedTrait($typeName, $subtypesArray, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"Subtype[$tc, $t, $p]($typeName, $id, $_, $_, $coder, $cast0, $cast1)" => + case q"Subtype[$tc_, $t_, $s_]($name, $idx, $_, $_, $_, $tc, $isType, $asType)" => super.transform( - q"_root_.magnolia.Subtype[$tc, $t, $p]($typeName, $id, Array(), Array(), $coder, $cast0, $cast1)" + q"_root_.magnolia1.Subtype[$tc_, $t_,$s_]($name, $idx, Array.empty[Any], Array.empty[Any], Array.empty[Any], $tc, $isType, $asType)" ) case t => super.transform(t) } + } } + // format:on removeAnnotations.transform(magnoliaTree) } diff --git a/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala b/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala index c053f2e98f..2e23d01b14 100644 --- a/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala +++ b/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala @@ -531,7 +531,7 @@ final class CoderTest extends AnyFlatSpec with Matchers { materializationStackTrace.map(_.getFileName) should contain("CoderTest.scala") } - it should "#1651: remove all anotations from derived coders" in { + it should "#1651: remove all annotations from derived coders" in { coderIsSerializable[TraitWithAnnotation] } From 7a445122c0bcbb5f4ff7d0428fa462c242e7c55e Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Tue, 14 Jun 2022 17:23:25 +0200 Subject: [PATCH 4/7] Disable formatting block --- .../src/main/scala/com/spotify/scio/MagnoliaMacros.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala index 632cfb3d24..416b8cb22d 100644 --- a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala +++ b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala @@ -37,7 +37,7 @@ private[scio] object MagnoliaMacros { val magnoliaTree = Magnolia.gen[T](c) - // format:off + // format: off // Remove annotations from magnolia since they are // not serializable and we don't use them anyway val removeAnnotations = new Transformer { @@ -64,7 +64,7 @@ private[scio] object MagnoliaMacros { } } } - // format:on + // format: on removeAnnotations.transform(magnoliaTree) } From 89775091e5a229bf19e6924790937f577dc1b54b Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Wed, 6 Jul 2022 09:47:05 +0200 Subject: [PATCH 5/7] Improve macro transformation to remove annotations --- .../src/main/scala/com/spotify/scio/MagnoliaMacros.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala index 416b8cb22d..4bf2093794 100644 --- a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala +++ b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala @@ -47,17 +47,17 @@ private[scio] object MagnoliaMacros { super.transform( q"$caseClass($typeName, $isObject, $isValueClass, $parametersArray, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"Param.apply[$tc_, $t_, $p_]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, $_, $_, $_)" => + case q"Param.apply[$TC, $T, $P]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, $_, $_, $_)" => super.transform( - q"_root_.magnolia1.Param[$tc_, $t_, $p_]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, Array.empty[Any], Array.empty[Any], Array.empty[Any])" + q"_root_.magnolia1.Param[$TC, $T, $P]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) case q"new SealedTrait($typeName, $subtypesArray, $_, $_, $_)" => super.transform( q"new _root_.magnolia1.SealedTrait($typeName, $subtypesArray, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"Subtype[$tc_, $t_, $s_]($name, $idx, $_, $_, $_, $tc, $isType, $asType)" => + case q"Subtype[$TC, $T, $S]($name, $idx, $_, $_, $_, $tc, $isType, $asType)" => super.transform( - q"_root_.magnolia1.Subtype[$tc_, $t_,$s_]($name, $idx, Array.empty[Any], Array.empty[Any], Array.empty[Any], $tc, $isType, $asType)" + q"_root_.magnolia1.Subtype[$TC, $T, $S]($name, $idx, Array.empty[Any], Array.empty[Any], Array.empty[Any], $tc, $isType, $asType)" ) case t => super.transform(t) From efbdfff7b16c8db2d02f58d52199d269dc88b3ea Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Wed, 6 Jul 2022 10:13:17 +0200 Subject: [PATCH 6/7] Prefix type param matchers with tp --- .../src/main/scala/com/spotify/scio/MagnoliaMacros.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala index 4bf2093794..aa996495bc 100644 --- a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala +++ b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala @@ -47,17 +47,17 @@ private[scio] object MagnoliaMacros { super.transform( q"$caseClass($typeName, $isObject, $isValueClass, $parametersArray, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"Param.apply[$TC, $T, $P]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, $_, $_, $_)" => + case q"Param.apply[$tpTC, $tpT, $tpP]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, $_, $_, $_)" => super.transform( - q"_root_.magnolia1.Param[$TC, $T, $P]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, Array.empty[Any], Array.empty[Any], Array.empty[Any])" + q"_root_.magnolia1.Param[$tpTC, $tpT, $tpP]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) case q"new SealedTrait($typeName, $subtypesArray, $_, $_, $_)" => super.transform( q"new _root_.magnolia1.SealedTrait($typeName, $subtypesArray, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"Subtype[$TC, $T, $S]($name, $idx, $_, $_, $_, $tc, $isType, $asType)" => + case q"Subtype[$tpTC, $tpT, $tpS]($name, $idx, $_, $_, $_, $tc, $isType, $asType)" => super.transform( - q"_root_.magnolia1.Subtype[$TC, $T, $S]($name, $idx, Array.empty[Any], Array.empty[Any], Array.empty[Any], $tc, $isType, $asType)" + q"_root_.magnolia1.Subtype[$tpTC, $tpT, $tpS]($name, $idx, Array.empty[Any], Array.empty[Any], Array.empty[Any], $tc, $isType, $asType)" ) case t => super.transform(t) From 8f654eb0ff3077ebbe2b7d39de3a5fc60f9b7005 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Mon, 11 Jul 2022 10:19:14 +0200 Subject: [PATCH 7/7] Update to released version of magnolify --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 5f46e50619..040c2172ab 100644 --- a/build.sbt +++ b/build.sbt @@ -109,7 +109,7 @@ val kantanCodecsVersion = "0.5.1" val kantanCsvVersion = "0.6.2" val kryoVersion = "4.0.2" val magnoliaVersion = "1.1.2" -val magnolifyVersion = "0.4.8+10-0c6f9747-SNAPSHOT" +val magnolifyVersion = "0.5.0" val metricsVersion = "3.2.6" val parquetExtraVersion = "0.4.3" val parquetVersion = "1.12.3" @@ -219,7 +219,7 @@ val commonSettings = Def excludeDependencies ++= Seq( "org.apache.beam" % "beam-sdks-java-io-kafka" ), - resolvers ++= Seq(Resolver.sonatypeRepo("public"), Resolver.sonatypeRepo("snapshots")), + resolvers += Resolver.sonatypeRepo("public"), Test / javaOptions += "-Dscio.ignoreVersionWarning=true", Test / testOptions += Tests.Argument("-oD"), testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v", "-a"),