diff --git a/build.sbt b/build.sbt index 6a11cfeef6..040c2172ab 100644 --- a/build.sbt +++ b/build.sbt @@ -108,8 +108,8 @@ val junitVersion = "4.13.2" val kantanCodecsVersion = "0.5.1" val kantanCsvVersion = "0.6.2" val kryoVersion = "4.0.2" -val magnoliaVersion = "1.0.0-M4" -val magnolifyVersion = "0.4.8" +val magnoliaVersion = "1.1.2" +val magnolifyVersion = "0.5.0" val metricsVersion = "3.2.6" val parquetExtraVersion = "0.4.3" val parquetVersion = "1.12.3" @@ -512,7 +512,7 @@ lazy val `scio-core`: Project = project "org.slf4j" % "slf4j-api" % slf4jVersion, "org.typelevel" %% "algebra" % algebraVersion, "org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion, - "com.softwaremill.magnolia" %% "magnolia-core" % magnoliaVersion + "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaVersion ), buildInfoKeys := Seq[BuildInfoKey](scalaVersion, version, "beamVersion" -> beamVersion), buildInfoPackage := "com.spotify.scio" @@ -564,7 +564,7 @@ lazy val `scio-test`: Project = project "org.apache.beam" % "beam-sdks-java-core" % beamVersion, "org.hamcrest" % "hamcrest" % hamcrestVersion, "org.scalactic" %% "scalactic" % scalacticVersion, - "com.softwaremill.magnolia" %% "magnolia-core" % magnoliaVersion + "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaVersion ), Test / compileOrder := CompileOrder.JavaThenScala, Test / testGrouping := splitTests( @@ -592,7 +592,7 @@ lazy val `scio-macros`: Project = project "com.esotericsoftware" % "kryo-shaded" % kryoVersion, "org.apache.beam" % "beam-sdks-java-extensions-sql" % beamVersion, "org.apache.avro" % "avro" % avroVersion, - "com.softwaremill.magnolia" %% "magnolia-core" % magnoliaVersion + "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaVersion ) ) @@ -1019,7 +1019,7 @@ lazy val `scio-examples`: Project = project "org.apache.beam" % "beam-sdks-java-extensions-sql" % beamVersion, "org.apache.httpcomponents" % "httpcore" % httpCoreVersion, "org.elasticsearch" % "elasticsearch" % elasticsearch7Version, - "com.softwaremill.magnolia" %% "magnolia-core" % magnoliaVersion + "com.softwaremill.magnolia1_2" %% "magnolia" % magnoliaVersion ), // exclude problematic sources if we don't have GCP credentials unmanagedSources / excludeFilter := { diff --git a/scio-core/src/main/scala/com/spotify/scio/coders/DerivedCoder.scala b/scio-core/src/main/scala/com/spotify/scio/coders/DerivedCoder.scala index 588068f5da..6f19b6259a 100644 --- a/scio-core/src/main/scala/com/spotify/scio/coders/DerivedCoder.scala +++ b/scio-core/src/main/scala/com/spotify/scio/coders/DerivedCoder.scala @@ -18,7 +18,7 @@ package com.spotify.scio.coders private object Derived extends Serializable { - import magnolia._ + import magnolia1._ @inline private def catching[T](msg: => String)(v: => T): T = try { @@ -32,7 +32,7 @@ private object Derived extends Serializable { throw CoderStackTrace.append(e, msg) } - def combineCoder[T]( + def joinCoder[T]( typeName: TypeName, ps: Seq[Param[Coder, T]], rawConstruct: Seq[Any] => T @@ -72,23 +72,23 @@ private object Derived extends Serializable { } trait LowPriorityCoderDerivation { - import magnolia._ + import magnolia1._ type Typeclass[T] = Coder[T] - def combine[T](ctx: CaseClass[Coder, T]): Coder[T] = + def join[T](ctx: CaseClass[Coder, T]): Coder[T] = if (ctx.isValueClass) { - Coder.xmap(ctx.parameters(0).typeclass.asInstanceOf[Coder[Any]])( + Coder.xmap(ctx.parameters.head.typeclass.asInstanceOf[Coder[Any]])( a => ctx.rawConstruct(Seq(a)), - ctx.parameters(0).dereference + ctx.parameters.head.dereference ) } else { - Derived.combineCoder(ctx.typeName, ctx.parameters, ctx.rawConstruct) + Derived.joinCoder(ctx.typeName, ctx.parameters, ctx.rawConstruct) } - def dispatch[T](sealedTrait: SealedTrait[Coder, T]): Coder[T] = { + def split[T](sealedTrait: SealedTrait[Coder, T]): Coder[T] = { val typeName = sealedTrait.typeName.full - val idx: Map[magnolia.TypeName, Int] = + val idx: Map[TypeName, Int] = sealedTrait.subtypes.map(_.typeName).zipWithIndex.toMap val coders: Map[Int, Coder[T]] = sealedTrait.subtypes @@ -101,11 +101,11 @@ trait LowPriorityCoderDerivation { val booleanId: Int => Boolean = _ != 0 val cs = coders.map { case (key, v) => (booleanId(key), v) } Coder.disjunction[T, Boolean](typeName, cs) { t => - sealedTrait.dispatch(t)(subtype => booleanId(idx(subtype.typeName))) + sealedTrait.split(t)(subtype => booleanId(idx(subtype.typeName))) } } else { Coder.disjunction[T, Int](typeName, coders) { t => - sealedTrait.dispatch(t)(subtype => idx(subtype.typeName)) + sealedTrait.split(t)(subtype => idx(subtype.typeName)) } } } diff --git a/scio-core/src/main/scala/com/spotify/scio/schemas/instances/LowPrioritySchemaDerivation.scala b/scio-core/src/main/scala/com/spotify/scio/schemas/instances/LowPrioritySchemaDerivation.scala index c984ee735b..2cdaa7a643 100644 --- a/scio-core/src/main/scala/com/spotify/scio/schemas/instances/LowPrioritySchemaDerivation.scala +++ b/scio-core/src/main/scala/com/spotify/scio/schemas/instances/LowPrioritySchemaDerivation.scala @@ -20,8 +20,8 @@ package com.spotify.scio.schemas.instances import com.spotify.scio.schemas._ private object Derived extends Serializable { - import magnolia._ - def combineSchema[T](ps: Seq[Param[Schema, T]], rawConstruct: Seq[Any] => T): Record[T] = { + import magnolia1._ + def joinSchema[T](ps: Seq[Param[Schema, T]], rawConstruct: Seq[Any] => T): Record[T] = { @inline def destruct(v: T): Array[Any] = { val arr = new Array[Any](ps.length) var i = 0 @@ -39,11 +39,11 @@ private object Derived extends Serializable { } trait LowPrioritySchemaDerivation { - import magnolia._ + import magnolia1._ type Typeclass[T] = Schema[T] - def combine[T](ctx: CaseClass[Schema, T]): Record[T] = - Derived.combineSchema(ctx.parameters, ctx.rawConstruct) + def join[T](ctx: CaseClass[Schema, T]): Record[T] = + Derived.joinSchema(ctx.parameters, ctx.rawConstruct) import com.spotify.scio.MagnoliaMacros implicit def gen[T]: Schema[T] = macro MagnoliaMacros.genWithoutAnnotations[T] diff --git a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala index 64b89a2ce9..aa996495bc 100644 --- a/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala +++ b/scio-macros/src/main/scala/com/spotify/scio/MagnoliaMacros.scala @@ -20,6 +20,7 @@ package com.spotify.scio import scala.reflect.macros._ private[scio] object MagnoliaMacros { + import magnolia1._ // Add a level of indirection to prevent the macro from capturing // $outer which would make the Coder serialization fail @@ -34,35 +35,36 @@ private[scio] object MagnoliaMacros { ) } - val magnoliaTree = magnolia.Magnolia.gen[T](c) + val magnoliaTree = Magnolia.gen[T](c) + // format: off // Remove annotations from magnolia since they are // not serializable and we don't use them anyway - val removeAnnotations = new Transformer { - override def transform(tree: Tree): c.universe.Tree = + override def transform(tree: Tree): c.universe.Tree = { tree match { - case Apply(tt: TypeTree, List(typeName, isObject, isValueClass, params, _, _)) - if tt.symbol.name == TypeName("CaseClass") => + case q"$caseClass($typeName, $isObject, $isValueClass, $parametersArray, $_, $_, $_)" if caseClass.symbol.name == TypeName("CaseClass") => super.transform( - Apply(tt, List(typeName, isObject, isValueClass, params, q"Array()", q"Array()")) + q"$caseClass($typeName, $isObject, $isValueClass, $parametersArray, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"Param.apply[$tc, $t, $p]($name, $tpn, $idx, $repeated, $tcParam, $defaultVal, $_, $_)" => + case q"Param.apply[$tpTC, $tpT, $tpP]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, $_, $_, $_)" => super.transform( - q"_root_.magnolia.Param.apply[$tc, $t, $p]($name, $tpn, $idx, $repeated, $tcParam, $defaultVal, Array(), Array())" + q"_root_.magnolia1.Param[$tpTC, $tpT, $tpP]($name, $typeNameParam, $idx, $isRepeated, $typeclassParam, $defaultVal, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"new SealedTrait($typeName, $subtypes, $_, $_)" => + case q"new SealedTrait($typeName, $subtypesArray, $_, $_, $_)" => super.transform( - q"new _root_.magnolia.SealedTrait($typeName, $subtypes, Array(), Array())" + q"new _root_.magnolia1.SealedTrait($typeName, $subtypesArray, Array.empty[Any], Array.empty[Any], Array.empty[Any])" ) - case q"Subtype[$tc, $t, $p]($typeName, $id, $_, $_, $coder, $cast0, $cast1)" => + case q"Subtype[$tpTC, $tpT, $tpS]($name, $idx, $_, $_, $_, $tc, $isType, $asType)" => super.transform( - q"_root_.magnolia.Subtype[$tc, $t, $p]($typeName, $id, Array(), Array(), $coder, $cast0, $cast1)" + q"_root_.magnolia1.Subtype[$tpTC, $tpT, $tpS]($name, $idx, Array.empty[Any], Array.empty[Any], Array.empty[Any], $tc, $isType, $asType)" ) case t => super.transform(t) } + } } + // format: on removeAnnotations.transform(magnoliaTree) } diff --git a/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala b/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala index c053f2e98f..2e23d01b14 100644 --- a/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala +++ b/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala @@ -531,7 +531,7 @@ final class CoderTest extends AnyFlatSpec with Matchers { materializationStackTrace.map(_.getFileName) should contain("CoderTest.scala") } - it should "#1651: remove all anotations from derived coders" in { + it should "#1651: remove all annotations from derived coders" in { coderIsSerializable[TraitWithAnnotation] } diff --git a/site/src/main/paradox/Changelog.md b/site/src/main/paradox/Changelog.md index 839e59b622..d42f013d08 100644 --- a/site/src/main/paradox/Changelog.md +++ b/site/src/main/paradox/Changelog.md @@ -33,7 +33,7 @@ ## Breaking changes since Scio 0.7.0 (@ref:[v0.7.0 Migration Guide](migrations/v0.7.0-Migration-Guide.md)) -- New [Magnolia](https://github.com/propensive/magnolia) based @ref:[Coders](internals/Coders.md) derivation +- New [Magnolia](https://github.com/softwaremill/magnolia) based @ref:[Coders](internals/Coders.md) derivation - New @ref:[ScioIO](internals/ScioIO.md) replaces `TestIO[T]` to simplify IO implementation and stubbing in `JobTest` ## Breaking changes since Scio 0.6.0 diff --git a/site/src/main/paradox/index.md b/site/src/main/paradox/index.md index 448d2ee74d..a2a7ee5468 100644 --- a/site/src/main/paradox/index.md +++ b/site/src/main/paradox/index.md @@ -45,7 +45,7 @@ See @scaladoc[Scio Scaladocs](com.spotify.scio.index) for current API documenta - @ref:[ScioIO](internals/ScioIO.md) - new IO system to simplify implementation and stubbing in `JobTest` - @ref:[OverrideTypeProvider](internals/OverrideTypeProvider.md) - custom mappings for type-safe BigQuery - @ref:[Kryo](internals/Kryo.md) - [Kryo](https://github.com/EsotericSoftware/kryo) data serialization - - @ref:[Coders](internals/Coders.md) - new [Magnolia](https://github.com/propensive/magnolia) based Coders derivation + - @ref:[Coders](internals/Coders.md) - new [Magnolia](https://github.com/softwaremill/magnolia) based Coders derivation ## Further Readings - [Spotify Unwrapped: How We Brought You a Decade of Data](https://engineering.atspotify.com/2020/02/18/spotify-unwrapped-how-we-brought-you-a-decade-of-data/)