From 64120b7590a6c5207a761ac19fc97b68f429b3e2 Mon Sep 17 00:00:00 2001 From: Martin Duhem Date: Fri, 24 Apr 2020 17:21:16 +0200 Subject: [PATCH] Upgrade to sbt 1.3.10 --- build.sbt | 778 ++++--- project/build.properties | 2 +- project/plugins.sbt | 26 +- .../com/twitter/scalding/avro/package.scala | 6 +- .../commons/source/LzoGenericScheme.scala | 6 +- .../commons/source/LzoGenericSource.scala | 1 - .../source/VersionedKeyValSource.scala | 6 +- .../twitter/scalding/examples/KMeans.scala | 12 +- .../com/twitter/scalding/CascadingMode.scala | 3 +- .../scala/com/twitter/scalding/Config.scala | 12 +- .../com/twitter/scalding/CumulativeSum.scala | 6 +- .../com/twitter/scalding/Execution.scala | 82 +- .../scalding/ExecutionOptimizationRules.scala | 45 +- .../com/twitter/scalding/FileSource.scala | 33 +- .../com/twitter/scalding/FlowState.scala | 3 +- .../scalding/GeneratedConversions.scala | 1863 +++++++++-------- .../com/twitter/scalding/GroupBuilder.scala | 15 +- .../twitter/scalding/IntegralComparator.scala | 3 +- .../main/scala/com/twitter/scalding/Job.scala | 26 +- .../scala/com/twitter/scalding/JobTest.scala | 4 +- .../com/twitter/scalding/JoinAlgorithms.scala | 3 +- .../com/twitter/scalding/Operations.scala | 3 +- .../com/twitter/scalding/PipeDebug.scala | 3 +- .../twitter/scalding/ReduceOperations.scala | 12 +- .../scala/com/twitter/scalding/RichPipe.scala | 7 +- .../com/twitter/scalding/TestTapFactory.scala | 3 +- .../twitter/scalding/TimePathedSource.scala | 3 +- .../com/twitter/scalding/TypedDelimited.scala | 5 +- .../impl/CaseClassBasedSetterImpl.scala | 6 +- .../macros/impl/TupleConverterImpl.scala | 3 +- .../impl/TypeDescriptorProviderImpl.scala | 3 +- .../twitter/scalding/mathematics/Matrix.scala | 30 +- .../scalding/mathematics/Matrix2.scala | 27 +- .../mathematics/TypedSimilarity.scala | 21 +- .../scalding/serialization/KryoHadoop.scala | 15 +- .../twitter/scalding/source/TypedText.scala | 26 +- .../com/twitter/scalding/typed/Grouped.scala | 26 +- .../twitter/scalding/typed/KeyedList.scala | 4 +- .../twitter/scalding/typed/LookupJoin.scala | 9 +- .../scalding/typed/NoStackAndThen.scala | 3 +- .../scalding/typed/OptimizationRules.scala | 338 ++- .../typed/PartitionedDelimitedSource.scala | 2 +- .../scalding/typed/PartitionedTextLine.scala | 2 +- .../com/twitter/scalding/typed/Sketched.scala | 11 +- .../twitter/scalding/typed/TypedPipe.scala | 182 +- .../scalding/typed/TypedPipeDiff.scala | 3 +- .../scalding/typed/WritePartitioner.scala | 51 +- .../AsyncFlowDefRunner.scala | 14 +- .../cascading_backend/CascadingBackend.scala | 111 +- .../cascading_backend/CoGroupJoiner.scala | 3 +- .../DistinctCoGroupJoiner.scala | 3 +- .../typed/functions/FlatMappedFn.scala | 2 +- .../scalding/typed/functions/SubTypes.scala | 7 +- .../typed/memory_backend/MemoryBackend.scala | 2 +- .../scalding/typed/memory_backend/Op.scala | 33 +- .../twitter/scalding/AbsoluteDuration.scala | 6 +- .../com/twitter/scalding/DateRange.scala | 3 +- .../scalding/db/ColumnDefinition.scala | 3 +- .../scalding/db/DBColumnTransformer.scala | 15 +- .../impl/ColumnDefinitionProviderImpl.scala | 6 +- .../macros/impl/JdbcStatementSetterImpl.scala | 3 +- .../db/macros/impl/handler/ColumnFormat.scala | 3 +- .../macros/impl/handler/DateTypeHandler.scala | 3 +- .../impl/handler/NumericTypeHandler.scala | 3 +- .../impl/handler/StringTypeHandler.scala | 3 +- .../estimation/HRavenHistoryService.scala | 3 +- .../scrooge/Parquet346ScroogeScheme.scala | 6 +- .../PartitionedParquetScroogeSource.scala | 3 +- .../parquet/HasColumnProjection.scala | 3 +- .../thrift/Parquet346TBaseScheme.scala | 6 +- .../PartitionedParquetThriftSource.scala | 3 +- .../impl/ParquetReadSupportProvider.scala | 3 +- .../macros/impl/WriteSupportProvider.scala | 3 +- .../scheme/TypedParquetTupleScheme.scala | 3 +- .../scalding/quotation/TextMacro.scala | 2 +- .../com/twitter/scalding/ReplImplicits.scala | 15 +- .../com/twitter/scalding/ScaldingShell.scala | 3 +- .../scalding/serialization/Boxed.scala | 3 +- .../serialization/OrderedSerialization.scala | 18 +- .../serialization/Serialization.scala | 9 +- .../serialization/Serialization2.scala | 3 +- .../StringOrderedSerialization.scala | 9 +- .../scalding/serialization/Writer.scala | 3 +- .../ordered_serialization/ProductLike.scala | 6 +- .../TreeOrderedBuf.scala | 6 +- .../providers/CaseClassOrderedBuf.scala | 3 +- .../providers/EitherOrderedBuf.scala | 3 +- .../providers/OptionOrderedBuf.scala | 3 +- .../providers/ProductOrderedBuf.scala | 145 +- .../providers/SealedTraitOrderedBuf.scala | 6 +- .../providers/TraversablesOrderedBuf.scala | 3 +- .../runtime_helpers/TraversableHelpers.scala | 12 +- .../twitter/scalding/spark_backend/Op.scala | 10 +- .../scalding/spark_backend/SparkMode.scala | 7 +- .../scalding/spark_backend/SparkWriter.scala | 9 +- 95 files changed, 2236 insertions(+), 2031 deletions(-) diff --git a/build.sbt b/build.sbt index d42b61c24c..eae0ec7e75 100644 --- a/build.sbt +++ b/build.sbt @@ -1,12 +1,6 @@ -import AssemblyKeys._ import ReleaseTransformations._ -import com.typesafe.sbt.SbtGhPages.GhPagesKeys._ import com.typesafe.sbt.SbtScalariform._ -import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings -import sbtassembly.Plugin._ -import sbtunidoc.Plugin.UnidocKeys._ import scala.collection.JavaConverters._ -import scalariform.formatter.preferences._ def scalaBinaryVersion(scalaVersion: String) = scalaVersion match { case version if version startsWith "2.11" => "2.11" @@ -44,7 +38,11 @@ val jlineVersion = "2.14.3" val printDependencyClasspath = taskKey[Unit]("Prints location of the dependencies") -val sharedSettings = assemblySettings ++ scalariformSettings ++ Seq( +val mimaSettings = Seq( + mimaPreviousArtifacts := youngestForwardCompatible(name.value) +) + +val sharedSettings = Seq( organization := "com.twitter", scalaVersion := "2.11.12", @@ -153,7 +151,7 @@ val sharedSettings = assemblySettings ++ scalariformSettings ++ Seq( ), // Janino includes a broken signature, and is not needed: - excludedJars in assembly := { + assemblyExcludedJars in assembly := { val excludes = Set("jsp-api-2.1-6.1.14.jar", "jsp-2.1-6.1.14.jar", "jasper-compiler-5.5.12.jar", "janino-2.5.16.jar") (fullClasspath in assembly).value filter { @@ -161,7 +159,7 @@ val sharedSettings = assemblySettings ++ scalariformSettings ++ Seq( } }, // Some of these files have duplicates, let's ignore: - mergeStrategy in assembly := { + assemblyMergeStrategy in assembly := { case s if s.endsWith(".class") => MergeStrategy.last case s if s.endsWith("project.clj") => MergeStrategy.concat case s if s.endsWith(".html") => MergeStrategy.last @@ -173,7 +171,7 @@ val sharedSettings = assemblySettings ++ scalariformSettings ++ Seq( case s if s.endsWith("jansi.dll") => MergeStrategy.rename case s if s.endsWith("libjansi.so") => MergeStrategy.rename case s if s.endsWith("properties") => MergeStrategy.filterDistinctLines - case x => (mergeStrategy in assembly).value(x) + case x => (assemblyMergeStrategy in assembly).value(x) }, pomExtra := ( @@ -207,58 +205,60 @@ val sharedSettings = assemblySettings ++ scalariformSettings ++ Seq( https://twitter.com/argyris ) -) ++ mimaDefaultSettings - -lazy val scalding = Project( - id = "scalding", - base = file("."), - settings = sharedSettings) - .settings(noPublishSettings) - .aggregate( - scaldingArgs, - scaldingDate, - scaldingQuotation, - scaldingCats, - scaldingCore, - scaldingCommons, - scaldingAvro, - scaldingParquet, - scaldingParquetScrooge, - scaldingHRaven, - scaldingRepl, - scaldingJson, - scaldingJdbc, - scaldingHadoopTest, - scaldingEstimatorsTest, - scaldingDb, - maple, - executionTutorial, - scaldingSerialization, - scaldingSpark, - scaldingThriftMacros ) -lazy val scaldingAssembly = Project( - id = "scalding-assembly", - base = file("assembly"), - settings = sharedSettings) - .settings(noPublishSettings) - .aggregate( - scaldingArgs, - scaldingDate, - scaldingQuotation, - scaldingCore, - scaldingCommons, - scaldingAvro, - scaldingParquet, - scaldingParquetScrooge, - scaldingHRaven, - scaldingRepl, - scaldingJson, - scaldingJdbc, - maple, - scaldingSerialization -) +lazy val scalding = project + .in(file(".")) + .settings( + sharedSettings, + noPublishSettings + ) + .aggregate( + scaldingArgs, + scaldingDate, + scaldingQuotation, + scaldingCats, + scaldingCore, + scaldingCommons, + scaldingAvro, + scaldingParquet, + scaldingParquetScrooge, + scaldingHRaven, + scaldingRepl, + scaldingJson, + scaldingJdbc, + scaldingHadoopTest, + scaldingEstimatorsTest, + scaldingDb, + maple, + executionTutorial, + scaldingSerialization, + scaldingSpark, + scaldingThriftMacros + ) + +lazy val scaldingAssembly = project + .in(file("assembly")) + .settings( + sharedSettings, + noPublishSettings + ) + .aggregate( + scaldingArgs, + scaldingDate, + scaldingQuotation, + scaldingCore, + scaldingCommons, + scaldingAvro, + scaldingParquet, + scaldingParquetScrooge, + scaldingHRaven, + scaldingRepl, + scaldingJson, + scaldingJdbc, + maple, + scaldingSerialization + ) lazy val formattingPreferences = { import scalariform.formatter.preferences._ @@ -268,10 +268,7 @@ lazy val formattingPreferences = { } lazy val noPublishSettings = Seq( - publish := (), - publishLocal := (), - test := (), - publishArtifact := false + skip in publish := true ) /** @@ -280,8 +277,8 @@ lazy val noPublishSettings = Seq( */ val ignoredModules = Set[String]("benchmarks") -def youngestForwardCompatible(subProj: String) = - None +def youngestForwardCompatible(subProj: String): Set[ModuleID] = + Set.empty // Enable mima binary check back after releasing 0.18.0 // Some(subProj) // .filterNot(ignoredModules.contains(_)) @@ -289,17 +286,22 @@ def youngestForwardCompatible(subProj: String) = // s => "com.twitter" %% (s"scalding-$s") % "0.17.0" // } -def module(name: String) = { - val id = "scalding-%s".format(name) - Project(id = id, base = file(id), settings = sharedSettings ++ Seq( - Keys.name := id, - mimaPreviousArtifacts := youngestForwardCompatible(name).toSet) - ) -} -lazy val scaldingArgs = module("args") +lazy val scaldingArgs = project + .in(file("scalding-args")) + .settings( + name := "scalding-args", + sharedSettings, + mimaSettings, + ) -lazy val scaldingDate = module("date") +lazy val scaldingDate = project + .in(file("scalding-date")) + .settings( + name := "scalding-date", + sharedSettings, + mimaSettings, + ) lazy val cascadingVersion = System.getenv.asScala.getOrElse("SCALDING_CASCADING_VERSION", "2.6.1") @@ -307,8 +309,12 @@ lazy val cascadingVersion = lazy val cascadingJDBCVersion = System.getenv.asScala.getOrElse("SCALDING_CASCADING_JDBC_VERSION", "2.6.0") -lazy val scaldingBenchmarks = module("benchmarks") +lazy val scaldingBenchmarks = project + .in(file("scalding-benchmarks")) .settings( + name := "scalding-benchmarks", + sharedSettings, + mimaSettings, libraryDependencies ++= Seq( "com.storm-enroute" %% "scalameter" % scalameterVersion % "test", "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test" @@ -317,121 +323,171 @@ lazy val scaldingBenchmarks = module("benchmarks") parallelExecution in Test := false ).dependsOn(scaldingCore) -lazy val scaldingQuotation = module("quotation").settings( - libraryDependencies ++= Seq( - "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided", - "org.scala-lang" % "scala-compiler" % scalaVersion.value % "provided" +lazy val scaldingQuotation = project + .in(file("scalding-quotation")) + .settings( + name := "scalding-quotation", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided", + "org.scala-lang" % "scala-compiler" % scalaVersion.value % "provided" + ) ) -) -lazy val scaldingCore = module("core").settings( - libraryDependencies ++= Seq( - "cascading" % "cascading-core" % cascadingVersion, - "cascading" % "cascading-hadoop" % cascadingVersion, - "cascading" % "cascading-local" % cascadingVersion, - "com.stripe" %% "dagon-core" % dagonVersion, - "com.twitter" % "chill-hadoop" % chillVersion, - "com.twitter" % "chill-java" % chillVersion, - "com.twitter" %% "chill-bijection" % chillVersion, - "com.twitter" %% "algebird-core" % algebirdVersion, - "com.twitter" %% "algebird-test" % algebirdVersion % "test", - "com.twitter" %% "bijection-core" % bijectionVersion, - "com.twitter" %% "bijection-macros" % bijectionVersion, - "com.twitter" %% "chill" % chillVersion, - "com.twitter" %% "chill-algebird" % chillVersion, - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", - "org.scala-lang" % "scala-library" % scalaVersion.value, - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "provided"), - addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full) -).dependsOn(scaldingArgs, scaldingDate, scaldingSerialization, maple, scaldingQuotation) - -lazy val scaldingCats = module("cats").settings( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", - "org.typelevel" %% "cats-core" % catsVersion, - "org.typelevel" %% "cats-laws" % catsVersion % "test", - "org.typelevel" %% "cats-effect" % catsEffectVersion, - "org.typelevel" %% "cats-effect-laws" % catsEffectVersion % "test" - )).dependsOn(scaldingArgs, scaldingDate, scaldingCore) +lazy val scaldingCore = project + .in(file("scalding-core")) + .dependsOn(scaldingArgs, scaldingDate, scaldingSerialization, maple, scaldingQuotation) + .settings( + name := "scalding-core", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "cascading" % "cascading-core" % cascadingVersion, + "cascading" % "cascading-hadoop" % cascadingVersion, + "cascading" % "cascading-local" % cascadingVersion, + "com.stripe" %% "dagon-core" % dagonVersion, + "com.twitter" % "chill-hadoop" % chillVersion, + "com.twitter" % "chill-java" % chillVersion, + "com.twitter" %% "chill-bijection" % chillVersion, + "com.twitter" %% "algebird-core" % algebirdVersion, + "com.twitter" %% "algebird-test" % algebirdVersion % "test", + "com.twitter" %% "bijection-core" % bijectionVersion, + "com.twitter" %% "bijection-macros" % bijectionVersion, + "com.twitter" %% "chill" % chillVersion, + "com.twitter" %% "chill-algebird" % chillVersion, + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", + "org.scala-lang" % "scala-library" % scalaVersion.value, + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "provided"), + addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full) + ) +lazy val scaldingCats = project + .in(file("scalding-cats")) + .dependsOn(scaldingArgs, scaldingDate, scaldingCore) + .settings( + name := "scalding-cats", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", + "org.typelevel" %% "cats-core" % catsVersion, + "org.typelevel" %% "cats-laws" % catsVersion % "test", + "org.typelevel" %% "cats-effect" % catsEffectVersion, + "org.typelevel" %% "cats-effect-laws" % catsEffectVersion % "test" + ) + ) -lazy val scaldingSpark = module("spark").settings( - libraryDependencies ++= Seq( - "org.apache.spark" %% "spark-core" % sparkVersion, - "org.apache.spark" %% "spark-sql" % sparkVersion + +lazy val scaldingSpark = project + .in(file("scalding-spark")) + .dependsOn(scaldingCore) + .settings( + name := "scalding-spark", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-core" % sparkVersion, + "org.apache.spark" %% "spark-sql" % sparkVersion ) - ).dependsOn(scaldingCore) + ) -lazy val scaldingCommons = module("commons").settings( - libraryDependencies ++= Seq( - // TODO: split into scalding-protobuf - "com.google.protobuf" % "protobuf-java" % protobufVersion, - "com.twitter" %% "bijection-core" % bijectionVersion, - "com.twitter" %% "algebird-core" % algebirdVersion, - "com.twitter" %% "chill" % chillVersion, - "com.twitter.elephantbird" % "elephant-bird-cascading2" % elephantbirdVersion, - "com.twitter.elephantbird" % "elephant-bird-core" % elephantbirdVersion, - "com.hadoop.gplcompression" % "hadoop-lzo" % hadoopLzoVersion, - // TODO: split this out into scalding-thrift - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", - "org.apache.thrift" % "libthrift" % thriftVersion, - // TODO: split this out into a scalding-scrooge - "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided" - exclude("com.google.guava", "guava"), - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "provided", - "junit" % "junit" % junitVersion % "test" +lazy val scaldingCommons = project + .in(file("scalding-commons")) + .dependsOn(scaldingArgs, scaldingDate, scaldingCore, scaldingHadoopTest % "test") + .settings( + name := "scalding-commons", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + // TODO: split into scalding-protobuf + "com.google.protobuf" % "protobuf-java" % protobufVersion, + "com.twitter" %% "bijection-core" % bijectionVersion, + "com.twitter" %% "algebird-core" % algebirdVersion, + "com.twitter" %% "chill" % chillVersion, + "com.twitter.elephantbird" % "elephant-bird-cascading2" % elephantbirdVersion, + "com.twitter.elephantbird" % "elephant-bird-core" % elephantbirdVersion, + "com.hadoop.gplcompression" % "hadoop-lzo" % hadoopLzoVersion, + // TODO: split this out into scalding-thrift + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", + "org.apache.thrift" % "libthrift" % thriftVersion, + // TODO: split this out into a scalding-scrooge + "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided" + exclude("com.google.guava", "guava"), + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "provided", + "junit" % "junit" % junitVersion % "test" + ) ) -).dependsOn(scaldingArgs, scaldingDate, scaldingCore, scaldingHadoopTest % "test") -lazy val scaldingAvro = module("avro").settings( - libraryDependencies ++= Seq( - "cascading.avro" % "avro-scheme" % cascadingAvroVersion, - "org.apache.avro" % "avro" % avroVersion, - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided" +lazy val scaldingAvro = project + .in(file("scalding-avro")) + .dependsOn(scaldingCore) + .settings( + name := "scalding-avro", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "cascading.avro" % "avro-scheme" % cascadingAvroVersion, + "org.apache.avro" % "avro" % avroVersion, + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided" + ) ) -).dependsOn(scaldingCore) - -lazy val scaldingParquetFixtures = module("parquet-fixtures") - .settings( - scroogeThriftSourceFolder in Test := baseDirectory.value / "src/test/resources", - scroogeLanguages in Test := Seq("java", "scala"), - libraryDependencies ++= Seq( - "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided" - exclude("com.google.guava", "guava"), - "commons-lang" % "commons-lang" % apacheCommonsVersion, // needed for HashCodeBuilder used in thriftjava - "org.apache.thrift" % "libthrift" % thriftVersion - ) - ) - -lazy val scaldingParquet = module("parquet").settings( - libraryDependencies ++= Seq( - "org.apache.parquet" % "parquet-column" % parquetVersion, - "org.apache.parquet" % "parquet-hadoop" % parquetVersion, - "org.apache.parquet" % "parquet-thrift" % parquetVersion - // see https://issues.apache.org/jira/browse/PARQUET-143 for exclusions - exclude("org.apache.parquet", "parquet-pig") - exclude("com.twitter.elephantbird", "elephant-bird-pig") - exclude("com.twitter.elephantbird", "elephant-bird-core"), - "org.scala-lang" % "scala-compiler" % scalaVersion.value, - "org.apache.thrift" % "libthrift" % thriftVersion, - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "com.twitter" %% "bijection-macros" % bijectionVersion, - "com.twitter" %% "chill-bijection" % chillVersion, - "com.twitter.elephantbird" % "elephant-bird-core" % elephantbirdVersion % "test" - ), - addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full)) + +lazy val scaldingParquetFixtures = project + .in(file("scalding-parquet-fixtures")) + .settings( + name := "scalding-parquet-fixtures", + sharedSettings, + mimaSettings, + scroogeThriftSourceFolder in Test := baseDirectory.value / "src/test/resources", + scroogeLanguages in Test := Seq("java", "scala"), + libraryDependencies ++= Seq( + "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided" + exclude("com.google.guava", "guava"), + "commons-lang" % "commons-lang" % apacheCommonsVersion, // needed for HashCodeBuilder used in thriftjava + "org.apache.thrift" % "libthrift" % thriftVersion + ) + ) + +lazy val scaldingParquet = project + .in(file("scalding-parquet")) .dependsOn(scaldingCore, scaldingHadoopTest % "test", scaldingParquetFixtures % "test->test") + .settings( + name := "scalding-parquet", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.apache.parquet" % "parquet-column" % parquetVersion, + "org.apache.parquet" % "parquet-hadoop" % parquetVersion, + "org.apache.parquet" % "parquet-thrift" % parquetVersion + // see https://issues.apache.org/jira/browse/PARQUET-143 for exclusions + exclude("org.apache.parquet", "parquet-pig") + exclude("com.twitter.elephantbird", "elephant-bird-pig") + exclude("com.twitter.elephantbird", "elephant-bird-core"), + "org.scala-lang" % "scala-compiler" % scalaVersion.value, + "org.apache.thrift" % "libthrift" % thriftVersion, + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "com.twitter" %% "bijection-macros" % bijectionVersion, + "com.twitter" %% "chill-bijection" % chillVersion, + "com.twitter.elephantbird" % "elephant-bird-core" % elephantbirdVersion % "test" + ), + addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full) + ) -lazy val scaldingParquetScroogeFixtures = module("parquet-scrooge-fixtures") +lazy val scaldingParquetScroogeFixtures = project + .in(file("scalding-parquet-scrooge-fixtures")) .settings( + name := "scalding-parquet-scrooge-fixtures", + sharedSettings, + mimaSettings, scroogeThriftSourceFolder in Test := baseDirectory.value / "src/test/resources", scroogeLanguages in Test := Seq("java", "scala"), libraryDependencies ++= Seq( @@ -442,8 +498,13 @@ lazy val scaldingParquetScroogeFixtures = module("parquet-scrooge-fixtures") ) ) -lazy val scaldingParquetScrooge = module("parquet-scrooge") +lazy val scaldingParquetScrooge = project + .in(file("scalding-parquet-scrooge")) + .dependsOn(scaldingCore, scaldingParquet % "compile->compile;test->test", scaldingParquetScroogeFixtures % "test->test") .settings( + name := "scalding-parquet-scrooge", + sharedSettings, + mimaSettings, libraryDependencies ++= Seq( "org.slf4j" % "slf4j-api" % slf4jVersion, // see https://issues.apache.org/jira/browse/PARQUET-143 for exclusions @@ -456,35 +517,40 @@ lazy val scaldingParquetScrooge = module("parquet-scrooge") "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", "com.novocode" % "junit-interface" % "0.11" % "test", "junit" % "junit" % junitVersion % "test" - ) -).dependsOn(scaldingCore, scaldingParquet % "compile->compile;test->test", scaldingParquetScroogeFixtures % "test->test") + ) -lazy val scaldingHRaven = module("hraven").settings( - libraryDependencies ++= Seq( - "com.twitter.hraven" % "hraven-core" % hravenVersion - // These transitive dependencies cause sbt to give a ResolveException - // because they're not available on Maven. We don't need them anyway. - // See https://github.com/twitter/cassie/issues/13 - exclude("javax.jms", "jms") - exclude("com.sun.jdmk", "jmxtools") - exclude("com.sun.jmx", "jmxri") - - // These transitive dependencies of hRaven cause conflicts when - // running scalding-hraven/*assembly and aren't needed - // for the part of the hRaven API that we use anyway - exclude("com.twitter.common", "application-module-log") - exclude("com.twitter.common", "application-module-stats") - exclude("com.twitter.common", "args") - exclude("com.twitter.common", "application"), - "org.apache.hbase" % "hbase" % hbaseVersion, - "org.apache.hbase" % "hbase-client" % hbaseVersion % "provided", - "org.apache.hbase" % "hbase-common" % hbaseVersion % "provided", - "org.apache.hbase" % "hbase-server" % hbaseVersion % "provided", - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided" +lazy val scaldingHRaven = project + .in(file("scalding-hraven")) + .dependsOn(scaldingCore) + .settings( + name := "scalding-hraven", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "com.twitter.hraven" % "hraven-core" % hravenVersion + // These transitive dependencies cause sbt to give a ResolveException + // because they're not available on Maven. We don't need them anyway. + // See https://github.com/twitter/cassie/issues/13 + exclude("javax.jms", "jms") + exclude("com.sun.jdmk", "jmxtools") + exclude("com.sun.jmx", "jmxri") + + // These transitive dependencies of hRaven cause conflicts when + // running scalding-hraven/*assembly and aren't needed + // for the part of the hRaven API that we use anyway + exclude("com.twitter.common", "application-module-log") + exclude("com.twitter.common", "application-module-stats") + exclude("com.twitter.common", "args") + exclude("com.twitter.common", "application"), + "org.apache.hbase" % "hbase" % hbaseVersion, + "org.apache.hbase" % "hbase-client" % hbaseVersion % "provided", + "org.apache.hbase" % "hbase-common" % hbaseVersion % "provided", + "org.apache.hbase" % "hbase-server" % hbaseVersion % "provided", + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided" + ) ) -).dependsOn(scaldingCore) // create new configuration which will hold libs otherwise marked as 'provided' // so that we can re-include them in 'run'. unfortunately, we still have to @@ -492,9 +558,14 @@ lazy val scaldingHRaven = module("hraven").settings( // solution borrowed from: http://stackoverflow.com/a/18839656/1404395 val Unprovided = config("unprovided") extend Runtime -lazy val scaldingRepl = module("repl") +lazy val scaldingRepl = project + .in(file("scalding-repl")) + .dependsOn(scaldingCore) .configs(Unprovided) // include 'unprovided' as config option .settings( + name := "scalding-repl", + sharedSettings, + mimaSettings, initialCommands in console := """ import com.twitter.scalding._ import com.twitter.scalding.ReplImplicits._ @@ -509,124 +580,160 @@ lazy val scaldingRepl = module("repl") "org.slf4j" % "slf4j-api" % slf4jVersion, "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "provided", "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "unprovided" - ) -).dependsOn(scaldingCore) -// run with 'unprovided' config includes libs marked 'unprovided' in classpath -.settings(inConfig(Unprovided)(Classpaths.configSettings ++ Seq( - run := Defaults.runTask(fullClasspath, mainClass in (Runtime, run), runner in (Runtime, run)) -)): _*) -.settings( - // make scalding-repl/run use 'unprovided' config - run := (run in Unprovided) -) + ), + // run with 'unprovided' config includes libs marked 'unprovided' in classpath + inConfig(Unprovided)(Classpaths.configSettings ++ Seq( + run := Defaults.runTask(fullClasspath, mainClass in (Runtime, run), runner in (Runtime, run)) + )), + // make scalding-repl/run use 'unprovided' config + run := run in Unprovided + ) // zero dependency serialization module -lazy val scaldingSerialization = module("serialization").settings( - libraryDependencies ++= Seq( - "org.scala-lang" % "scala-reflect" % scalaVersion.value - ), -addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full) -) +lazy val scaldingSerialization = project + .in(file("scalding-serialization")) + .settings( + name := "scalding-serialization", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.scala-lang" % "scala-reflect" % scalaVersion.value + ), + addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full) + ) -lazy val scaldingJson = module("json").settings( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", - "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion, - "org.json4s" %% "json4s-native" % json4SVersion, - "com.twitter.elephantbird" % "elephant-bird-cascading2" % elephantbirdVersion % "provided" +lazy val scaldingJson = project + .in(file("scalding-json")) + .dependsOn(scaldingCore) + .settings( + name := "scalding-json", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", + "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion, + "org.json4s" %% "json4s-native" % json4SVersion, + "com.twitter.elephantbird" % "elephant-bird-cascading2" % elephantbirdVersion % "provided" ) -).dependsOn(scaldingCore) + ) -lazy val scaldingJdbc = module("jdbc").settings( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", - "cascading" % "cascading-jdbc-core" % cascadingJDBCVersion, - "cascading" % "cascading-jdbc-mysql" % cascadingJDBCVersion +lazy val scaldingJdbc = project + .in(file("scalding-jdbc")) + .dependsOn(scaldingCore) + .settings( + name := "scalding-jdbc", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", + "cascading" % "cascading-jdbc-core" % cascadingJDBCVersion, + "cascading" % "cascading-jdbc-mysql" % cascadingJDBCVersion + ) ) -).dependsOn(scaldingCore) -lazy val scaldingHadoopTest = module("hadoop-test").settings( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-client" % hadoopVersion, - "org.apache.hadoop" % "hadoop-minicluster" % hadoopVersion, - "org.apache.hadoop" % "hadoop-yarn-server-tests" % hadoopVersion classifier "tests", - "org.apache.hadoop" % "hadoop-yarn-server" % hadoopVersion, - "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion classifier "tests", - "org.apache.hadoop" % "hadoop-common" % hadoopVersion classifier "tests", - "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests", - "com.twitter" %% "chill-algebird" % chillVersion, - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "slf4j-log4j12" % slf4jVersion, - "org.scalacheck" %% "scalacheck" % scalaCheckVersion, - "org.scalatest" %% "scalatest" % scalaTestVersion +lazy val scaldingHadoopTest = project + .in(file("scalding-hadoop-test")) + .dependsOn(scaldingCore, scaldingSerialization) + .settings( + name := "scalding-hadoop-test", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-client" % hadoopVersion, + "org.apache.hadoop" % "hadoop-minicluster" % hadoopVersion, + "org.apache.hadoop" % "hadoop-yarn-server-tests" % hadoopVersion classifier "tests", + "org.apache.hadoop" % "hadoop-yarn-server" % hadoopVersion, + "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion classifier "tests", + "org.apache.hadoop" % "hadoop-common" % hadoopVersion classifier "tests", + "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests", + "com.twitter" %% "chill-algebird" % chillVersion, + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-log4j12" % slf4jVersion, + "org.scalacheck" %% "scalacheck" % scalaCheckVersion, + "org.scalatest" %% "scalatest" % scalaTestVersion + ) ) -).dependsOn(scaldingCore, scaldingSerialization) -lazy val scaldingEstimatorsTest = module("estimators-test").settings( - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-client" % hadoopVersion, - "org.apache.hadoop" % "hadoop-minicluster" % hadoopVersion, - "org.apache.hadoop" % "hadoop-yarn-server-tests" % hadoopVersion classifier "tests", - "org.apache.hadoop" % "hadoop-yarn-server" % hadoopVersion, - "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion classifier "tests", - "org.apache.hadoop" % "hadoop-common" % hadoopVersion classifier "tests", - "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests", - "com.twitter" %% "chill-algebird" % chillVersion, - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "slf4j-log4j12" % slf4jVersion, - "org.scalacheck" %% "scalacheck" % scalaCheckVersion, - "org.scalatest" %% "scalatest" % scalaTestVersion +lazy val scaldingEstimatorsTest = project + .in(file("scalding-estimators-test")) + .dependsOn(scaldingHadoopTest % "test") + .settings( + name := "scalding-estimators-test", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-client" % hadoopVersion, + "org.apache.hadoop" % "hadoop-minicluster" % hadoopVersion, + "org.apache.hadoop" % "hadoop-yarn-server-tests" % hadoopVersion classifier "tests", + "org.apache.hadoop" % "hadoop-yarn-server" % hadoopVersion, + "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion classifier "tests", + "org.apache.hadoop" % "hadoop-common" % hadoopVersion classifier "tests", + "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests", + "com.twitter" %% "chill-algebird" % chillVersion, + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-log4j12" % slf4jVersion, + "org.scalacheck" %% "scalacheck" % scalaCheckVersion, + "org.scalatest" %% "scalatest" % scalaTestVersion + ) ) -).dependsOn(scaldingHadoopTest % "test") // This one uses a different naming convention -lazy val maple = Project( - id = "maple", - base = file("maple"), - settings = sharedSettings -).settings( - name := "maple", - mimaPreviousArtifacts := Set.empty, - crossPaths := false, - autoScalaLibrary := false, - publishArtifact := true, - libraryDependencies ++= Seq( - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", - "org.apache.hbase" % "hbase" % hbaseVersion % "provided", - "org.apache.hbase" % "hbase-client" % hbaseVersion % "provided", - "org.apache.hbase" % "hbase-common" % hbaseVersion % "provided", - "org.apache.hbase" % "hbase-server" % hbaseVersion % "provided", - "cascading" % "cascading-hadoop" % cascadingVersion +lazy val maple = project + .in(file("maple")) + .settings( + name := "maple", + sharedSettings, + mimaPreviousArtifacts := Set.empty, + crossPaths := false, + autoScalaLibrary := false, + publishArtifact := true, + libraryDependencies ++= Seq( + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", + "org.apache.hbase" % "hbase" % hbaseVersion % "provided", + "org.apache.hbase" % "hbase-client" % hbaseVersion % "provided", + "org.apache.hbase" % "hbase-common" % hbaseVersion % "provided", + "org.apache.hbase" % "hbase-server" % hbaseVersion % "provided", + "cascading" % "cascading-hadoop" % cascadingVersion + ) ) -) -lazy val executionTutorial = Project( - id = "execution-tutorial", - base = file("tutorial/execution-tutorial"), - settings = sharedSettings -).settings( - name := "execution-tutorial", - libraryDependencies ++= Seq( - "org.scala-lang" % "scala-library" % scalaVersion.value, - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.apache.hadoop" % "hadoop-client" % hadoopVersion, - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "slf4j-log4j12" % slf4jVersion, - "cascading" % "cascading-hadoop" % cascadingVersion +lazy val executionTutorial = project + .in(file("tutorial") / "execution-tutorial") + .dependsOn(scaldingCore) + .settings( + name := "execution-tutorials", + sharedSettings, + libraryDependencies ++= Seq( + "org.scala-lang" % "scala-library" % scalaVersion.value, + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "org.apache.hadoop" % "hadoop-client" % hadoopVersion, + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-log4j12" % slf4jVersion, + "cascading" % "cascading-hadoop" % cascadingVersion + ) ) -).dependsOn(scaldingCore) -lazy val scaldingDb = module("db").settings( - libraryDependencies ++= Seq( - "org.scala-lang" % "scala-library" % scalaVersion.value, - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "com.twitter" %% "bijection-macros" % bijectionVersion - ), - addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full) -).dependsOn(scaldingCore) +lazy val scaldingDb = project + .in(file("scalding-db")) + .dependsOn(scaldingCore) + .settings( + name := "scalding-db", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.scala-lang" % "scala-library" % scalaVersion.value, + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "com.twitter" %% "bijection-macros" % bijectionVersion + ), + addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full) + ) -lazy val scaldingThriftMacrosFixtures = module("thrift-macros-fixtures") +lazy val scaldingThriftMacrosFixtures = project + .in(file("scalding-thrift-macros-fixtures")) .settings( + name := "scalding-thrift-macros-fixtures", + sharedSettings, + mimaSettings, scroogeThriftSourceFolder in Test := baseDirectory.value / "src/test/resources", libraryDependencies ++= Seq( "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided" @@ -635,31 +742,36 @@ lazy val scaldingThriftMacrosFixtures = module("thrift-macros-fixtures") ) ) -lazy val scaldingThriftMacros = module("thrift-macros") +lazy val scaldingThriftMacros = project + .in(file("scalding-thrift-macros")) + .dependsOn( + scaldingCore, + scaldingHadoopTest % "test", + scaldingSerialization, + scaldingThriftMacrosFixtures % "test->test") .settings( - libraryDependencies ++= Seq( - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "com.twitter" %% "bijection-macros" % bijectionVersion, - "com.twitter" % "chill-thrift" % chillVersion % "test", - "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided" - exclude("com.google.guava", "guava"), - "org.apache.thrift" % "libthrift" % thriftVersion, - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "test", - "org.apache.hadoop" % "hadoop-minicluster" % hadoopVersion % "test", - "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "test", - "org.apache.hadoop" % "hadoop-minicluster" % hadoopVersion % "test", - "org.apache.hadoop" % "hadoop-yarn-server-tests" % hadoopVersion classifier "tests", - "org.apache.hadoop" % "hadoop-yarn-server" % hadoopVersion % "test", - "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion classifier "tests", - "org.apache.hadoop" % "hadoop-common" % hadoopVersion classifier "tests", - "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests" - ), - addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full) -).dependsOn( - scaldingCore, - scaldingHadoopTest % "test", - scaldingSerialization, - scaldingThriftMacrosFixtures % "test->test") + name := "scalding-thrift-macros", + sharedSettings, + mimaSettings, + libraryDependencies ++= Seq( + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "com.twitter" %% "bijection-macros" % bijectionVersion, + "com.twitter" % "chill-thrift" % chillVersion % "test", + "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided" + exclude("com.google.guava", "guava"), + "org.apache.thrift" % "libthrift" % thriftVersion, + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "test", + "org.apache.hadoop" % "hadoop-minicluster" % hadoopVersion % "test", + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "test", + "org.apache.hadoop" % "hadoop-minicluster" % hadoopVersion % "test", + "org.apache.hadoop" % "hadoop-yarn-server-tests" % hadoopVersion classifier "tests", + "org.apache.hadoop" % "hadoop-yarn-server" % hadoopVersion % "test", + "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion classifier "tests", + "org.apache.hadoop" % "hadoop-common" % hadoopVersion classifier "tests", + "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests" + ), + addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full) + ) def docsSourcesAndProjects(sv: String): Seq[ProjectReference] = Seq( @@ -691,7 +803,7 @@ lazy val docSettings = Seq( micrositeBaseUrl := "scalding", micrositeDocumentationUrl := "api", micrositeGithubOwner := "twitter", - micrositeExtraMdFiles := Map(file("CONTRIBUTING.md") -> "contributing.md"), + micrositeExtraMdFiles := Map(file("CONTRIBUTING.md") -> microsites.ExtraMdFileConfig("contributing.md", "page")), micrositeGithubRepo := "scalding", micrositePalette := Map( "brand-primary" -> "#5B5988", @@ -723,12 +835,10 @@ lazy val docSettings = Seq( // Documentation is generated for projects defined in // `docsSourcesAndProjects`. lazy val docs = project - .enablePlugins(MicrositesPlugin) + .enablePlugins(MicrositesPlugin, ScalaUnidocPlugin, GhpagesPlugin) .settings(moduleName := "scalding-docs") .settings(sharedSettings) .settings(noPublishSettings) - .settings(unidocSettings) - .settings(ghpages.settings) .settings(docSettings) - .settings(tutScalacOptions ~= (_.filterNot(Set("-Ywarn-unused-import", "-Ywarn-dead-code")))) + .settings(scalacOptions in Tut ~= (_.filterNot(Set("-Ywarn-unused-import", "-Ywarn-dead-code")))) .dependsOn(scaldingCore) diff --git a/project/build.properties b/project/build.properties index 8e682c526d..797e7ccfdb 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.18 +sbt.version=1.3.10 diff --git a/project/plugins.sbt b/project/plugins.sbt index 2294d9e17b..149ede7830 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,16 +6,16 @@ resolvers ++= Seq( "Twitter Maven" at "https://maven.twttr.com" ) -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.10.2") -addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.3.3") -addSbtPlugin("com.fortysevendeg" % "sbt-microsites" % "0.3.3") -addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.0") -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") -addSbtPlugin("com.twitter" %% "scrooge-sbt-plugin" % "18.9.0") -addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.1.14") -addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.4") -addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.6.2") -addSbtPlugin("com.typesafe.sbt" % "sbt-scalariform" % "1.3.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.0") -addSbtPlugin("org.wartremover" % "sbt-wartremover" % "2.1.1") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") +addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3") +addSbtPlugin("com.47deg" % "sbt-microsites" % "1.1.4") +addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13") +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "2.0.1") +addSbtPlugin("com.twitter" %% "scrooge-sbt-plugin" % "20.4.0") +addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.7.0") +addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.3") +addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") +addSbtPlugin("org.scalariform" % "sbt-scalariform" % "1.8.3") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.2") +addSbtPlugin("org.wartremover" % "sbt-wartremover" % "2.4.7") diff --git a/scalding-avro/src/main/scala/com/twitter/scalding/avro/package.scala b/scalding-avro/src/main/scala/com/twitter/scalding/avro/package.scala index c5bb868103..bb502ff951 100644 --- a/scalding-avro/src/main/scala/com/twitter/scalding/avro/package.scala +++ b/scalding-avro/src/main/scala/com/twitter/scalding/avro/package.scala @@ -21,7 +21,8 @@ import collection.JavaConverters._ import cascading.tuple.Fields package object avro { - def writePackedAvro[T](pipe: TypedPipe[T], path: String)(implicit mf: Manifest[T], + def writePackedAvro[T](pipe: TypedPipe[T], path: String)(implicit + mf: Manifest[T], st: AvroSchemaType[T], conv: TupleConverter[T], set: TupleSetter[T], @@ -31,7 +32,8 @@ package object avro { pipe.write(sink) } - def writeUnpackedAvro[T <: Product](pipe: TypedPipe[T], path: String, schema: Schema)(implicit mf: Manifest[T], + def writeUnpackedAvro[T <: Product](pipe: TypedPipe[T], path: String, schema: Schema)(implicit + mf: Manifest[T], conv: TupleConverter[T], set: TupleSetter[T], flow: FlowDef, diff --git a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala index 09ad893905..456c365a67 100644 --- a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala +++ b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala @@ -120,7 +120,8 @@ class LzoGenericScheme[M](@transient conv: BinaryConverter[M], clazz: Class[M]) override protected def prepareBinaryWritable(): GenericWritable[M] = new GenericWritable(conv) - override def sourceConfInit(fp: FlowProcess[JobConf], + override def sourceConfInit( + fp: FlowProcess[JobConf], tap: Tap[JobConf, RecordReader[_, _], OutputCollector[_, _]], conf: JobConf): Unit = { @@ -131,7 +132,8 @@ class LzoGenericScheme[M](@transient conv: BinaryConverter[M], clazz: Class[M]) DelegateCombineFileInputFormat.setDelegateInputFormat(conf, classOf[MultiInputFormat[_]]) } - override def sinkConfInit(fp: FlowProcess[JobConf], + override def sinkConfInit( + fp: FlowProcess[JobConf], tap: Tap[JobConf, RecordReader[_, _], OutputCollector[_, _]], conf: JobConf): Unit = { LzoGenericScheme.setConverter(conv, conf, SinkConfigBinaryConverterProvider.ProviderConfKey) diff --git a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala index 07eabe5256..578fe5374a 100644 --- a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala +++ b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala @@ -16,7 +16,6 @@ limitations under the License. package com.twitter.scalding.commons.source - import com.twitter.elephantbird.mapreduce.io.BinaryConverter import com.twitter.scalding._ diff --git a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/VersionedKeyValSource.scala b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/VersionedKeyValSource.scala index c95f2c746e..5b876c88e2 100644 --- a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/VersionedKeyValSource.scala +++ b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/VersionedKeyValSource.scala @@ -54,7 +54,8 @@ object VersionedKeyValSource { class VersionedKeyValSource[K, V](val path: String, val sourceVersion: Option[Long], val sinkVersion: Option[Long], val maxFailures: Int, val versionsToKeep: Int)( - implicit @transient codec: Injection[(K, V), (Array[Byte], Array[Byte])]) extends Source + implicit + @transient codec: Injection[(K, V), (Array[Byte], Array[Byte])]) extends Source with Mappable[(K, V)] with TypedSink[(K, V)] { @@ -253,7 +254,8 @@ class RichPipeEx(pipe: Pipe) extends java.io.Serializable { // VersionedKeyValSource always merges with the most recent complete // version - def writeIncremental[K, V](src: VersionedKeyValSource[K, V], fields: Fields, reducers: Int = 1)(implicit monoid: Monoid[V], + def writeIncremental[K, V](src: VersionedKeyValSource[K, V], fields: Fields, reducers: Int = 1)(implicit + monoid: Monoid[V], flowDef: FlowDef, mode: Mode) = { def appendToken(pipe: Pipe, token: Int) = diff --git a/scalding-commons/src/main/scala/com/twitter/scalding/examples/KMeans.scala b/scalding-commons/src/main/scala/com/twitter/scalding/examples/KMeans.scala index d7db06d87e..972c89f80f 100644 --- a/scalding-commons/src/main/scala/com/twitter/scalding/examples/KMeans.scala +++ b/scalding-commons/src/main/scala/com/twitter/scalding/examples/KMeans.scala @@ -37,7 +37,8 @@ object KMeans { scale(1.0 / count, vec) } - private def closest[Id](from: Vector[Double], + private def closest[Id]( + from: Vector[Double], centroids: TraversableOnce[(Id, Vector[Double])]): (Id, Vector[Double]) = centroids // compute the distance to each center @@ -55,7 +56,8 @@ object KMeans { * the new clusters * and the new list of labeled vectors */ - def kmeansStep(k: Int, + def kmeansStep( + k: Int, s: Stat, clusters: ValuePipe[List[LabeledVector]], points: TypedPipe[LabeledVector]): Execution[(ValuePipe[List[LabeledVector]], TypedPipe[LabeledVector])] = { @@ -112,13 +114,15 @@ object KMeans { * Run the full k-means algorithm by flatMapping the above function into itself * while the number of vectors that changed is not zero */ - def kmeans(k: Int, + def kmeans( + k: Int, clusters: ValuePipe[List[LabeledVector]], points: TypedPipe[LabeledVector]): Execution[(Int, ValuePipe[List[LabeledVector]], TypedPipe[LabeledVector])] = { val key = StatKey("changed", "scalding.kmeans") - def go(s: Stat, + def go( + s: Stat, c: ValuePipe[List[LabeledVector]], p: TypedPipe[LabeledVector], step: Int): Execution[(Int, ValuePipe[List[LabeledVector]], TypedPipe[LabeledVector])] = diff --git a/scalding-core/src/main/scala/com/twitter/scalding/CascadingMode.scala b/scalding-core/src/main/scala/com/twitter/scalding/CascadingMode.scala index 6227145129..35bbe5da7f 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/CascadingMode.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/CascadingMode.scala @@ -171,7 +171,8 @@ object Hdfs { def default: Hdfs = Hdfs(true, new Configuration) } -case class HadoopTest(@transient conf: Configuration, +case class HadoopTest( + @transient conf: Configuration, @transient buffers: Source => Option[Buffer[Tuple]]) extends HadoopMode with TestMode { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/Config.scala b/scalding-core/src/main/scala/com/twitter/scalding/Config.scala index f89bc55489..532522368c 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/Config.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/Config.scala @@ -22,7 +22,7 @@ import org.apache.hadoop.io.serializer.{ Serialization => HSerialization } import com.twitter.chill.{ ExternalizerCodec, ExternalizerInjection, Externalizer, KryoInstantiator } import com.twitter.chill.config.{ ScalaMapConfig, ConfiguredInstantiator } import com.twitter.bijection.{ Base64String, Injection } -import com.twitter.scalding.filecache.{CachedFile, DistributedCacheFile, HadoopCachedFile} +import com.twitter.scalding.filecache.{ CachedFile, DistributedCacheFile, HadoopCachedFile } import cascading.pipe.assembly.AggregateBy import cascading.flow.{ FlowListener, FlowStepListener, FlowProps, FlowStepStrategy } @@ -63,7 +63,8 @@ abstract class Config extends Serializable { * @return new Config with cached files */ def addDistributedCacheFiles(cachedFiles: CachedFile*): Config = - cachedFiles.foldLeft(this) { case (config, file) => + cachedFiles.foldLeft(this) { + case (config, file) => file match { case hadoopFile: HadoopCachedFile => Config.addDistributedCacheFile(hadoopFile.sourceUri, config) @@ -116,7 +117,6 @@ abstract class Config extends Serializable { } catch { case err: Throwable => Failure(err) } } - /* * Used in joins to determine how much of the "right hand side" of * the join to keep in memory @@ -207,12 +207,14 @@ abstract class Config extends Serializable { * with a class to serialize to bootstrap the process: * Left((classOf[serialization.KryoHadoop], myInstance)) */ - def setSerialization(kryo: Either[(Class[_ <: KryoInstantiator], KryoInstantiator), Class[_ <: KryoInstantiator]], + def setSerialization( + kryo: Either[(Class[_ <: KryoInstantiator], KryoInstantiator), Class[_ <: KryoInstantiator]], userHadoop: Seq[Class[_ <: HSerialization[_]]] = Nil): Config = { // Hadoop and Cascading should come first val first: Seq[Class[_ <: HSerialization[_]]] = - Seq(classOf[org.apache.hadoop.io.serializer.WritableSerialization], + Seq( + classOf[org.apache.hadoop.io.serializer.WritableSerialization], classOf[cascading.tuple.hadoop.TupleSerialization], classOf[serialization.WrappedSerialization[_]]) // this must come last diff --git a/scalding-core/src/main/scala/com/twitter/scalding/CumulativeSum.scala b/scalding-core/src/main/scala/com/twitter/scalding/CumulativeSum.scala index b26882c732..5e91d17438 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/CumulativeSum.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/CumulativeSum.scala @@ -37,7 +37,8 @@ object CumulativeSum { val pipe: TypedPipe[(K, (U, V))]) { /** Takes a sortable field and a monoid and returns the cumulative sum of that monoid **/ def cumulativeSum( - implicit sg: Semigroup[V], + implicit + sg: Semigroup[V], ordU: Ordering[U], ordK: Ordering[K]): SortedGrouped[K, (U, V)] = { pipe.group @@ -59,7 +60,8 @@ object CumulativeSum { * partitions for a single key to go through a single scan. */ def cumulativeSum[S](partition: U => S)( - implicit ordS: Ordering[S], + implicit + ordS: Ordering[S], sg: Semigroup[V], ordU: Ordering[U], ordK: Ordering[K]): TypedPipe[(K, (U, V))] = { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/Execution.scala b/scalding-core/src/main/scala/com/twitter/scalding/Execution.scala index df767fc90c..0c32731674 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/Execution.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/Execution.scala @@ -20,11 +20,11 @@ import com.stripe.dagon.{ Dag, Id, Rule } import com.twitter.algebird.monad.Trampoline import com.twitter.algebird.{ Monoid, Monad, Semigroup } import com.twitter.scalding.cascading_interop.FlowListenerPromise -import com.twitter.scalding.filecache.{CachedFile, DistributedCacheFile} +import com.twitter.scalding.filecache.{ CachedFile, DistributedCacheFile } import com.twitter.scalding.typed.functions.{ ConsList, ReverseList } import com.twitter.scalding.typed.cascading_backend.AsyncFlowDefRunner import com.twitter.scalding.cascading_interop.FlowListenerPromise.FlowStopException -import com.stripe.dagon.{Memoize, RefPair} +import com.stripe.dagon.{ Memoize, RefPair } import java.io.Serializable import java.util.UUID import scala.collection.mutable @@ -176,7 +176,8 @@ sealed trait Execution[+T] extends Serializable { self: Product => * completes, and a future of the result, counters and cache after the future * is complete */ - protected def runStats(conf: Config, + protected def runStats( + conf: Config, mode: Mode, cache: EvalCache)(implicit cec: ConcurrentExecutionContext): Trampoline[CFuture[(T, Map[Long, ExecutionCounters])]] @@ -193,7 +194,8 @@ sealed trait Execution[+T] extends Serializable { self: Product => * always code smell. Very seldom should you need to wait on a future. */ def waitFor(conf: Config, mode: Mode): Try[T] = - Try(Await.result(run(conf, mode)(ConcurrentExecutionContext.global), + Try(Await.result( + run(conf, mode)(ConcurrentExecutionContext.global), scala.concurrent.duration.Duration.Inf)) /** @@ -397,11 +399,10 @@ object Execution { .asInstanceOf[(Boolean, CFuture[(T, Counters)])] def getOrElseInsert[T](cfg: Config, ex: Execution[T], - res: => CFuture[(T, Counters)] ): CFuture[(T, Counters)] = + res: => CFuture[(T, Counters)]): CFuture[(T, Counters)] = getOrElseInsertWithFeedback(cfg, ex, res)._2 } - private[scalding] final case class FutureConst[T](get: ConcurrentExecutionContext => Future[T]) extends Execution[T] { protected def runStats(conf: Config, mode: Mode, cache: EvalCache)(implicit cec: ConcurrentExecutionContext) = Trampoline { @@ -420,21 +421,23 @@ object Execution { } private[scalding] final case class FlatMapped[S, T](prev: Execution[S], fn: S => Execution[T]) extends Execution[T] { protected def runStats(conf: Config, mode: Mode, cache: EvalCache)(implicit cec: ConcurrentExecutionContext) = - Trampoline.call(prev.runStats(conf, mode, cache)).map { case CFuture(fut1, cancelHandler1) => - lazy val uncachedCFut = for { - (s, st1) <- fut1 - next0 = fn(s) - // next0 has not been optimized yet, we need to try - next = optimize(conf, next0) - } yield { - Trampoline.call(next.runStats(conf, mode, cache)).get.map { case (t, st2) => - (t, st1 ++ st2) + Trampoline.call(prev.runStats(conf, mode, cache)).map { + case CFuture(fut1, cancelHandler1) => + lazy val uncachedCFut = for { + (s, st1) <- fut1 + next0 = fn(s) + // next0 has not been optimized yet, we need to try + next = optimize(conf, next0) + } yield { + Trampoline.call(next.runStats(conf, mode, cache)).get.map { + case (t, st2) => + (t, st1 ++ st2) + } } - } - val futCancel = cache.getOrElseInsert(conf, this, CFuture.fromFuture(uncachedCFut)) + val futCancel = cache.getOrElseInsert(conf, this, CFuture.fromFuture(uncachedCFut)) - CFuture(futCancel.future, cancelHandler1.compose(futCancel.cancellationHandler)) + CFuture(futCancel.future, cancelHandler1.compose(futCancel.cancellationHandler)) } } @@ -514,10 +517,11 @@ object Execution { private[scalding] final case class RecoverWith[T](prev: Execution[T], fn: PartialFunction[Throwable, Execution[T]]) extends Execution[T] { protected def runStats(conf: Config, mode: Mode, cache: EvalCache)(implicit cec: ConcurrentExecutionContext) = - Trampoline.call(prev.runStats(conf, mode, cache)).map { case CFuture(fut, cancelHandler) => - lazy val uncachedFut = { + Trampoline.call(prev.runStats(conf, mode, cache)).map { + case CFuture(fut, cancelHandler) => + lazy val uncachedFut = { fut - .map {v => (v, CancellationHandler.empty) } // map this to the right shape + .map { v => (v, CancellationHandler.empty) } // map this to the right shape .recoverWith { case t: FlowStopException => // do not recover when the flow was stopped Future.failed(t) @@ -528,18 +532,18 @@ object Execution { val CFuture(f, c) = ex.runStats(conf, mode, cache).get f.map { v => (v, c) } } - chainedFn(t) + chainedFn(t) } - } + } - val recoveredFut = cache.getOrElseInsert(conf, this, CFuture(uncachedFut.map(_._1), CancellationHandler.fromFuture(uncachedFut.map(_._2)))) - CFuture(recoveredFut.future, cancelHandler.compose(recoveredFut.cancellationHandler)) + val recoveredFut = cache.getOrElseInsert(conf, this, CFuture(uncachedFut.map(_._1), CancellationHandler.fromFuture(uncachedFut.map(_._2)))) + CFuture(recoveredFut.future, cancelHandler.compose(recoveredFut.cancellationHandler)) } } /** - * Standard scala zip waits forever on the left side, even if the right side fails - */ + * Standard scala zip waits forever on the left side, even if the right side fails + */ def failFastZip[T, U](ft: Future[T], fu: Future[U])(implicit cec: ConcurrentExecutionContext): Future[(T, U)] = { type State = Either[(T, Promise[U]), (U, Promise[T])] val middleState = Promise[State]() @@ -678,7 +682,8 @@ object Execution { val idA = optWriteId.original val origPipe = d1.evaluate(idA) val optPipe = d2.evaluate(idA) - OptimizedWrite(original = origPipe, + OptimizedWrite( + original = origPipe, toWrite = optWriteId.toWrite.replacePipe(optPipe)) } go(optWrite) :: tail @@ -716,16 +721,14 @@ object Execution { */ private[Execution] def getForced[T]( conf: Config, - initial: TypedPipe[T] - )(implicit cec: ConcurrentExecutionContext): Future[TypedPipe[T]] + initial: TypedPipe[T])(implicit cec: ConcurrentExecutionContext): Future[TypedPipe[T]] /** * This should only be called after a call to execute */ private[Execution] def getIterable[T]( conf: Config, - initial: TypedPipe[T] - )(implicit cec: ConcurrentExecutionContext): Future[Iterable[T]] + initial: TypedPipe[T])(implicit cec: ConcurrentExecutionContext): Future[Iterable[T]] } /** @@ -755,7 +758,8 @@ object Execution { */ override def map[U](mapFn: T => U): Execution[U] = - WriteExecution(head, + WriteExecution( + head, tail, ExecutionOptimizationRules.MapWrite.ComposeMap(result, mapFn)) @@ -818,7 +822,6 @@ object Execution { Trampoline(cache.getOrElseInsert(conf, this, uncachedFutureCancel)) } - /** * This is such an important optimization, that we apply it locally. * It is a bit ugly to have it here and in ExecutionOptimizationRules @@ -841,7 +844,7 @@ object Execution { */ override def zip[U](that: Execution[U]): Execution[(T, U)] = that match { - case w1@WriteExecution(_, _, _) => + case w1 @ WriteExecution(_, _, _) => ExecutionOptimizationRules.ZipWrite.mergeWrite(this, w1) case that => Zipped(this, that) } @@ -926,7 +929,8 @@ object Execution { write(pipe, sink, ()) private[scalding] def write[T, U](pipe: TypedPipe[T], sink: TypedSink[T], presentType: => U): Execution[U] = - WriteExecution(ToWrite.SimpleWrite(pipe, sink), + WriteExecution( + ToWrite.SimpleWrite(pipe, sink), Nil, { tup => Future(presentType)(tup._4) }) @@ -1002,7 +1006,8 @@ object Execution { /** * combine several executions and run them in parallel when .run is called */ - def zip[A, B, C, D](ax: Execution[A], + def zip[A, B, C, D]( + ax: Execution[A], bx: Execution[B], cx: Execution[C], dx: Execution[D]): Execution[(A, B, C, D)] = @@ -1011,7 +1016,8 @@ object Execution { /** * combine several executions and run them in parallel when .run is called */ - def zip[A, B, C, D, E](ax: Execution[A], + def zip[A, B, C, D, E]( + ax: Execution[A], bx: Execution[B], cx: Execution[C], dx: Execution[D], diff --git a/scalding-core/src/main/scala/com/twitter/scalding/ExecutionOptimizationRules.scala b/scalding-core/src/main/scala/com/twitter/scalding/ExecutionOptimizationRules.scala index 072715aa37..bb856dba5a 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/ExecutionOptimizationRules.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/ExecutionOptimizationRules.scala @@ -1,11 +1,11 @@ package com.twitter.scalding -import com.stripe.dagon.{Dag, FunctionK, Literal, Memoize, PartialRule, Rule} -import com.twitter.scalding.ExecutionOptimizationRules.ZipMap.{MapLeft, MapRight} +import com.stripe.dagon.{ Dag, FunctionK, Literal, Memoize, PartialRule, Rule } +import com.twitter.scalding.ExecutionOptimizationRules.ZipMap.{ MapLeft, MapRight } import com.twitter.scalding.typed.functions.ComposedFunctions.ComposedMapFn -import com.twitter.scalding.typed.functions.{ComposedFunctions, Identity, Swap} +import com.twitter.scalding.typed.functions.{ ComposedFunctions, Identity, Swap } import scala.annotation.tailrec -import scala.concurrent.{Future, ExecutionContext => ConcurrentExecutionContext} +import scala.concurrent.{ Future, ExecutionContext => ConcurrentExecutionContext } object ExecutionOptimizationRules { type LiteralExecution[T] = Literal[Execution, T] @@ -30,7 +30,7 @@ object ExecutionOptimizationRules { Memoize.functionK[Execution, LiteralExecution]( new Memoize.RecursiveK[Execution, LiteralExecution] { override def toFunction[A] = { - case (e@Execution.ReaderExecution, _) => + case (e @ Execution.ReaderExecution, _) => Literal.Const(e) case (e: Execution.FutureConst[a], _) => Literal.Const(e) @@ -59,8 +59,7 @@ object ExecutionOptimizationRules { case (e: Execution.Zipped[a, b], f) => Literal.Binary(f(e.one), f(e.two), Execution.Zipped(_: Execution[a], _: Execution[b])) } - } - ) + }) /** * If `Execution` is `FlowDefExecution` or `WriteExecution`, @@ -143,8 +142,7 @@ object ExecutionOptimizationRules { } case class ComposedFn[A1, A2, A, B1, B2, B]( fn1: Function1[(A1, A2), A], - fn2: Function1[(B1, B2), B] - ) extends Function1[((A1, B1), (A2, B2)), (A, B)] { + fn2: Function1[(B1, B2), B]) extends Function1[((A1, B1), (A2, B2)), (A, B)] { override def apply(v1: ((A1, B1), (A2, B2))): (A, B) = (fn1(v1._1._1, v1._2._1), fn2(v1._1._2, v1._2._2)) } @@ -181,7 +179,7 @@ object ExecutionOptimizationRules { def toExecution[A](ex: FlattenedZip[A]): Execution[A] = ex match { case NonWrite(nonWrite) => nonWrite case Write(write) => write - case c@Composed(_, _, _) => c.write.zip(c.nonWrite).map(c.compose) + case c @ Composed(_, _, _) => c.write.zip(c.nonWrite).map(c.compose) } def map[A, B](ex: FlattenedZip[A], fn: A => B): FlattenedZip[B] = ex match { @@ -194,25 +192,25 @@ object ExecutionOptimizationRules { } def zip[A, B](left: FlattenedZip[A], right: FlattenedZip[B]): FlattenedZip[(A, B)] = (left, right) match { - case (left@NonWrite(_), right@NonWrite(_)) => + case (left @ NonWrite(_), right @ NonWrite(_)) => NonWrite(left.nonWrite.zip(right.nonWrite)) - case (left@NonWrite(_), right@Write(_)) => + case (left @ NonWrite(_), right @ Write(_)) => Composed(right.write, left.nonWrite, Swap[B, A]()) - case (left@NonWrite(_), right@Composed(_, _, _)) => + case (left @ NonWrite(_), right @ Composed(_, _, _)) => zipNonWriteComposed(left, right) - case (left@Write(_), right@NonWrite(_)) => + case (left @ Write(_), right @ NonWrite(_)) => Composed(left.write, right.nonWrite, Identity[(A, B)]()) - case (left@Write(_), right@Write(_)) => + case (left @ Write(_), right @ Write(_)) => Write(mergeWrite(left.write, right.write)) - case (left@Write(_), right@Composed(_, _, _)) => + case (left @ Write(_), right @ Composed(_, _, _)) => zipWriteComposed(left, right) - case (left@Composed(_, _, _), right@NonWrite(_)) => + case (left @ Composed(_, _, _), right @ NonWrite(_)) => map(zipNonWriteComposed(right, left), Swap[B, A]()) - case (left@Composed(_, _, _), right@Write(_)) => + case (left @ Composed(_, _, _), right @ Write(_)) => map(zipWriteComposed(right, left), Swap[B, A]()) - case (left@Composed(_, _, _), right@Composed(_, _, _)) => + case (left @ Composed(_, _, _), right @ Composed(_, _, _)) => Composed(mergeWrite(left.write, right.write), left.nonWrite.zip(right.nonWrite), ComposedFn(left.compose, right.compose)) } @@ -232,7 +230,7 @@ object ExecutionOptimizationRules { ex match { case Zipped(left, right) => zip(apply(left), apply(right)) case Mapped(that, fn) => map(apply(that), fn) - case write@WriteExecution(_, _, _) => FlattenedZip.Write(write) + case write @ WriteExecution(_, _, _) => FlattenedZip.Write(write) case notZipMap => FlattenedZip.NonWrite(notZipMap) } } @@ -263,7 +261,7 @@ object ExecutionOptimizationRules { } def apply[A](on: Dag[Execution]) = { - case z@Zipped(_, _) => optimize(z) + case z @ Zipped(_, _) => optimize(z) case _ => // since this optimization only applies to zips, there // is no need to check on nodes that aren't zips. @@ -342,7 +340,6 @@ object ExecutionOptimizationRules { } } - val std: Rule[Execution] = Rule.orElse( List( @@ -350,9 +347,7 @@ object ExecutionOptimizationRules { MapWrite, ZipMap, ZipFlatMap, - FuseMaps - ) - ) + FuseMaps)) def apply[A](e: Execution[A], r: Rule[Execution]): Execution[A] = { try { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/FileSource.scala b/scalding-core/src/main/scala/com/twitter/scalding/FileSource.scala index 9fa03781d3..8a3601f923 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/FileSource.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/FileSource.scala @@ -15,23 +15,23 @@ limitations under the License. */ package com.twitter.scalding -import java.io.{InputStream, OutputStream} -import java.util.{Properties, UUID} +import java.io.{ InputStream, OutputStream } +import java.util.{ Properties, UUID } import cascading.scheme.Scheme -import cascading.scheme.hadoop.{SequenceFile => CHSequenceFile, TextDelimited => CHTextDelimited, TextLine => CHTextLine} -import cascading.scheme.local.{TextDelimited => CLTextDelimited, TextLine => CLTextLine} +import cascading.scheme.hadoop.{ SequenceFile => CHSequenceFile, TextDelimited => CHTextDelimited, TextLine => CHTextLine } +import cascading.scheme.local.{ TextDelimited => CLTextDelimited, TextLine => CLTextLine } import cascading.tap.hadoop.Hfs -import cascading.tap.{MultiSourceTap, SinkMode, Tap} +import cascading.tap.{ MultiSourceTap, SinkMode, Tap } import cascading.tap.local.FileTap import cascading.tuple.Fields import com.etsy.cascading.tap.local.LocalTap -import com.twitter.algebird.{MapAlgebra, OrVal} +import com.twitter.algebird.{ MapAlgebra, OrVal } import com.twitter.scalding.tap.ScaldingHfs import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path, PathFilter} -import org.apache.hadoop.mapred.{JobConf, OutputCollector, RecordReader} +import org.apache.hadoop.fs.{ FileStatus, Path, PathFilter } +import org.apache.hadoop.mapred.{ JobConf, OutputCollector, RecordReader } import org.slf4j.LoggerFactory -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } /** * A base class for sources that take a scheme trait. @@ -51,7 +51,8 @@ abstract class SchemedSource extends Source { } trait HfsTapProvider { - def createHfsTap(scheme: Scheme[JobConf, RecordReader[_, _], OutputCollector[_, _], _, _], + def createHfsTap( + scheme: Scheme[JobConf, RecordReader[_, _], OutputCollector[_, _], _, _], path: String, sinkMode: SinkMode): Hfs = new ScaldingHfs(scheme, path, sinkMode) @@ -195,8 +196,7 @@ object FileSource { // dir -> ( OrVal(SuccessFileFilter.accept(fileStatus.getPath) && fileStatus.isFile), - OrVal(HiddenFileFilter.accept(fileStatus.getPath)) - ) + OrVal(HiddenFileFilter.accept(fileStatus.getPath))) } // OR by key @@ -502,7 +502,8 @@ case class MultipleTsvFiles(p: Seq[String], override val fields: Fields = Fields * Csv value source * separated by commas and quotes wrapping all fields */ -case class Csv(p: String, +case class Csv( + p: String, override val separator: String = ",", override val fields: Fields = Fields.ALL, override val skipHeader: Boolean = false, @@ -540,7 +541,8 @@ class TextLine(p: String, override val sinkMode: SinkMode, override val textEnco /** * Alternate typed TextLine source that keeps both 'offset and 'line fields. */ -class OffsetTextLine(filepath: String, +class OffsetTextLine( + filepath: String, override val sinkMode: SinkMode, override val textEncoding: String) extends FixedPathSource(filepath) with Mappable[(Long, String)] with TextSourceScheme { @@ -574,7 +576,8 @@ case class MultipleTextLineFiles(p: String*) extends FixedPathSource(p: _*) with * Delimited files source * allowing to override separator and quotation characters and header configuration */ -case class MultipleDelimitedFiles(f: Fields, +case class MultipleDelimitedFiles( + f: Fields, override val separator: String, override val quote: String, override val skipHeader: Boolean, diff --git a/scalding-core/src/main/scala/com/twitter/scalding/FlowState.scala b/scalding-core/src/main/scala/com/twitter/scalding/FlowState.scala index f7773d049b..18f35e4462 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/FlowState.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/FlowState.scala @@ -48,7 +48,8 @@ case class FlowState( sourceMap.values.toSet[Source].foreach(_.validateTaps(mode)) def merge(that: FlowState): FlowState = - FlowState(sourceMap = sourceMap ++ that.sourceMap, + FlowState( + sourceMap = sourceMap ++ that.sourceMap, flowConfigUpdates = flowConfigUpdates ++ that.flowConfigUpdates, pendingTypedWrites = pendingTypedWrites ::: that.pendingTypedWrites) } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/GeneratedConversions.scala b/scalding-core/src/main/scala/com/twitter/scalding/GeneratedConversions.scala index 1470b93a20..d9f8d1bcae 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/GeneratedConversions.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/GeneratedConversions.scala @@ -5,918 +5,959 @@ import cascading.tuple.TupleEntry trait GeneratedTupleConverters extends LowPriorityTupleConverters { - case class TupleConverter1[A]( gA : TupleGetter[A]) extends TupleConverter[Tuple1[A]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple1(gA.get(tup, 0)) - } - def arity = 1 + case class TupleConverter1[A](gA: TupleGetter[A]) extends TupleConverter[Tuple1[A]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple1(gA.get(tup, 0)) + } + def arity = 1 } - implicit def tuple1Converter[A](implicit - gA : TupleGetter[A]): TupleConverter[Tuple1[A]] = TupleConverter1(gA) + implicit def tuple1Converter[A](implicit gA: TupleGetter[A]): TupleConverter[Tuple1[A]] = TupleConverter1(gA) - case class TupleConverter2[A,B]( gA : TupleGetter[A], - gB : TupleGetter[B]) extends TupleConverter[Tuple2[A,B]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple2(gA.get(tup, 0), - gB.get(tup, 1)) - } - def arity = 2 + case class TupleConverter2[A, B]( + gA: TupleGetter[A], + gB: TupleGetter[B]) extends TupleConverter[Tuple2[A, B]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple2( + gA.get(tup, 0), + gB.get(tup, 1)) + } + def arity = 2 } - implicit def tuple2Converter[A,B](implicit - gA : TupleGetter[A], - gB : TupleGetter[B]): TupleConverter[Tuple2[A,B]] = TupleConverter2(gA, gB) + implicit def tuple2Converter[A, B](implicit + gA: TupleGetter[A], + gB: TupleGetter[B]): TupleConverter[Tuple2[A, B]] = TupleConverter2(gA, gB) - case class TupleConverter3[A,B,C]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C]) extends TupleConverter[Tuple3[A,B,C]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple3(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2)) - } - def arity = 3 + case class TupleConverter3[A, B, C]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C]) extends TupleConverter[Tuple3[A, B, C]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple3( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2)) + } + def arity = 3 } - implicit def tuple3Converter[A,B,C](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C]): TupleConverter[Tuple3[A,B,C]] = TupleConverter3(gA, gB, gC) + implicit def tuple3Converter[A, B, C](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C]): TupleConverter[Tuple3[A, B, C]] = TupleConverter3(gA, gB, gC) - case class TupleConverter4[A,B,C,D]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D]) extends TupleConverter[Tuple4[A,B,C,D]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple4(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3)) - } - def arity = 4 + case class TupleConverter4[A, B, C, D]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D]) extends TupleConverter[Tuple4[A, B, C, D]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple4( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3)) + } + def arity = 4 } - implicit def tuple4Converter[A,B,C,D](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D]): TupleConverter[Tuple4[A,B,C,D]] = TupleConverter4(gA, gB, gC, gD) + implicit def tuple4Converter[A, B, C, D](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D]): TupleConverter[Tuple4[A, B, C, D]] = TupleConverter4(gA, gB, gC, gD) - case class TupleConverter5[A,B,C,D,E]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E]) extends TupleConverter[Tuple5[A,B,C,D,E]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple5(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4)) - } - def arity = 5 + case class TupleConverter5[A, B, C, D, E]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E]) extends TupleConverter[Tuple5[A, B, C, D, E]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple5( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4)) + } + def arity = 5 } - implicit def tuple5Converter[A,B,C,D,E](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E]): TupleConverter[Tuple5[A,B,C,D,E]] = TupleConverter5(gA, gB, gC, gD, gE) + implicit def tuple5Converter[A, B, C, D, E](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E]): TupleConverter[Tuple5[A, B, C, D, E]] = TupleConverter5(gA, gB, gC, gD, gE) - case class TupleConverter6[A,B,C,D,E,F]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F]) extends TupleConverter[Tuple6[A,B,C,D,E,F]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple6(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5)) - } - def arity = 6 + case class TupleConverter6[A, B, C, D, E, F]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F]) extends TupleConverter[Tuple6[A, B, C, D, E, F]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple6( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5)) + } + def arity = 6 } - implicit def tuple6Converter[A,B,C,D,E,F](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F]): TupleConverter[Tuple6[A,B,C,D,E,F]] = TupleConverter6(gA, gB, gC, gD, gE, gF) + implicit def tuple6Converter[A, B, C, D, E, F](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F]): TupleConverter[Tuple6[A, B, C, D, E, F]] = TupleConverter6(gA, gB, gC, gD, gE, gF) - case class TupleConverter7[A,B,C,D,E,F,G]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G]) extends TupleConverter[Tuple7[A,B,C,D,E,F,G]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple7(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6)) - } - def arity = 7 + case class TupleConverter7[A, B, C, D, E, F, G]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G]) extends TupleConverter[Tuple7[A, B, C, D, E, F, G]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple7( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6)) + } + def arity = 7 } - implicit def tuple7Converter[A,B,C,D,E,F,G](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G]): TupleConverter[Tuple7[A,B,C,D,E,F,G]] = TupleConverter7(gA, gB, gC, gD, gE, gF, gG) + implicit def tuple7Converter[A, B, C, D, E, F, G](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G]): TupleConverter[Tuple7[A, B, C, D, E, F, G]] = TupleConverter7(gA, gB, gC, gD, gE, gF, gG) - case class TupleConverter8[A,B,C,D,E,F,G,H]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H]) extends TupleConverter[Tuple8[A,B,C,D,E,F,G,H]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple8(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7)) - } - def arity = 8 + case class TupleConverter8[A, B, C, D, E, F, G, H]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H]) extends TupleConverter[Tuple8[A, B, C, D, E, F, G, H]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple8( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7)) + } + def arity = 8 } - implicit def tuple8Converter[A,B,C,D,E,F,G,H](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H]): TupleConverter[Tuple8[A,B,C,D,E,F,G,H]] = TupleConverter8(gA, gB, gC, gD, gE, gF, gG, gH) + implicit def tuple8Converter[A, B, C, D, E, F, G, H](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H]): TupleConverter[Tuple8[A, B, C, D, E, F, G, H]] = TupleConverter8(gA, gB, gC, gD, gE, gF, gG, gH) - case class TupleConverter9[A,B,C,D,E,F,G,H,I]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I]) extends TupleConverter[Tuple9[A,B,C,D,E,F,G,H,I]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple9(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8)) - } - def arity = 9 + case class TupleConverter9[A, B, C, D, E, F, G, H, I]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I]) extends TupleConverter[Tuple9[A, B, C, D, E, F, G, H, I]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple9( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8)) + } + def arity = 9 } - implicit def tuple9Converter[A,B,C,D,E,F,G,H,I](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I]): TupleConverter[Tuple9[A,B,C,D,E,F,G,H,I]] = TupleConverter9(gA, gB, gC, gD, gE, gF, gG, gH, gI) + implicit def tuple9Converter[A, B, C, D, E, F, G, H, I](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I]): TupleConverter[Tuple9[A, B, C, D, E, F, G, H, I]] = TupleConverter9(gA, gB, gC, gD, gE, gF, gG, gH, gI) - case class TupleConverter10[A,B,C,D,E,F,G,H,I,J]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J]) extends TupleConverter[Tuple10[A,B,C,D,E,F,G,H,I,J]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple10(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9)) - } - def arity = 10 + case class TupleConverter10[A, B, C, D, E, F, G, H, I, J]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J]) extends TupleConverter[Tuple10[A, B, C, D, E, F, G, H, I, J]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple10( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9)) + } + def arity = 10 } - implicit def tuple10Converter[A,B,C,D,E,F,G,H,I,J](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J]): TupleConverter[Tuple10[A,B,C,D,E,F,G,H,I,J]] = TupleConverter10(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ) + implicit def tuple10Converter[A, B, C, D, E, F, G, H, I, J](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J]): TupleConverter[Tuple10[A, B, C, D, E, F, G, H, I, J]] = TupleConverter10(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ) - case class TupleConverter11[A,B,C,D,E,F,G,H,I,J,K]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K]) extends TupleConverter[Tuple11[A,B,C,D,E,F,G,H,I,J,K]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple11(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10)) - } - def arity = 11 + case class TupleConverter11[A, B, C, D, E, F, G, H, I, J, K]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K]) extends TupleConverter[Tuple11[A, B, C, D, E, F, G, H, I, J, K]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple11( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10)) + } + def arity = 11 } - implicit def tuple11Converter[A,B,C,D,E,F,G,H,I,J,K](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K]): TupleConverter[Tuple11[A,B,C,D,E,F,G,H,I,J,K]] = TupleConverter11(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK) + implicit def tuple11Converter[A, B, C, D, E, F, G, H, I, J, K](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K]): TupleConverter[Tuple11[A, B, C, D, E, F, G, H, I, J, K]] = TupleConverter11(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK) - case class TupleConverter12[A,B,C,D,E,F,G,H,I,J,K,L]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L]) extends TupleConverter[Tuple12[A,B,C,D,E,F,G,H,I,J,K,L]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple12(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11)) - } - def arity = 12 + case class TupleConverter12[A, B, C, D, E, F, G, H, I, J, K, L]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L]) extends TupleConverter[Tuple12[A, B, C, D, E, F, G, H, I, J, K, L]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple12( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11)) + } + def arity = 12 } - implicit def tuple12Converter[A,B,C,D,E,F,G,H,I,J,K,L](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L]): TupleConverter[Tuple12[A,B,C,D,E,F,G,H,I,J,K,L]] = TupleConverter12(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL) + implicit def tuple12Converter[A, B, C, D, E, F, G, H, I, J, K, L](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L]): TupleConverter[Tuple12[A, B, C, D, E, F, G, H, I, J, K, L]] = TupleConverter12(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL) - case class TupleConverter13[A,B,C,D,E,F,G,H,I,J,K,L,M]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M]) extends TupleConverter[Tuple13[A,B,C,D,E,F,G,H,I,J,K,L,M]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple13(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12)) - } - def arity = 13 + case class TupleConverter13[A, B, C, D, E, F, G, H, I, J, K, L, M]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M]) extends TupleConverter[Tuple13[A, B, C, D, E, F, G, H, I, J, K, L, M]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple13( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12)) + } + def arity = 13 } - implicit def tuple13Converter[A,B,C,D,E,F,G,H,I,J,K,L,M](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M]): TupleConverter[Tuple13[A,B,C,D,E,F,G,H,I,J,K,L,M]] = TupleConverter13(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM) + implicit def tuple13Converter[A, B, C, D, E, F, G, H, I, J, K, L, M](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M]): TupleConverter[Tuple13[A, B, C, D, E, F, G, H, I, J, K, L, M]] = TupleConverter13(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM) - case class TupleConverter14[A,B,C,D,E,F,G,H,I,J,K,L,M,N]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N]) extends TupleConverter[Tuple14[A,B,C,D,E,F,G,H,I,J,K,L,M,N]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple14(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12), - gN.get(tup, 13)) - } - def arity = 14 + case class TupleConverter14[A, B, C, D, E, F, G, H, I, J, K, L, M, N]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N]) extends TupleConverter[Tuple14[A, B, C, D, E, F, G, H, I, J, K, L, M, N]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple14( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12), + gN.get(tup, 13)) + } + def arity = 14 } - implicit def tuple14Converter[A,B,C,D,E,F,G,H,I,J,K,L,M,N](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N]): TupleConverter[Tuple14[A,B,C,D,E,F,G,H,I,J,K,L,M,N]] = TupleConverter14(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN) + implicit def tuple14Converter[A, B, C, D, E, F, G, H, I, J, K, L, M, N](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N]): TupleConverter[Tuple14[A, B, C, D, E, F, G, H, I, J, K, L, M, N]] = TupleConverter14(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN) - case class TupleConverter15[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O]) extends TupleConverter[Tuple15[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple15(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12), - gN.get(tup, 13), - gO.get(tup, 14)) - } - def arity = 15 + case class TupleConverter15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O]) extends TupleConverter[Tuple15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple15( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12), + gN.get(tup, 13), + gO.get(tup, 14)) + } + def arity = 15 } - implicit def tuple15Converter[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O]): TupleConverter[Tuple15[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O]] = TupleConverter15(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO) + implicit def tuple15Converter[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O]): TupleConverter[Tuple15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O]] = TupleConverter15(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO) - case class TupleConverter16[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P]) extends TupleConverter[Tuple16[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple16(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12), - gN.get(tup, 13), - gO.get(tup, 14), - gP.get(tup, 15)) - } - def arity = 16 + case class TupleConverter16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P]) extends TupleConverter[Tuple16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple16( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12), + gN.get(tup, 13), + gO.get(tup, 14), + gP.get(tup, 15)) + } + def arity = 16 } - implicit def tuple16Converter[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P]): TupleConverter[Tuple16[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P]] = TupleConverter16(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP) + implicit def tuple16Converter[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P]): TupleConverter[Tuple16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P]] = TupleConverter16(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP) - case class TupleConverter17[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q]) extends TupleConverter[Tuple17[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple17(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12), - gN.get(tup, 13), - gO.get(tup, 14), - gP.get(tup, 15), - gQ.get(tup, 16)) - } - def arity = 17 + case class TupleConverter17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q]) extends TupleConverter[Tuple17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple17( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12), + gN.get(tup, 13), + gO.get(tup, 14), + gP.get(tup, 15), + gQ.get(tup, 16)) + } + def arity = 17 } - implicit def tuple17Converter[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q]): TupleConverter[Tuple17[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q]] = TupleConverter17(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ) + implicit def tuple17Converter[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q]): TupleConverter[Tuple17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q]] = TupleConverter17(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ) - case class TupleConverter18[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R]) extends TupleConverter[Tuple18[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple18(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12), - gN.get(tup, 13), - gO.get(tup, 14), - gP.get(tup, 15), - gQ.get(tup, 16), - gR.get(tup, 17)) - } - def arity = 18 + case class TupleConverter18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R]) extends TupleConverter[Tuple18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple18( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12), + gN.get(tup, 13), + gO.get(tup, 14), + gP.get(tup, 15), + gQ.get(tup, 16), + gR.get(tup, 17)) + } + def arity = 18 } - implicit def tuple18Converter[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R]): TupleConverter[Tuple18[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R]] = TupleConverter18(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR) + implicit def tuple18Converter[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R]): TupleConverter[Tuple18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R]] = TupleConverter18(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR) - case class TupleConverter19[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R], - gS : TupleGetter[S]) extends TupleConverter[Tuple19[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple19(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12), - gN.get(tup, 13), - gO.get(tup, 14), - gP.get(tup, 15), - gQ.get(tup, 16), - gR.get(tup, 17), - gS.get(tup, 18)) - } - def arity = 19 + case class TupleConverter19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R], + gS: TupleGetter[S]) extends TupleConverter[Tuple19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple19( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12), + gN.get(tup, 13), + gO.get(tup, 14), + gP.get(tup, 15), + gQ.get(tup, 16), + gR.get(tup, 17), + gS.get(tup, 18)) + } + def arity = 19 } - implicit def tuple19Converter[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R], - gS : TupleGetter[S]): TupleConverter[Tuple19[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S]] = TupleConverter19(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR, gS) + implicit def tuple19Converter[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R], + gS: TupleGetter[S]): TupleConverter[Tuple19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S]] = TupleConverter19(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR, gS) - case class TupleConverter20[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R], - gS : TupleGetter[S], - gT : TupleGetter[T]) extends TupleConverter[Tuple20[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple20(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12), - gN.get(tup, 13), - gO.get(tup, 14), - gP.get(tup, 15), - gQ.get(tup, 16), - gR.get(tup, 17), - gS.get(tup, 18), - gT.get(tup, 19)) - } - def arity = 20 + case class TupleConverter20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R], + gS: TupleGetter[S], + gT: TupleGetter[T]) extends TupleConverter[Tuple20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple20( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12), + gN.get(tup, 13), + gO.get(tup, 14), + gP.get(tup, 15), + gQ.get(tup, 16), + gR.get(tup, 17), + gS.get(tup, 18), + gT.get(tup, 19)) + } + def arity = 20 } - implicit def tuple20Converter[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R], - gS : TupleGetter[S], - gT : TupleGetter[T]): TupleConverter[Tuple20[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T]] = TupleConverter20(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR, gS, gT) + implicit def tuple20Converter[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R], + gS: TupleGetter[S], + gT: TupleGetter[T]): TupleConverter[Tuple20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T]] = TupleConverter20(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR, gS, gT) - case class TupleConverter21[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R], - gS : TupleGetter[S], - gT : TupleGetter[T], - gU : TupleGetter[U]) extends TupleConverter[Tuple21[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple21(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12), - gN.get(tup, 13), - gO.get(tup, 14), - gP.get(tup, 15), - gQ.get(tup, 16), - gR.get(tup, 17), - gS.get(tup, 18), - gT.get(tup, 19), - gU.get(tup, 20)) - } - def arity = 21 + case class TupleConverter21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R], + gS: TupleGetter[S], + gT: TupleGetter[T], + gU: TupleGetter[U]) extends TupleConverter[Tuple21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple21( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12), + gN.get(tup, 13), + gO.get(tup, 14), + gP.get(tup, 15), + gQ.get(tup, 16), + gR.get(tup, 17), + gS.get(tup, 18), + gT.get(tup, 19), + gU.get(tup, 20)) + } + def arity = 21 } - implicit def tuple21Converter[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R], - gS : TupleGetter[S], - gT : TupleGetter[T], - gU : TupleGetter[U]): TupleConverter[Tuple21[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U]] = TupleConverter21(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR, gS, gT, gU) + implicit def tuple21Converter[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R], + gS: TupleGetter[S], + gT: TupleGetter[T], + gU: TupleGetter[U]): TupleConverter[Tuple21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U]] = TupleConverter21(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR, gS, gT, gU) - case class TupleConverter22[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V]( gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R], - gS : TupleGetter[S], - gT : TupleGetter[T], - gU : TupleGetter[U], - gV : TupleGetter[V]) extends TupleConverter[Tuple22[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V]] { - def apply(te : TupleEntry) = { - val tup = te.getTuple - Tuple22(gA.get(tup, 0), - gB.get(tup, 1), - gC.get(tup, 2), - gD.get(tup, 3), - gE.get(tup, 4), - gF.get(tup, 5), - gG.get(tup, 6), - gH.get(tup, 7), - gI.get(tup, 8), - gJ.get(tup, 9), - gK.get(tup, 10), - gL.get(tup, 11), - gM.get(tup, 12), - gN.get(tup, 13), - gO.get(tup, 14), - gP.get(tup, 15), - gQ.get(tup, 16), - gR.get(tup, 17), - gS.get(tup, 18), - gT.get(tup, 19), - gU.get(tup, 20), - gV.get(tup, 21)) - } - def arity = 22 + case class TupleConverter22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V]( + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R], + gS: TupleGetter[S], + gT: TupleGetter[T], + gU: TupleGetter[U], + gV: TupleGetter[V]) extends TupleConverter[Tuple22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V]] { + def apply(te: TupleEntry) = { + val tup = te.getTuple + Tuple22( + gA.get(tup, 0), + gB.get(tup, 1), + gC.get(tup, 2), + gD.get(tup, 3), + gE.get(tup, 4), + gF.get(tup, 5), + gG.get(tup, 6), + gH.get(tup, 7), + gI.get(tup, 8), + gJ.get(tup, 9), + gK.get(tup, 10), + gL.get(tup, 11), + gM.get(tup, 12), + gN.get(tup, 13), + gO.get(tup, 14), + gP.get(tup, 15), + gQ.get(tup, 16), + gR.get(tup, 17), + gS.get(tup, 18), + gT.get(tup, 19), + gU.get(tup, 20), + gV.get(tup, 21)) + } + def arity = 22 } - implicit def tuple22Converter[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V](implicit - gA : TupleGetter[A], - gB : TupleGetter[B], - gC : TupleGetter[C], - gD : TupleGetter[D], - gE : TupleGetter[E], - gF : TupleGetter[F], - gG : TupleGetter[G], - gH : TupleGetter[H], - gI : TupleGetter[I], - gJ : TupleGetter[J], - gK : TupleGetter[K], - gL : TupleGetter[L], - gM : TupleGetter[M], - gN : TupleGetter[N], - gO : TupleGetter[O], - gP : TupleGetter[P], - gQ : TupleGetter[Q], - gR : TupleGetter[R], - gS : TupleGetter[S], - gT : TupleGetter[T], - gU : TupleGetter[U], - gV : TupleGetter[V]): TupleConverter[Tuple22[A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V]] = TupleConverter22(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR, gS, gT, gU, gV) + implicit def tuple22Converter[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V](implicit + gA: TupleGetter[A], + gB: TupleGetter[B], + gC: TupleGetter[C], + gD: TupleGetter[D], + gE: TupleGetter[E], + gF: TupleGetter[F], + gG: TupleGetter[G], + gH: TupleGetter[H], + gI: TupleGetter[I], + gJ: TupleGetter[J], + gK: TupleGetter[K], + gL: TupleGetter[L], + gM: TupleGetter[M], + gN: TupleGetter[N], + gO: TupleGetter[O], + gP: TupleGetter[P], + gQ: TupleGetter[Q], + gR: TupleGetter[R], + gS: TupleGetter[S], + gT: TupleGetter[T], + gU: TupleGetter[U], + gV: TupleGetter[V]): TupleConverter[Tuple22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V]] = TupleConverter22(gA, gB, gC, gD, gE, gF, gG, gH, gI, gJ, gK, gL, gM, gN, gO, gP, gQ, gR, gS, gT, gU, gV) } trait GeneratedTupleSetters extends LowPriorityTupleSetters { @@ -930,7 +971,7 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } implicit def tup1Setter[Z <: Tuple1[_]]: TupleSetter[Z] = TupleSetter1[Z]() - case class TupleSetter2[Z <: Tuple2[_,_]]() extends TupleSetter[Z] { + case class TupleSetter2[Z <: Tuple2[_, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(2) tup.set(0, arg._1) @@ -939,9 +980,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 2 } - implicit def tup2Setter[Z <: Tuple2[_,_]]: TupleSetter[Z] = TupleSetter2[Z]() + implicit def tup2Setter[Z <: Tuple2[_, _]]: TupleSetter[Z] = TupleSetter2[Z]() - case class TupleSetter3[Z <: Tuple3[_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter3[Z <: Tuple3[_, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(3) tup.set(0, arg._1) @@ -951,9 +992,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 3 } - implicit def tup3Setter[Z <: Tuple3[_,_,_]]: TupleSetter[Z] = TupleSetter3[Z]() + implicit def tup3Setter[Z <: Tuple3[_, _, _]]: TupleSetter[Z] = TupleSetter3[Z]() - case class TupleSetter4[Z <: Tuple4[_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter4[Z <: Tuple4[_, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(4) tup.set(0, arg._1) @@ -964,9 +1005,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 4 } - implicit def tup4Setter[Z <: Tuple4[_,_,_,_]]: TupleSetter[Z] = TupleSetter4[Z]() + implicit def tup4Setter[Z <: Tuple4[_, _, _, _]]: TupleSetter[Z] = TupleSetter4[Z]() - case class TupleSetter5[Z <: Tuple5[_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter5[Z <: Tuple5[_, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(5) tup.set(0, arg._1) @@ -978,9 +1019,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 5 } - implicit def tup5Setter[Z <: Tuple5[_,_,_,_,_]]: TupleSetter[Z] = TupleSetter5[Z]() + implicit def tup5Setter[Z <: Tuple5[_, _, _, _, _]]: TupleSetter[Z] = TupleSetter5[Z]() - case class TupleSetter6[Z <: Tuple6[_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter6[Z <: Tuple6[_, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(6) tup.set(0, arg._1) @@ -993,9 +1034,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 6 } - implicit def tup6Setter[Z <: Tuple6[_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter6[Z]() + implicit def tup6Setter[Z <: Tuple6[_, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter6[Z]() - case class TupleSetter7[Z <: Tuple7[_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter7[Z <: Tuple7[_, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(7) tup.set(0, arg._1) @@ -1009,9 +1050,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 7 } - implicit def tup7Setter[Z <: Tuple7[_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter7[Z]() + implicit def tup7Setter[Z <: Tuple7[_, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter7[Z]() - case class TupleSetter8[Z <: Tuple8[_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter8[Z <: Tuple8[_, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(8) tup.set(0, arg._1) @@ -1026,9 +1067,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 8 } - implicit def tup8Setter[Z <: Tuple8[_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter8[Z]() + implicit def tup8Setter[Z <: Tuple8[_, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter8[Z]() - case class TupleSetter9[Z <: Tuple9[_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter9[Z <: Tuple9[_, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(9) tup.set(0, arg._1) @@ -1044,9 +1085,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 9 } - implicit def tup9Setter[Z <: Tuple9[_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter9[Z]() + implicit def tup9Setter[Z <: Tuple9[_, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter9[Z]() - case class TupleSetter10[Z <: Tuple10[_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter10[Z <: Tuple10[_, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(10) tup.set(0, arg._1) @@ -1063,9 +1104,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 10 } - implicit def tup10Setter[Z <: Tuple10[_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter10[Z]() + implicit def tup10Setter[Z <: Tuple10[_, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter10[Z]() - case class TupleSetter11[Z <: Tuple11[_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter11[Z <: Tuple11[_, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(11) tup.set(0, arg._1) @@ -1083,9 +1124,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 11 } - implicit def tup11Setter[Z <: Tuple11[_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter11[Z]() + implicit def tup11Setter[Z <: Tuple11[_, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter11[Z]() - case class TupleSetter12[Z <: Tuple12[_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter12[Z <: Tuple12[_, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(12) tup.set(0, arg._1) @@ -1104,9 +1145,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 12 } - implicit def tup12Setter[Z <: Tuple12[_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter12[Z]() + implicit def tup12Setter[Z <: Tuple12[_, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter12[Z]() - case class TupleSetter13[Z <: Tuple13[_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter13[Z <: Tuple13[_, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(13) tup.set(0, arg._1) @@ -1126,9 +1167,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 13 } - implicit def tup13Setter[Z <: Tuple13[_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter13[Z]() + implicit def tup13Setter[Z <: Tuple13[_, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter13[Z]() - case class TupleSetter14[Z <: Tuple14[_,_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter14[Z <: Tuple14[_, _, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(14) tup.set(0, arg._1) @@ -1149,9 +1190,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 14 } - implicit def tup14Setter[Z <: Tuple14[_,_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter14[Z]() + implicit def tup14Setter[Z <: Tuple14[_, _, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter14[Z]() - case class TupleSetter15[Z <: Tuple15[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter15[Z <: Tuple15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(15) tup.set(0, arg._1) @@ -1173,9 +1214,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 15 } - implicit def tup15Setter[Z <: Tuple15[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter15[Z]() + implicit def tup15Setter[Z <: Tuple15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter15[Z]() - case class TupleSetter16[Z <: Tuple16[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter16[Z <: Tuple16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(16) tup.set(0, arg._1) @@ -1198,9 +1239,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 16 } - implicit def tup16Setter[Z <: Tuple16[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter16[Z]() + implicit def tup16Setter[Z <: Tuple16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter16[Z]() - case class TupleSetter17[Z <: Tuple17[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter17[Z <: Tuple17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(17) tup.set(0, arg._1) @@ -1224,9 +1265,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 17 } - implicit def tup17Setter[Z <: Tuple17[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter17[Z]() + implicit def tup17Setter[Z <: Tuple17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter17[Z]() - case class TupleSetter18[Z <: Tuple18[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter18[Z <: Tuple18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(18) tup.set(0, arg._1) @@ -1251,9 +1292,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 18 } - implicit def tup18Setter[Z <: Tuple18[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter18[Z]() + implicit def tup18Setter[Z <: Tuple18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter18[Z]() - case class TupleSetter19[Z <: Tuple19[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter19[Z <: Tuple19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(19) tup.set(0, arg._1) @@ -1279,9 +1320,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 19 } - implicit def tup19Setter[Z <: Tuple19[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter19[Z]() + implicit def tup19Setter[Z <: Tuple19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter19[Z]() - case class TupleSetter20[Z <: Tuple20[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter20[Z <: Tuple20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(20) tup.set(0, arg._1) @@ -1308,9 +1349,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 20 } - implicit def tup20Setter[Z <: Tuple20[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter20[Z]() + implicit def tup20Setter[Z <: Tuple20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter20[Z]() - case class TupleSetter21[Z <: Tuple21[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter21[Z <: Tuple21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(21) tup.set(0, arg._1) @@ -1338,9 +1379,9 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 21 } - implicit def tup21Setter[Z <: Tuple21[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter21[Z]() + implicit def tup21Setter[Z <: Tuple21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter21[Z]() - case class TupleSetter22[Z <: Tuple22[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]() extends TupleSetter[Z] { + case class TupleSetter22[Z <: Tuple22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]() extends TupleSetter[Z] { override def apply(arg: Z) = { val tup = Tuple.size(22) tup.set(0, arg._1) @@ -1369,6 +1410,6 @@ trait GeneratedTupleSetters extends LowPriorityTupleSetters { } override def arity = 22 } - implicit def tup22Setter[Z <: Tuple22[_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_]]: TupleSetter[Z] = TupleSetter22[Z]() + implicit def tup22Setter[Z <: Tuple22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]]: TupleSetter[Z] = TupleSetter22[Z]() } // end of autogenerated diff --git a/scalding-core/src/main/scala/com/twitter/scalding/GroupBuilder.scala b/scalding-core/src/main/scala/com/twitter/scalding/GroupBuilder.scala index 9677cad7c7..fc7f90bf8e 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/GroupBuilder.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/GroupBuilder.scala @@ -182,7 +182,8 @@ class GroupBuilder(val groupFields: Fields) extends FoldOperations[GroupBuilder] * The previous output goes into the reduce function on the left, like foldLeft, * so if your operation is faster for the accumulator to be on one side, be aware. */ - def mapReduceMap[T, X, U](fieldDef: (Fields, Fields))(mapfn: T => X)(redfn: (X, X) => X)(mapfn2: X => U)(implicit startConv: TupleConverter[T], + def mapReduceMap[T, X, U](fieldDef: (Fields, Fields))(mapfn: T => X)(redfn: (X, X) => X)(mapfn2: X => U)(implicit + startConv: TupleConverter[T], middleSetter: TupleSetter[X], middleConv: TupleConverter[X], endSetter: TupleSetter[U]): GroupBuilder = { @@ -198,7 +199,8 @@ class GroupBuilder(val groupFields: Fields) extends FoldOperations[GroupBuilder] projectFields = projectFields.map { Fields.merge(_, fromFields) } val ag = new MRMAggregator[T, X, U](mapfn, redfn, mapfn2, toFields, startConv, endSetter) val ev = (pipe => new Every(pipe, fromFields, ag)): Pipe => Every - assert(middleSetter.arity > 0, + assert( + middleSetter.arity > 0, "The middle arity must have definite size, try wrapping in scala.Tuple1 if you need a hack") // Create the required number of middlefields based on the arity of middleSetter val middleFields = strFields(ScalaRange(0, middleSetter.arity).map { i => getNextMiddlefield }) @@ -231,7 +233,8 @@ class GroupBuilder(val groupFields: Fields) extends FoldOperations[GroupBuilder] //Check arity conv.assertArityMatches(inFields) setter.assertArityMatches(outFields) - val b = new BufferOp[Unit, T, X]((), + val b = new BufferOp[Unit, T, X]( + (), (u: Unit, it: Iterator[T]) => mapfn(it), outFields, conv, setter) every(pipe => new Every(pipe, inFields, b, defaultMode(inFields, outFields))) } @@ -262,7 +265,8 @@ class GroupBuilder(val groupFields: Fields) extends FoldOperations[GroupBuilder] //Check arity conv.assertArityMatches(inFields) setter.assertArityMatches(outFields) - val b = new BufferOp[X, T, X](init, + val b = new BufferOp[X, T, X]( + init, // On scala 2.8, there is no scanLeft // On scala 2.9, their implementation creates an off-by-one bug with the unused fields (i: X, it: Iterator[T]) => new ScanLeftIterator(it, i, fn), @@ -306,7 +310,8 @@ class GroupBuilder(val groupFields: Fields) extends FoldOperations[GroupBuilder] case AggregateByMode => //There is some non-empty AggregateBy to do: val redlist = reds.get - val ag = new AggregateBy(name, + val ag = new AggregateBy( + name, maybeProjectedPipe, groupFields, spillThreshold.getOrElse(0), // cascading considers 0 to be the default diff --git a/scalding-core/src/main/scala/com/twitter/scalding/IntegralComparator.scala b/scalding-core/src/main/scala/com/twitter/scalding/IntegralComparator.scala index 9d1844b773..cc2803b276 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/IntegralComparator.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/IntegralComparator.scala @@ -26,7 +26,8 @@ import java.util.Comparator */ class IntegralComparator extends Comparator[AnyRef] with Hasher[AnyRef] with Serializable { - val integralTypes: Set[Class[_]] = Set(classOf[java.lang.Long], + val integralTypes: Set[Class[_]] = Set( + classOf[java.lang.Long], classOf[java.lang.Integer], classOf[java.lang.Short], classOf[java.lang.Byte]) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/Job.scala b/scalding-core/src/main/scala/com/twitter/scalding/Job.scala index 203278991a..3a2801a1f2 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/Job.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/Job.scala @@ -26,7 +26,7 @@ import com.twitter.scalding.typed.cascading_backend.CascadingBackend import org.apache.hadoop.io.serializer.{ Serialization => HSerialization } import scala.concurrent.{ Future, Promise } -import scala.util.{Try, Success, Failure} +import scala.util.{ Try, Success, Failure } import java.io.{ BufferedWriter, FileOutputStream, OutputStreamWriter } import java.util.{ List => JList } @@ -52,13 +52,14 @@ object Job { * and the Args contained in the Config. */ def makeJob[J <: Job](cls: Class[J]): Execution[J] = - Execution.getConfigMode.flatMap { case (conf, mode) => - // Now we need to allocate the job - Execution.from { - val argsWithMode = Mode.putMode(mode, conf.getArgs) - cls.getConstructor(classOf[Args]) - .newInstance(argsWithMode) - } + Execution.getConfigMode.flatMap { + case (conf, mode) => + // Now we need to allocate the job + Execution.from { + val argsWithMode = Mode.putMode(mode, conf.getArgs) + cls.getConstructor(classOf[Args]) + .newInstance(argsWithMode) + } } /** @@ -99,7 +100,7 @@ object Job { _ <- Execution.withConfig(ex)(_ => conf.setExecutionCleanupOnFinish(true)) _ <- nextJobEx } yield () - } + } } /** @@ -230,7 +231,8 @@ class Job(val args: Args) extends FieldConversions with java.io.Serializable { .setMapSideAggregationThreshold(defaultSpillThreshold) // This is setting a property for cascading/driven - AppProps.addApplicationFramework(null, + AppProps.addApplicationFramework( + null, String.format("scalding:%s", scaldingVersion)) val modeConf = mode match { @@ -259,7 +261,6 @@ class Job(val args: Args) extends FieldConversions with java.io.Serializable { ReferencedClassFinder.findReferencedClasses(getClass) } else Set.empty - /** * This is here so that Mappable.toIterator can find an implicit config */ @@ -523,7 +524,8 @@ abstract class ExecutionJob[+T](args: Args) extends Job(args) { final override def run = { val r = Config.tryFrom(config) .map { conf => - Await.result(execution.run(conf, mode)(concurrentExecutionContext), + Await.result( + execution.run(conf, mode)(concurrentExecutionContext), scala.concurrent.duration.Duration.Inf) } if (!resultPromise.tryComplete(r)) { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/JobTest.scala b/scalding-core/src/main/scala/com/twitter/scalding/JobTest.scala index fd09fac010..e5e11297b9 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/JobTest.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/JobTest.scala @@ -24,7 +24,6 @@ import cascading.stats.CascadingStats import org.apache.hadoop.conf.Configuration import org.apache.hadoop.mapred.JobConf - object JobTest { @deprecated(message = "Use the non-reflection based JobTest apply methods", since = "0.16.1") @@ -49,8 +48,7 @@ object JobTest { // We have to memoize to return the same buffer each time. private case class MemoizedSourceFn[T]( fn: Source => Option[Iterable[T]], - setter: TupleSetter[T] - ) extends (Source => Option[mutable.Buffer[Tuple]]) { + setter: TupleSetter[T]) extends (Source => Option[mutable.Buffer[Tuple]]) { private val memo = mutable.Map[Source, Option[mutable.Buffer[Tuple]]]() private val lock = new Object() diff --git a/scalding-core/src/main/scala/com/twitter/scalding/JoinAlgorithms.scala b/scalding-core/src/main/scala/com/twitter/scalding/JoinAlgorithms.scala index bcc6e29a70..fb57a0ade5 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/JoinAlgorithms.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/JoinAlgorithms.scala @@ -257,7 +257,8 @@ trait JoinAlgorithms { * You can only use an InnerJoin or a LeftJoin with a leftReplication of 1 * (or a RightJoin with a rightReplication of 1) when doing a blockJoin. */ - def blockJoinWithSmaller(fs: (Fields, Fields), + def blockJoinWithSmaller( + fs: (Fields, Fields), otherPipe: Pipe, rightReplication: Int = 1, leftReplication: Int = 1, joiner: Joiner = new InnerJoin, reducers: Int = -1): Pipe = { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/Operations.scala b/scalding-core/src/main/scala/com/twitter/scalding/Operations.scala index 2f1337f2ab..4c5aa8e99a 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/Operations.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/Operations.scala @@ -593,7 +593,8 @@ package com.twitter.scalding { /** * MapReduceMapBy Class */ - class MRMBy[T, X, U](arguments: Fields, + class MRMBy[T, X, U]( + arguments: Fields, middleFields: Fields, declaredFields: Fields, mfn: T => X, diff --git a/scalding-core/src/main/scala/com/twitter/scalding/PipeDebug.scala b/scalding-core/src/main/scala/com/twitter/scalding/PipeDebug.scala index 3c5ab7347d..b6e847adde 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/PipeDebug.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/PipeDebug.scala @@ -26,7 +26,8 @@ import cascading.operation.Debug.Output * This is based on work by: https://github.com/granthenke * https://github.com/twitter/scalding/pull/559 */ -case class PipeDebug(output: Output = Output.STDERR, +case class PipeDebug( + output: Output = Output.STDERR, prefix: String = null, printFieldsEvery: Option[Int] = None, printTuplesEvery: Int = 1) { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/ReduceOperations.scala b/scalding-core/src/main/scala/com/twitter/scalding/ReduceOperations.scala index 7645cac59c..6c4cea2c8a 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/ReduceOperations.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/ReduceOperations.scala @@ -55,7 +55,8 @@ trait ReduceOperations[+Self <: ReduceOperations[Self]] extends java.io.Serializ * * Assumed to be a commutative operation. If you don't want that, use .forceToReducers */ - def mapReduceMap[T, X, U](fieldDef: (Fields, Fields))(mapfn: T => X)(redfn: (X, X) => X)(mapfn2: X => U)(implicit startConv: TupleConverter[T], + def mapReduceMap[T, X, U](fieldDef: (Fields, Fields))(mapfn: T => X)(redfn: (X, X) => X)(mapfn2: X => U)(implicit + startConv: TupleConverter[T], middleSetter: TupleSetter[X], middleConv: TupleConverter[X], endSetter: TupleSetter[U]): Self @@ -65,7 +66,8 @@ trait ReduceOperations[+Self <: ReduceOperations[Self]] extends java.io.Serializ ///////////////////////////////////////// /** Pretty much a synonym for mapReduceMap with the methods collected into a trait. */ - def aggregate[A, B, C](fieldDef: (Fields, Fields))(ag: Aggregator[A, B, C])(implicit startConv: TupleConverter[A], + def aggregate[A, B, C](fieldDef: (Fields, Fields))(ag: Aggregator[A, B, C])(implicit + startConv: TupleConverter[A], middleSetter: TupleSetter[B], middleConv: TupleConverter[B], endSetter: TupleSetter[C]): Self = @@ -220,7 +222,8 @@ trait ReduceOperations[+Self <: ReduceOperations[Self]] extends java.io.Serializ } { fn(_) }(conv, midset, midconv, setter) } - def mapPlusMap[T, X, U](fieldDef: (Fields, Fields))(mapfn: T => X)(mapfn2: X => U)(implicit startConv: TupleConverter[T], + def mapPlusMap[T, X, U](fieldDef: (Fields, Fields))(mapfn: T => X)(mapfn2: X => U)(implicit + startConv: TupleConverter[T], middleSetter: TupleSetter[X], middleConv: TupleConverter[X], endSetter: TupleSetter[U], @@ -286,7 +289,8 @@ trait ReduceOperations[+Self <: ReduceOperations[Self]] extends java.io.Serializ mapReduceMap[T, T, T](fieldDef)({ t => t })(fn)({ t => t })(conv, setter, conv, setter) } //Same as reduce(f->f) - def reduce[T](fieldDef: Symbol*)(fn: (T, T) => T)(implicit setter: TupleSetter[T], + def reduce[T](fieldDef: Symbol*)(fn: (T, T) => T)(implicit + setter: TupleSetter[T], conv: TupleConverter[T]): Self = { reduce(fieldDef -> fieldDef)(fn)(setter, conv) } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/RichPipe.scala b/scalding-core/src/main/scala/com/twitter/scalding/RichPipe.scala index 3fdaa253b2..a630df5951 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/RichPipe.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/RichPipe.scala @@ -445,9 +445,10 @@ class RichPipe(val pipe: Pipe) extends java.io.Serializable with JoinAlgorithms */ def partition[A, R](fs: (Fields, Fields))(fn: (A) => R)( builder: GroupBuilder => GroupBuilder)( - implicit conv: TupleConverter[A], - ord: Ordering[R], - rset: TupleSetter[R]): Pipe = { + implicit + conv: TupleConverter[A], + ord: Ordering[R], + rset: TupleSetter[R]): Pipe = { val (fromFields, toFields) = fs conv.assertArityMatches(fromFields) rset.assertArityMatches(toFields) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TestTapFactory.scala b/scalding-core/src/main/scala/com/twitter/scalding/TestTapFactory.scala index 5ef5dd8209..3691f72cdd 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TestTapFactory.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TestTapFactory.scala @@ -45,7 +45,8 @@ object TestTapFactory extends Serializable { override def sinkFields: Fields = fields } def apply[A, B](src: Source, scheme: Scheme[JobConf, RecordReader[_, _], OutputCollector[_, _], A, B]): TestTapFactory = apply(src, scheme, SinkMode.REPLACE) - def apply[A, B](src: Source, + def apply[A, B]( + src: Source, scheme: Scheme[JobConf, RecordReader[_, _], OutputCollector[_, _], A, B], sinkMode: SinkMode): TestTapFactory = new TestTapFactory(src, sinkMode) { override def hdfsScheme = Some(scheme) } } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TimePathedSource.scala b/scalding-core/src/main/scala/com/twitter/scalding/TimePathedSource.scala index 749f7fb4bd..e5a3c75927 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TimePathedSource.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TimePathedSource.scala @@ -124,7 +124,8 @@ abstract class TimeSeqPathedSource(val patterns: Seq[String], val dateRange: Dat * THIS MEANS YOU MUST END WITH A / followed by * to match a file * For writing, we write to the directory specified by the END time. */ -abstract class TimePathedSource(val pattern: String, +abstract class TimePathedSource( + val pattern: String, dateRange: DateRange, tz: TimeZone) extends TimeSeqPathedSource(Seq(pattern), dateRange, tz) { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TypedDelimited.scala b/scalding-core/src/main/scala/com/twitter/scalding/TypedDelimited.scala index ed5b9fe1b2..25dd8394e0 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TypedDelimited.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TypedDelimited.scala @@ -126,12 +126,13 @@ trait TypedDelimited[T] extends DelimitedScheme } @deprecated("Use FixedTypedText instead", "2015-07") -class FixedPathTypedDelimited[T](p: Seq[String], +class FixedPathTypedDelimited[T]( + p: Seq[String], override val fields: Fields = Fields.ALL, override val skipHeader: Boolean = false, override val writeHeader: Boolean = false, override val separator: String = "\t")(implicit override val mf: Manifest[T], override val conv: TupleConverter[T], - override val tset: TupleSetter[T]) extends FixedPathSource(p: _*) + override val tset: TupleSetter[T]) extends FixedPathSource(p: _*) with TypedDelimited[T] { override lazy val toString: String = "FixedPathTypedDelimited" + diff --git a/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/CaseClassBasedSetterImpl.scala b/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/CaseClassBasedSetterImpl.scala index 7202be2096..9e80de9e30 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/CaseClassBasedSetterImpl.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/CaseClassBasedSetterImpl.scala @@ -39,7 +39,8 @@ object CaseClassBasedSetterImpl { def columns = 1 def setTree(value: Tree, offset: Int) = fsetter.from(c)(tpe, offset, container, value) match { case Success(tree) => tree - case Failure(e) => c.abort(c.enclosingPosition, + case Failure(e) => c.abort( + c.enclosingPosition, s"Case class ${T} is supported. Error on $tpe, ${e.getMessage}") } } @@ -101,7 +102,8 @@ object CaseClassBasedSetterImpl { case tpe if allowUnknownTypes => DefaultSetter case _ => - c.abort(c.enclosingPosition, + c.abort( + c.enclosingPosition, s"Case class ${T.tpe} is not supported at type: $outerType") } } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/TupleConverterImpl.scala b/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/TupleConverterImpl.scala index e8b43768c5..66db3fbbd0 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/TupleConverterImpl.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/TupleConverterImpl.scala @@ -109,7 +109,8 @@ object TupleConverterImpl { case tpe if allowUnknownTypes => PrimitiveBuilder(idx => q"""t.getObject(${idx}).asInstanceOf[$tpe]""") case tpe => - c.abort(c.enclosingPosition, + c.abort( + c.enclosingPosition, s"${T.tpe} is not pure primitives, Option of a primitive, nested case classes when looking at type ${tpe}") } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/TypeDescriptorProviderImpl.scala b/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/TypeDescriptorProviderImpl.scala index 349cfe70f3..d0c4bd4abd 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/TypeDescriptorProviderImpl.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/macros/impl/TypeDescriptorProviderImpl.scala @@ -93,7 +93,8 @@ object TypeDescriptorProviderImpl { def isTuple[T](c: Context)(implicit T: c.WeakTypeTag[T]): Boolean = { import c.universe._ - val tupleTypes = List(typeOf[Tuple1[Any]], + val tupleTypes = List( + typeOf[Tuple1[Any]], typeOf[Tuple2[Any, Any]], typeOf[Tuple3[Any, Any, Any]], typeOf[Tuple4[Any, Any, Any, Any]], diff --git a/scalding-core/src/main/scala/com/twitter/scalding/mathematics/Matrix.scala b/scalding-core/src/main/scala/com/twitter/scalding/mathematics/Matrix.scala index b20598937c..be7c60172c 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/mathematics/Matrix.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/mathematics/Matrix.scala @@ -123,7 +123,8 @@ class MatrixPipeExtensions(pipe: Pipe) { * This is the enrichment pattern on Mappable[T] for converting to Matrix types */ class MatrixMappableExtensions[T](mappable: Mappable[T])(implicit fd: FlowDef, mode: Mode) { - def toMatrix[Row, Col, Val](implicit ev: <:<[T, (Row, Col, Val)], + def toMatrix[Row, Col, Val](implicit + ev: <:<[T, (Row, Col, Val)], setter: TupleSetter[(Row, Col, Val)]): Matrix[Row, Col, Val] = mapToMatrix { _.asInstanceOf[(Row, Col, Val)] } @@ -633,7 +634,8 @@ class Matrix[RowT, ColT, ValT](val rowSym: Symbol, val colSym: Symbol, val valSy // TODO optimize the number of reducers val zipped = cleanUpZipJoin[ValU](getField(newRFields, 2), pairMonoid) { pipe - .joinWithSmaller((rowSym, colSym) -> + .joinWithSmaller( + (rowSym, colSym) -> (getField(newRFields, 0).append(getField(newRFields, 1))), newRPipe, new OuterJoin) .thenDo{ p: RichPipe => cleanUpIndexZipJoin(rowSym.append(getField(newRFields, 0)), p) } @@ -654,7 +656,8 @@ class Matrix[RowT, ColT, ValT](val rowSym: Symbol, val colSym: Symbol, val valSy val filterC = '___filterC___ val filterV = '___filterV___ - val joined = pipe.joinWithSmaller((rowSym, colSym) -> (filterR, filterC), + val joined = pipe.joinWithSmaller( + (rowSym, colSym) -> (filterR, filterC), that.pipe.rename((that.rowSym, that.colSym, that.valSym) -> (filterR, filterC, filterV)), new LeftJoin) val filtered = joined.filter(filterV){ x: ValU => null == x } new Matrix[RowT, ColT, ValT](rowSym, colSym, valSym, filtered.project(rowSym, colSym, valSym)) @@ -668,7 +671,8 @@ class Matrix[RowT, ColT, ValT](val rowSym: Symbol, val colSym: Symbol, val valSy val keepC = '___keepC___ val keepV = '___keepV___ - val joined = pipe.joinWithSmaller((rowSym, colSym) -> (keepR, keepC), + val joined = pipe.joinWithSmaller( + (rowSym, colSym) -> (keepR, keepC), that.pipe.rename((that.rowSym, that.colSym, that.valSym) -> (keepR, keepC, keepV))) new Matrix[RowT, ColT, ValT](rowSym, colSym, valSym, joined.project(rowSym, colSym, valSym)) } @@ -739,7 +743,8 @@ class Scalar[ValT](val valSym: Symbol, inPipe: Pipe) extends WrappedPipe with ja } } -class DiagonalMatrix[IdxT, ValT](val idxSym: Symbol, +class DiagonalMatrix[IdxT, ValT]( + val idxSym: Symbol, val valSym: Symbol, inPipe: Pipe, val sizeHint: SizeHint = FiniteHint(1L, -1L)) extends WrappedPipe with java.io.Serializable { @@ -840,14 +845,16 @@ class RowVector[ColT, ValT](val colS: Symbol, val valS: Symbol, inPipe: Pipe, va def L0Normalize(implicit ev: =:=[ValT, Double]): RowVector[ColT, ValT] = { val normedMatrix = this.toMatrix(0).rowL0Normalize - new RowVector(normedMatrix.colSym, + new RowVector( + normedMatrix.colSym, normedMatrix.valSym, normedMatrix.pipe.project(normedMatrix.colSym, normedMatrix.valSym)) } def L1Normalize(implicit ev: =:=[ValT, Double]): RowVector[ColT, ValT] = { val normedMatrix = this.toMatrix(0).rowL1Normalize - new RowVector(normedMatrix.colSym, + new RowVector( + normedMatrix.colSym, normedMatrix.valSym, normedMatrix.pipe.project(normedMatrix.colSym, normedMatrix.valSym)) } @@ -960,14 +967,16 @@ class ColVector[RowT, ValT](val rowS: Symbol, val valS: Symbol, inPipe: Pipe, va def L0Normalize(implicit ev: =:=[ValT, Double]): ColVector[RowT, ValT] = { val normedMatrix = this.toMatrix(0).colL0Normalize - new ColVector(normedMatrix.rowSym, + new ColVector( + normedMatrix.rowSym, normedMatrix.valSym, normedMatrix.pipe.project(normedMatrix.rowSym, normedMatrix.valSym)) } def L1Normalize(implicit ev: =:=[ValT, Double]): ColVector[RowT, ValT] = { val normedMatrix = this.toMatrix(0).colL1Normalize - new ColVector(normedMatrix.rowSym, + new ColVector( + normedMatrix.rowSym, normedMatrix.valSym, normedMatrix.pipe.project(normedMatrix.rowSym, normedMatrix.valSym)) } @@ -1023,7 +1032,8 @@ class ColVector[RowT, ValT](val rowS: Symbol, val valS: Symbol, inPipe: Pipe, va * For example, grouping users by countries and calculating products only between users from the same country */ class BlockMatrix[RowT, GroupT, ColT, ValT](private val mat: Matrix[RowT, GroupT, Map[ColT, ValT]]) { - def dotProd[RowT2](that: BlockMatrix[GroupT, RowT2, ColT, ValT])(implicit prod: MatrixProduct[Matrix[RowT, GroupT, Map[ColT, ValT]], Matrix[GroupT, RowT2, Map[ColT, ValT]], Matrix[RowT, RowT2, Map[ColT, ValT]]], + def dotProd[RowT2](that: BlockMatrix[GroupT, RowT2, ColT, ValT])(implicit + prod: MatrixProduct[Matrix[RowT, GroupT, Map[ColT, ValT]], Matrix[GroupT, RowT2, Map[ColT, ValT]], Matrix[RowT, RowT2, Map[ColT, ValT]]], mon: Monoid[ValT]): Matrix[RowT, RowT2, ValT] = { prod(mat, that.mat).mapValues(_.values.foldLeft(mon.zero)(mon.plus)) } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/mathematics/Matrix2.scala b/scalding-core/src/main/scala/com/twitter/scalding/mathematics/Matrix2.scala index fd3fe49daa..795e5603fa 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/mathematics/Matrix2.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/mathematics/Matrix2.scala @@ -96,7 +96,8 @@ sealed trait Matrix2[R, C, V] extends Serializable { * BloomFilters or CountMinSketch. * TODO This is a special kind of product that could be optimized like Product is */ - def propagate[C2, VecV](vec: Matrix2[C, C2, VecV])(implicit ev: =:=[V, Boolean], + def propagate[C2, VecV](vec: Matrix2[C, C2, VecV])(implicit + ev: =:=[V, Boolean], mon: Monoid[VecV], mj: MatrixJoiner2): Matrix2[R, C2, VecV] = { @@ -202,7 +203,8 @@ object MatrixJoiner2 { // comment this out to verify we are not hiding the user's suppled values implicit def default: MatrixJoiner2 = new DefaultMatrixJoiner(10000L) - def join[R, C, V, C2, V2](left: Matrix2[R, C, V], + def join[R, C, V, C2, V2]( + left: Matrix2[R, C, V], right: Matrix2[C, C2, V2])(implicit mj: MatrixJoiner2): TypedPipe[(C, ((R, V), (C2, V2)))] = mj.join(left, right) } @@ -212,7 +214,8 @@ object MatrixJoiner2 { * otherwise, if one is much smaller than the other, we use a hash join */ class DefaultMatrixJoiner(sizeRatioThreshold: Long) extends MatrixJoiner2 { - def join[R, C, V, C2, V2](left: Matrix2[R, C, V], + def join[R, C, V, C2, V2]( + left: Matrix2[R, C, V], right: Matrix2[C, C2, V2]): TypedPipe[(C, ((R, V), (C2, V2)))] = { implicit val cOrd: Ordering[C] = left.colOrd val one = left.toTypedPipe.map { case (r, c, v) => (c, (r, v)) }.group @@ -267,7 +270,8 @@ final case class OneR[C, V](implicit override val colOrd: Ordering[C]) extends M * @param ring * @param expressions a HashMap of common subtrees; None if possibly not optimal (did not go through optimize), Some(...) with a HashMap that was created in optimize */ -final case class Product[R, C, C2, V](left: Matrix2[R, C, V], +final case class Product[R, C, C2, V]( + left: Matrix2[R, C, V], right: Matrix2[C, C2, V], ring: Ring[V], expressions: Option[Map[Matrix2[R, C2, V], TypedPipe[(R, C2, V)]]] = None)(implicit val joiner: MatrixJoiner2) extends Matrix2[R, C2, V] { @@ -447,7 +451,8 @@ final case class Sum[R, C, V](left: Matrix2[R, C, V], right: Matrix2[R, C, V], m }.reduce(_ ++ _).sum) } -final case class HadamardProduct[R, C, V](left: Matrix2[R, C, V], +final case class HadamardProduct[R, C, V]( + left: Matrix2[R, C, V], right: Matrix2[R, C, V], ring: Ring[V]) extends Matrix2[R, C, V] { @@ -480,7 +485,8 @@ final case class HadamardProduct[R, C, V](left: Matrix2[R, C, V], implicit def withOrderedSerialization: Ordering[(R, C)] = OrderedSerialization2.maybeOrderedSerialization2(rowOrd, colOrd) } -final case class MatrixLiteral[R, C, V](override val toTypedPipe: TypedPipe[(R, C, V)], +final case class MatrixLiteral[R, C, V]( + override val toTypedPipe: TypedPipe[(R, C, V)], override val sizeHint: SizeHint)(implicit override val rowOrd: Ordering[R], override val colOrd: Ordering[C]) extends Matrix2[R, C, V] { @@ -580,7 +586,8 @@ object Matrix2 { def apply[R: Ordering, C: Ordering, V](t: TypedPipe[(R, C, V)], hint: SizeHint): Matrix2[R, C, V] = MatrixLiteral(t, hint) - def read[R, C, V](t: TypedSource[(R, C, V)], + def read[R, C, V]( + t: TypedSource[(R, C, V)], hint: SizeHint)(implicit ordr: Ordering[R], ordc: Ordering[C]): Matrix2[R, C, V] = MatrixLiteral(TypedPipe.from(t), hint) @@ -674,7 +681,8 @@ object Matrix2 { val (lastRChain, lastCost2, ringR, joinerR) = optimizeBasicBlocks(right) val (cost1, newLeft) = optimizeProductChain(lastLChain.toIndexedSeq, pair(ringL, joinerL)) // linter:ignore val (cost2, newRight) = optimizeProductChain(lastRChain.toIndexedSeq, pair(ringR, joinerR)) // linter:ignore - (List(Sum(newLeft, newRight, mon)), + ( + List(Sum(newLeft, newRight, mon)), lastCost1 + lastCost2 + cost1 + cost2, ringL.orElse(ringR), joinerL.orElse(joinerR)) @@ -684,7 +692,8 @@ object Matrix2 { val (lastRChain, lastCost2, ringR, joinerR) = optimizeBasicBlocks(right) val (cost1, newLeft) = optimizeProductChain(lastLChain.toIndexedSeq, pair(ringL, joinerL)) // linter:ignore val (cost2, newRight) = optimizeProductChain(lastRChain.toIndexedSeq, pair(ringR, joinerR)) // linter:ignore - (List(HadamardProduct(newLeft, newRight, ring)), + ( + List(HadamardProduct(newLeft, newRight, ring)), lastCost1 + lastCost2 + cost1 + cost2, ringL.orElse(ringR), joinerL.orElse(joinerR)) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/mathematics/TypedSimilarity.scala b/scalding-core/src/main/scala/com/twitter/scalding/mathematics/TypedSimilarity.scala index 42da7c6bbc..21092b7c27 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/mathematics/TypedSimilarity.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/mathematics/TypedSimilarity.scala @@ -95,7 +95,8 @@ trait TypedSimilarity[N, E, S] extends Serializable { * The Edge.from nodes in the result will all satisfy smallpred, and the Edge.to will * all satisfy bigpred. It is more efficient if you keep the smallpred set smaller. */ - def apply(g: TypedPipe[Edge[N, E]], + def apply( + g: TypedPipe[Edge[N, E]], smallpred: N => Boolean, bigpred: N => Boolean): TypedPipe[Edge[N, S]] // Do similarity on all the nodes @@ -115,7 +116,8 @@ object TypedSimilarity extends Serializable { // key: document, // value: (word, documentsWithWord) // return: Edge of similarity between words measured by documents - def exactSetSimilarity[N: Ordering](g: Grouped[N, (N, Int)], + def exactSetSimilarity[N: Ordering]( + g: Grouped[N, (N, Int)], smallpred: N => Boolean, bigpred: N => Boolean): TypedPipe[Edge[N, SetSimilarity]] = /* E_{ij} = 1 if document -> word exists * (E^T E)_ij = # of shared documents of i,j @@ -146,7 +148,8 @@ object TypedSimilarity extends Serializable { * return: Edge of similarity between words measured by documents * See: https://arxiv.org/pdf/1206.2082v2.pdf */ - def discoCosineSimilarity[N: Ordering](smallG: Grouped[N, (N, Int)], + def discoCosineSimilarity[N: Ordering]( + smallG: Grouped[N, (N, Int)], bigG: Grouped[N, (N, Int)], oversample: Double): TypedPipe[Edge[N, Double]] = { // 1) make rnd lazy due to serialization, // 2) fix seed so that map-reduce speculative execution does not give inconsistent results. @@ -183,7 +186,8 @@ object TypedSimilarity extends Serializable { * return: Edge of similarity between words measured by documents * See: https://stanford.edu/~rezab/papers/dimsum.pdf */ - def dimsumCosineSimilarity[N: Ordering](smallG: Grouped[N, (N, Double, Double)], + def dimsumCosineSimilarity[N: Ordering]( + smallG: Grouped[N, (N, Double, Double)], bigG: Grouped[N, (N, Double, Double)], oversample: Double): TypedPipe[Edge[N, Double]] = { lazy val rnd = new scala.util.Random(1024) maybeWithReducers(smallG.cogroup(bigG) { (n: N, leftit: Iterator[(N, Double, Double)], rightit: Iterable[(N, Double, Double)]) => @@ -219,7 +223,8 @@ object TypedSimilarity extends Serializable { */ class ExactInCosine[N](reducers: Int = -1)(implicit override val nodeOrdering: Ordering[N]) extends TypedSimilarity[N, InDegree, Double] { - def apply(graph: TypedPipe[Edge[N, InDegree]], + def apply( + graph: TypedPipe[Edge[N, InDegree]], smallpred: N => Boolean, bigpred: N => Boolean): TypedPipe[Edge[N, Double]] = { val groupedOnSrc = graph .filter { e => smallpred(e.to) || bigpred(e.to) } @@ -244,7 +249,8 @@ class DiscoInCosine[N](minCos: Double, delta: Double, boundedProb: Double, reduc // boundedProb ~ exp(-p delta^2 / 2) private val oversample = (-2.0 * scala.math.log(boundedProb) / (delta * delta)) / minCos - def apply(graph: TypedPipe[Edge[N, InDegree]], + def apply( + graph: TypedPipe[Edge[N, InDegree]], smallpred: N => Boolean, bigpred: N => Boolean): TypedPipe[Edge[N, Double]] = { val bigGroupedOnSrc = graph .filter { e => bigpred(e.to) } @@ -268,7 +274,8 @@ class DimsumInCosine[N](minCos: Double, delta: Double, boundedProb: Double, redu // boundedProb ~ exp(-p delta^2 / 2) private val oversample = (-2.0 * scala.math.log(boundedProb) / (delta * delta)) / minCos - def apply(graph: TypedPipe[Edge[N, (Weight, L2Norm)]], + def apply( + graph: TypedPipe[Edge[N, (Weight, L2Norm)]], smallpred: N => Boolean, bigpred: N => Boolean): TypedPipe[Edge[N, Double]] = { val bigGroupedOnSrc = graph .filter { e => bigpred(e.to) } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/serialization/KryoHadoop.scala b/scalding-core/src/main/scala/com/twitter/scalding/serialization/KryoHadoop.scala index 1d36f8cc9f..52a648cb21 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/serialization/KryoHadoop.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/serialization/KryoHadoop.scala @@ -66,15 +66,20 @@ class KryoHadoop(@transient config: Config) extends KryoInstantiator { * AdaptiveVector is IndexedSeq, which picks up the chill IndexedSeq serializer * (which is its own bug), force using the fields serializer here */ - newK.register(classOf[com.twitter.algebird.DenseVector[_]], - new FieldSerializer[com.twitter.algebird.DenseVector[_]](newK, + newK.register( + classOf[com.twitter.algebird.DenseVector[_]], + new FieldSerializer[com.twitter.algebird.DenseVector[_]]( + newK, classOf[com.twitter.algebird.DenseVector[_]])) - newK.register(classOf[com.twitter.algebird.SparseVector[_]], - new FieldSerializer[com.twitter.algebird.SparseVector[_]](newK, + newK.register( + classOf[com.twitter.algebird.SparseVector[_]], + new FieldSerializer[com.twitter.algebird.SparseVector[_]]( + newK, classOf[com.twitter.algebird.SparseVector[_]])) - newK.addDefaultSerializer(classOf[com.twitter.algebird.AdaptiveVector[_]], + newK.addDefaultSerializer( + classOf[com.twitter.algebird.AdaptiveVector[_]], classOf[FieldSerializer[_]]) /** diff --git a/scalding-core/src/main/scala/com/twitter/scalding/source/TypedText.scala b/scalding-core/src/main/scala/com/twitter/scalding/source/TypedText.scala index 8f5e2da6c4..9dab5ccccc 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/source/TypedText.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/source/TypedText.scala @@ -33,46 +33,46 @@ object TypedText { new TimePathTypedText[T](sep, prefix + TimePathedSource.YEAR_MONTH_DAY_HOUR + "/*") } - def hourlyTsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = + def hourlyTsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = hourly(TAB, prefix) - def hourlyOsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = + def hourlyOsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = hourly(ONE, prefix) - def hourlyCsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = + def hourlyCsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = hourly(COMMA, prefix) private def daily[T]( - sep: TypedSep, prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = { + sep: TypedSep, prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = { require(prefix.last != '/', "prefix should not include trailing /") new TimePathTypedText[T](sep, prefix + TimePathedSource.YEAR_MONTH_DAY + "/*") } - def dailyTsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = + def dailyTsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = daily(TAB, prefix) - def dailyOsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = + def dailyOsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = daily(ONE, prefix) - def dailyCsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = + def dailyCsv[T](prefix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = daily(COMMA, prefix) private def dailyPrefixSuffix[T]( - sep: TypedSep, - prefix: String, - suffix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = { + sep: TypedSep, + prefix: String, + suffix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = { require(prefix.last != '/', "prefix should not include trailing /") require(suffix.head == '/', "suffix should include a preceding /") new TimePathTypedText[T](sep, prefix + TimePathedSource.YEAR_MONTH_DAY + suffix + "/*") } - def dailyPrefixSuffixTsv[T](prefix: String, suffix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = + def dailyPrefixSuffixTsv[T](prefix: String, suffix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = dailyPrefixSuffix(TAB, prefix, suffix) - def dailyPrefixSuffixOsv[T](prefix: String, suffix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = + def dailyPrefixSuffixOsv[T](prefix: String, suffix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = dailyPrefixSuffix(ONE, prefix, suffix) - def dailyPrefixSuffixCsv[T](prefix: String, suffix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = + def dailyPrefixSuffixCsv[T](prefix: String, suffix: String)(implicit dr: DateRange, td: TypeDescriptor[T]): TypedTextDelimited[T] = dailyPrefixSuffix(COMMA, prefix, suffix) } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/Grouped.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/Grouped.scala index d8a8eb0940..47e450c95f 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/Grouped.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/Grouped.scala @@ -304,18 +304,18 @@ sealed trait HashJoinable[K, +V] extends CoGroupable[K, V] with KeyedPipe[K] { object HashJoinable extends Serializable { def toReduceStep[A, B](hj: HashJoinable[A, B]): ReduceStep[A, _, _ <: B] = hj match { - case step@IdentityReduce(_, _, _, _, _) => step - case step@UnsortedIdentityReduce(_, _, _, _, _) => step - case step@IteratorMappedReduce(_, _, _, _, _) => step + case step @ IdentityReduce(_, _, _, _, _) => step + case step @ UnsortedIdentityReduce(_, _, _, _, _) => step + case step @ IteratorMappedReduce(_, _, _, _, _) => step } def filterKeys[A, B](hj: HashJoinable[A, B], fn: A => Boolean): HashJoinable[A, B] = hj match { - case step@IdentityReduce(_, _, _, _, _) => + case step @ IdentityReduce(_, _, _, _, _) => step.copy(mapped = TypedPipe.FilterKeys(step.mapped, fn)) - case step@UnsortedIdentityReduce(_, _, _, _, _) => + case step @ UnsortedIdentityReduce(_, _, _, _, _) => step.copy(mapped = TypedPipe.FilterKeys(step.mapped, fn)) - case step@IteratorMappedReduce(_, _, _, _, _) => + case step @ IteratorMappedReduce(_, _, _, _, _) => step.copy(mapped = TypedPipe.FilterKeys(step.mapped, fn)) } } @@ -365,7 +365,7 @@ object Grouped extends Serializable { def addEmptyGuard[K, V1, V2](fn: (K, Iterator[V1]) => Iterator[V2]): (K, Iterator[V1]) => Iterator[V2] = fn match { - case alreadyGuarded@EmptyGuard(_) => alreadyGuarded + case alreadyGuarded @ EmptyGuard(_) => alreadyGuarded case ami if CoGroupable.atMostInputSizeFn(ami) => ami // already safe case needGuard => EmptyGuard(needGuard) } @@ -466,7 +466,8 @@ object ReduceStep extends Serializable { val step = step0.evidence.subst[IR](step0) val revEv = step0.evidence.reverse val res = - IdentityReduce[A, C, C](step.keyOrdering, + IdentityReduce[A, C, C]( + step.keyOrdering, step0.evidence.subst[In](input), step.reducers, step.descriptions, @@ -478,7 +479,8 @@ object ReduceStep extends Serializable { val step = step0.evidence.subst[IR](step0) val revEv = step0.evidence.reverse val res = - UnsortedIdentityReduce[A, C, C](step.keyOrdering, + UnsortedIdentityReduce[A, C, C]( + step.keyOrdering, step0.evidence.subst[In](input), step.reducers, step.descriptions, @@ -490,7 +492,8 @@ object ReduceStep extends Serializable { val step = step0.evidence.subst[IVSR](step0) val revEv = step0.evidence.reverse val res = - IdentityValueSortedReduce[A, C, C](step.keyOrdering, + IdentityValueSortedReduce[A, C, C]( + step.keyOrdering, step0.evidence.subst[In](input), step.valueSort, step.reducers, @@ -499,7 +502,8 @@ object ReduceStep extends Serializable { // Put the type back to what scala expects ReduceStep[A, B, C] revEv.subst[Res](res) case step @ ValueSortedReduce(_, _, _, _, _, _) => - ValueSortedReduce[A, B, C](step.keyOrdering, + ValueSortedReduce[A, B, C]( + step.keyOrdering, input, step.valueSort, step.reduceFn, step.reducers, step.descriptions) case step @ IteratorMappedReduce(_, _, _, _, _) => def go(imr: IteratorMappedReduce[A, B, C]): IteratorMappedReduce[A, B, C] = diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/KeyedList.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/KeyedList.scala index cf9f35775e..8ef69e267a 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/KeyedList.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/KeyedList.scala @@ -355,8 +355,8 @@ trait KeyedListLike[K, +T, +This[K, +T] <: KeyedListLike[K, T, This]] extends Se def minBy[B](fn: T => B)(implicit cmp: Ordering[B]): This[K, T] = reduce(MinOrdBy(fn, cmp)) - - /** Use this to error if there is more than 1 value per key + /** + * Use this to error if there is more than 1 value per key * Using this makes it easier to detect when data does * not have the shape you expect and to communicate to * scalding that certain optimizations are safe to do diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/LookupJoin.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/LookupJoin.scala index 4df34601a0..c02b8ef28e 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/LookupJoin.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/LookupJoin.scala @@ -90,7 +90,8 @@ object LookupJoin extends Serializable { * In this case, the right pipe is fed through a scanLeft doing a Semigroup.plus * before joined to the left */ - def rightSumming[T: Ordering, K: Ordering, V, JoinedV: Semigroup](left: TypedPipe[(T, (K, V))], + def rightSumming[T: Ordering, K: Ordering, V, JoinedV: Semigroup]( + left: TypedPipe[(T, (K, V))], right: TypedPipe[(T, (K, JoinedV))], reducers: Option[Int] = None): TypedPipe[(T, (K, (V, Option[JoinedV])))] = withWindowRightSumming(left, right, reducers)((_, _) => true) @@ -100,7 +101,8 @@ object LookupJoin extends Serializable { * as the joined value. * Useful for bounding the time of the join to a recent window */ - def withWindow[T: Ordering, K: Ordering, V, JoinedV](left: TypedPipe[(T, (K, V))], + def withWindow[T: Ordering, K: Ordering, V, JoinedV]( + left: TypedPipe[(T, (K, V))], right: TypedPipe[(T, (K, JoinedV))], reducers: Option[Int] = None)(gate: (T, T) => Boolean): TypedPipe[(T, (K, (V, Option[JoinedV])))] = { @@ -113,7 +115,8 @@ object LookupJoin extends Serializable { * as the joined value, and sums are only done as long as they they come * within the gate interval as well */ - def withWindowRightSumming[T: Ordering, K: Ordering, V, JoinedV: Semigroup](left: TypedPipe[(T, (K, V))], + def withWindowRightSumming[T: Ordering, K: Ordering, V, JoinedV: Semigroup]( + left: TypedPipe[(T, (K, V))], right: TypedPipe[(T, (K, JoinedV))], reducers: Option[Int] = None)(gate: (T, T) => Boolean): TypedPipe[(T, (K, (V, Option[JoinedV])))] = { /** diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/NoStackAndThen.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/NoStackAndThen.scala index b935e1a91d..6fcd47effe 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/NoStackAndThen.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/NoStackAndThen.scala @@ -28,7 +28,8 @@ sealed trait NoStackAndThen[-A, +B] extends java.io.Serializable { def andThen[C](that: NoStackAndThen[B, C]): NoStackAndThen[A, C] = { import NoStackAndThen._ @annotation.tailrec - def push(front: NoStackAndThen[A, Any], + def push( + front: NoStackAndThen[A, Any], next: NoStackAndThen[Any, Any], toAndThen: ReversedStack[Any, C]): NoStackAndThen[A, C] = (next, toAndThen) match { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/OptimizationRules.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/OptimizationRules.scala index 43992cf18f..7435864f47 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/OptimizationRules.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/OptimizationRules.scala @@ -40,7 +40,7 @@ object OptimizationRules { Unary(f(cp.pipe), CounterPipe(_: TypedPipe[(a, Iterable[((String, String), Long)])])) case (c: CrossPipe[a, b], f) => Binary(f(c.left), f(c.right), CrossPipe(_: TypedPipe[a], _: TypedPipe[b])) - case (cv@CrossValue(_, _), f) => + case (cv @ CrossValue(_, _), f) => def go[A, B](cv: CrossValue[A, B]): LiteralPipe[(A, B)] = cv match { case CrossValue(a, ComputedValue(v)) => @@ -65,7 +65,7 @@ object OptimizationRules { Unary(f(p.input), FlatMapped(_: TypedPipe[a], p.fn)) case (p: ForceToDisk[a], f) => Unary(f(p.input), ForceToDisk(_: TypedPipe[a])) - case (it@IterablePipe(_), _) => + case (it @ IterablePipe(_), _) => Literal.Const(it) case (p: MapValues[a, b, c], f) => widen(Unary(f(p.input), MapValues(_: TypedPipe[(a, b)], p.fn))) @@ -73,7 +73,7 @@ object OptimizationRules { Unary(f(p.input), Mapped(_: TypedPipe[a], p.fn)) case (p: MergedTypedPipe[a], f) => Binary(f(p.left), f(p.right), MergedTypedPipe(_: TypedPipe[a], _: TypedPipe[a])) - case (src@SourcePipe(_), _) => + case (src @ SourcePipe(_), _) => Literal.Const(src) case (p: SumByLocalKeys[a, b], f) => widen(Unary(f(p.input), SumByLocalKeys(_: TypedPipe[(a, b)], p.semigroup))) @@ -94,150 +94,151 @@ object OptimizationRules { } }) - private def handleReduceStep[K, V1, V2](rs: ReduceStep[K, V1, V2], recurse: FunctionK[TypedPipe, LiteralPipe]): LiteralPipe[(K, V2)] = { - // zero out the input so we can potentially GC it - val emptyRs = ReduceStep.setInput[K, V1, V2](rs, TypedPipe.empty) + private def handleReduceStep[K, V1, V2](rs: ReduceStep[K, V1, V2], recurse: FunctionK[TypedPipe, LiteralPipe]): LiteralPipe[(K, V2)] = { + // zero out the input so we can potentially GC it + val emptyRs = ReduceStep.setInput[K, V1, V2](rs, TypedPipe.empty) - Unary(widen[(K, V1)](recurse(rs.mapped)), { (tp: TypedPipe[(K, V1)]) => - ReduceStepPipe(ReduceStep.setInput[K, V1, V2](emptyRs, tp)) - }) - } + Unary(widen[(K, V1)](recurse(rs.mapped)), { (tp: TypedPipe[(K, V1)]) => + ReduceStepPipe(ReduceStep.setInput[K, V1, V2](emptyRs, tp)) + }) + } - private def handleCoGrouped[K, V](cg: CoGroupable[K, V], recurse: FunctionK[TypedPipe, LiteralPipe]): LiteralPipe[(K, V)] = { - import CoGrouped._ + private def handleCoGrouped[K, V](cg: CoGroupable[K, V], recurse: FunctionK[TypedPipe, LiteralPipe]): LiteralPipe[(K, V)] = { + import CoGrouped._ - def pipeToCG[V1](t: TypedPipe[(K, V1)]): (CoGroupable[K, V1], List[(String, Boolean)]) = - t match { - case ReduceStepPipe(cg: CoGroupable[K @unchecked, V1 @unchecked]) => - // we are relying on the fact that we use Ordering[K] - // as a contravariant type, despite it not being defined - // that way. - (cg, Nil) - case CoGroupedPipe(cg) => - // we are relying on the fact that we use Ordering[K] - // as a contravariant type, despite it not being defined - // that way. - (cg.asInstanceOf[CoGroupable[K, V1]], Nil) - case WithDescriptionTypedPipe(pipe, descs) => - val (cg, d1) = pipeToCG(pipe) - (cg, ComposeDescriptions.combine(d1, descs)) - case kvPipe => - (IdentityReduce[K, V1, V1](cg.keyOrdering, kvPipe, None, Nil, implicitly), Nil) - } + def pipeToCG[V1](t: TypedPipe[(K, V1)]): (CoGroupable[K, V1], List[(String, Boolean)]) = + t match { + case ReduceStepPipe(cg: CoGroupable[K @unchecked, V1 @unchecked]) => + // we are relying on the fact that we use Ordering[K] + // as a contravariant type, despite it not being defined + // that way. + (cg, Nil) + case CoGroupedPipe(cg) => + // we are relying on the fact that we use Ordering[K] + // as a contravariant type, despite it not being defined + // that way. + (cg.asInstanceOf[CoGroupable[K, V1]], Nil) + case WithDescriptionTypedPipe(pipe, descs) => + val (cg, d1) = pipeToCG(pipe) + (cg, ComposeDescriptions.combine(d1, descs)) + case kvPipe => + (IdentityReduce[K, V1, V1](cg.keyOrdering, kvPipe, None, Nil, implicitly), Nil) + } - cg match { - case p@Pair(_, _, _) => - def go[A, B, C](pair: Pair[K, A, B, C]): LiteralPipe[(K, C)] = { - val llit = handleCoGrouped(pair.larger, recurse) - val rlit = handleCoGrouped(pair.smaller, recurse) - val fn = pair.fn - Binary(llit, rlit, { (l: TypedPipe[(K, A)], r: TypedPipe[(K, B)]) => - val (left, d1) = pipeToCG(l) - val (right, d2) = pipeToCG(r) - val d3 = ComposeDescriptions.combine(d1, d2) - val pair = Pair(left, right, fn) - val withD = d3.foldLeft(pair: CoGrouped[K, C]) { case (p, (d, _)) => + cg match { + case p @ Pair(_, _, _) => + def go[A, B, C](pair: Pair[K, A, B, C]): LiteralPipe[(K, C)] = { + val llit = handleCoGrouped(pair.larger, recurse) + val rlit = handleCoGrouped(pair.smaller, recurse) + val fn = pair.fn + Binary(llit, rlit, { (l: TypedPipe[(K, A)], r: TypedPipe[(K, B)]) => + val (left, d1) = pipeToCG(l) + val (right, d2) = pipeToCG(r) + val d3 = ComposeDescriptions.combine(d1, d2) + val pair = Pair(left, right, fn) + val withD = d3.foldLeft(pair: CoGrouped[K, C]) { + case (p, (d, _)) => p.withDescription(d) - } - CoGroupedPipe(withD) - }) - } - widen(go(p)) - case wr@WithReducers(_, _) => - def go[V1 <: V](wr: WithReducers[K, V1]): LiteralPipe[(K, V)] = { - val reds = wr.reds - Unary[TypedPipe, (K, V1), (K, V)](handleCoGrouped(wr.on, recurse), { (tp: TypedPipe[(K, V1)]) => - tp match { - case ReduceStepPipe(rs) => - ReduceStepPipe(ReduceStep.withReducers(rs, reds)) - case CoGroupedPipe(cg) => - CoGroupedPipe(WithReducers(cg, reds)) - case kvPipe => - ReduceStepPipe(IdentityReduce[K, V1, V1](cg.keyOrdering, kvPipe, None, Nil, implicitly) - .withReducers(reds)) - } - }) - } - go(wr) - case wd@WithDescription(_, _) => - def go[V1 <: V](wd: WithDescription[K, V1]): LiteralPipe[(K, V)] = { - val desc = wd.description - Unary[TypedPipe, (K, V1), (K, V)](handleCoGrouped(wd.on, recurse), { (tp: TypedPipe[(K, V1)]) => - tp match { - case ReduceStepPipe(rs) => - ReduceStepPipe(ReduceStep.withDescription(rs, desc)) - case CoGroupedPipe(cg) => - CoGroupedPipe(WithDescription(cg, desc)) - case kvPipe => - kvPipe.withDescription(desc) - } - }) - } - go(wd) - case fk@FilterKeys(_, _) => - def go[V1 <: V](fk: FilterKeys[K, V1]): LiteralPipe[(K, V)] = { - val fn = fk.fn - Unary[TypedPipe, (K, V1), (K, V)](handleCoGrouped(fk.on, recurse), { (tp: TypedPipe[(K, V1)]) => - tp match { - case ReduceStepPipe(rs) => - val mapped = rs.mapped - val mappedF = TypedPipe.FilterKeys(mapped, fn) - ReduceStepPipe(ReduceStep.setInput(rs, mappedF)) - case CoGroupedPipe(cg) => - CoGroupedPipe(FilterKeys(cg, fn)) - case kvPipe => - TypedPipe.FilterKeys(kvPipe, fn) - } - }) - } - go(fk) - case mg@MapGroup(_, _) => - def go[V1, V2 <: V](mg: MapGroup[K, V1, V2]): LiteralPipe[(K, V)] = { - val fn = mg.fn - Unary[TypedPipe, (K, V1), (K, V)](handleCoGrouped(mg.on, recurse), { (tp: TypedPipe[(K, V1)]) => - tp match { - case ReduceStepPipe(rs) => - ReduceStepPipe(ReduceStep.mapGroup(rs)(fn)) - case CoGroupedPipe(cg) => - CoGroupedPipe(MapGroup(cg, fn)) - case kvPipe => - ReduceStepPipe( - IdentityReduce[K, V1, V1](cg.keyOrdering, kvPipe, None, Nil, implicitly) - .mapGroup(fn)) - } - }) - } - go(mg) - case step@IdentityReduce(_, _, _, _, _) => - widen(handleReduceStep(step, recurse)) - case step@UnsortedIdentityReduce(_, _, _, _, _) => - widen(handleReduceStep(step, recurse)) - case step@IteratorMappedReduce(_, _, _, _, _) => - widen(handleReduceStep(step, recurse)) - } + } + CoGroupedPipe(withD) + }) + } + widen(go(p)) + case wr @ WithReducers(_, _) => + def go[V1 <: V](wr: WithReducers[K, V1]): LiteralPipe[(K, V)] = { + val reds = wr.reds + Unary[TypedPipe, (K, V1), (K, V)](handleCoGrouped(wr.on, recurse), { (tp: TypedPipe[(K, V1)]) => + tp match { + case ReduceStepPipe(rs) => + ReduceStepPipe(ReduceStep.withReducers(rs, reds)) + case CoGroupedPipe(cg) => + CoGroupedPipe(WithReducers(cg, reds)) + case kvPipe => + ReduceStepPipe(IdentityReduce[K, V1, V1](cg.keyOrdering, kvPipe, None, Nil, implicitly) + .withReducers(reds)) + } + }) + } + go(wr) + case wd @ WithDescription(_, _) => + def go[V1 <: V](wd: WithDescription[K, V1]): LiteralPipe[(K, V)] = { + val desc = wd.description + Unary[TypedPipe, (K, V1), (K, V)](handleCoGrouped(wd.on, recurse), { (tp: TypedPipe[(K, V1)]) => + tp match { + case ReduceStepPipe(rs) => + ReduceStepPipe(ReduceStep.withDescription(rs, desc)) + case CoGroupedPipe(cg) => + CoGroupedPipe(WithDescription(cg, desc)) + case kvPipe => + kvPipe.withDescription(desc) + } + }) + } + go(wd) + case fk @ FilterKeys(_, _) => + def go[V1 <: V](fk: FilterKeys[K, V1]): LiteralPipe[(K, V)] = { + val fn = fk.fn + Unary[TypedPipe, (K, V1), (K, V)](handleCoGrouped(fk.on, recurse), { (tp: TypedPipe[(K, V1)]) => + tp match { + case ReduceStepPipe(rs) => + val mapped = rs.mapped + val mappedF = TypedPipe.FilterKeys(mapped, fn) + ReduceStepPipe(ReduceStep.setInput(rs, mappedF)) + case CoGroupedPipe(cg) => + CoGroupedPipe(FilterKeys(cg, fn)) + case kvPipe => + TypedPipe.FilterKeys(kvPipe, fn) + } + }) + } + go(fk) + case mg @ MapGroup(_, _) => + def go[V1, V2 <: V](mg: MapGroup[K, V1, V2]): LiteralPipe[(K, V)] = { + val fn = mg.fn + Unary[TypedPipe, (K, V1), (K, V)](handleCoGrouped(mg.on, recurse), { (tp: TypedPipe[(K, V1)]) => + tp match { + case ReduceStepPipe(rs) => + ReduceStepPipe(ReduceStep.mapGroup(rs)(fn)) + case CoGroupedPipe(cg) => + CoGroupedPipe(MapGroup(cg, fn)) + case kvPipe => + ReduceStepPipe( + IdentityReduce[K, V1, V1](cg.keyOrdering, kvPipe, None, Nil, implicitly) + .mapGroup(fn)) + } + }) + } + go(mg) + case step @ IdentityReduce(_, _, _, _, _) => + widen(handleReduceStep(step, recurse)) + case step @ UnsortedIdentityReduce(_, _, _, _, _) => + widen(handleReduceStep(step, recurse)) + case step @ IteratorMappedReduce(_, _, _, _, _) => + widen(handleReduceStep(step, recurse)) } + } - private def handleHashCoGroup[K, V, V2, R](hj: HashCoGroup[K, V, V2, R], recurse: FunctionK[TypedPipe, LiteralPipe]): LiteralPipe[(K, R)] = { - val rightLit: LiteralPipe[(K, V2)] = { - val rs = HashJoinable.toReduceStep(hj.right) - def go[A, B, C](rs: ReduceStep[A, B, C]): LiteralPipe[(A, C)] = - Unary(recurse(rs.mapped), { tp: TypedPipe[(A, B)] => ReduceStepPipe(ReduceStep.setInput(rs, tp)) }) - widen(go(rs)) - } + private def handleHashCoGroup[K, V, V2, R](hj: HashCoGroup[K, V, V2, R], recurse: FunctionK[TypedPipe, LiteralPipe]): LiteralPipe[(K, R)] = { + val rightLit: LiteralPipe[(K, V2)] = { + val rs = HashJoinable.toReduceStep(hj.right) + def go[A, B, C](rs: ReduceStep[A, B, C]): LiteralPipe[(A, C)] = + Unary(recurse(rs.mapped), { tp: TypedPipe[(A, B)] => ReduceStepPipe(ReduceStep.setInput(rs, tp)) }) + widen(go(rs)) + } - val ordK: Ordering[K] = hj.right.keyOrdering - val joiner = hj.joiner + val ordK: Ordering[K] = hj.right.keyOrdering + val joiner = hj.joiner - Binary(recurse(hj.left), rightLit, - { (ltp: TypedPipe[(K, V)], rtp: TypedPipe[(K, V2)]) => - rtp match { - case ReduceStepPipe(hg: HashJoinable[K @unchecked, V2 @unchecked]) => - HashCoGroup(ltp, hg, joiner) - case otherwise => - HashCoGroup(ltp, IdentityReduce[K, V2, V2](ordK, otherwise, None, Nil, implicitly), joiner) - } - }) - } + Binary(recurse(hj.left), rightLit, + { (ltp: TypedPipe[(K, V)], rtp: TypedPipe[(K, V2)]) => + rtp match { + case ReduceStepPipe(hg: HashJoinable[K @unchecked, V2 @unchecked]) => + HashCoGroup(ltp, hg, joiner) + case otherwise => + HashCoGroup(ltp, IdentityReduce[K, V2, V2](ordK, otherwise, None, Nil, implicitly), joiner) + } + }) + } /** * Unroll a set of merges up to the first non-merge node, dropping @@ -363,11 +364,11 @@ object OptimizationRules { private def forkHashJoinable[K, V](on: Dag[TypedPipe], hj: HashJoinable[K, V]): Option[HashJoinable[K, V]] = hj match { - case step@IdentityReduce(_, _, _, _, _) => + case step @ IdentityReduce(_, _, _, _, _) => maybeFork(on, step.mapped).map { p => step.copy(mapped = p) } - case step@UnsortedIdentityReduce(_, _, _, _, _) => + case step @ UnsortedIdentityReduce(_, _, _, _, _) => maybeFork(on, step.mapped).map { p => step.copy(mapped = p) } - case step@IteratorMappedReduce(_, _, _, _, _) => + case step @ IteratorMappedReduce(_, _, _, _, _) => maybeFork(on, step.mapped).map { p => step.copy(mapped = p) } } @@ -379,7 +380,7 @@ object OptimizationRules { case CrossValue(a, ComputedValue(b)) if needsFork(on, b) => maybeFork(on, b).map { fb => CrossValue(a, ComputedValue(fb)) } case DebugPipe(p) => maybeFork(on, p).map(DebugPipe(_)) case FilterKeys(p, fn) => maybeFork(on, p).map(FilterKeys(_, fn)) - case f@Filter(_, _) => + case f @ Filter(_, _) => def go[A](f: Filter[A]): Option[TypedPipe[A]] = { val Filter(p, fn) = f maybeFork(on, p).map(Filter(_, fn)) @@ -396,7 +397,7 @@ object OptimizationRules { case MergedTypedPipe(a, b) if needsFork(on, b) => maybeFork(on, b).map(MergedTypedPipe(a, _)) case ReduceStepPipe(rs) => forkReduceStep(on, rs).map(ReduceStepPipe(_)) case SumByLocalKeys(p, sg) => maybeFork(on, p).map(SumByLocalKeys(_, sg)) - case t@TrappedPipe(_, _, _) => + case t @ TrappedPipe(_, _, _) => def go[A](t: TrappedPipe[A]): Option[TypedPipe[A]] = { val TrappedPipe(p, sink, conv) = t maybeFork(on, p).map(TrappedPipe(_, sink, conv)) @@ -444,7 +445,7 @@ object OptimizationRules { // scala can't type check this, so we hold its hand: // case Filter(Filter(in, fn0), fn1) => // Some(Filter(in, ComposedFilterFn(fn0, fn1))) - case f@Filter(_, _) => + case f @ Filter(_, _) => def go[A](f: Filter[A]): Option[TypedPipe[A]] = f.input match { case f1: Filter[a] => @@ -518,7 +519,6 @@ object OptimizationRules { } } - /** * a.filter(f).flatMap(g) == a.flatMap { x => if (f(x)) g(x) else Iterator.empty } * a.flatMap(f).filter(g) == a.flatMap { x => f(x).filter(g) } @@ -574,7 +574,7 @@ object OptimizationRules { val combined = descs1 ::: descs2 combined.foldLeft((Set.empty[String], List.empty[(String, Boolean)])) { - case (state@(s, acc), item@(m, true)) => + case (state @ (s, acc), item @ (m, true)) => if (s(m)) state else (s + m, item :: acc) case ((s, acc), item) => @@ -582,7 +582,6 @@ object OptimizationRules { }._2.reverse } - def applyWhere[T](on: Dag[TypedPipe]) = { case WithDescriptionTypedPipe(WithDescriptionTypedPipe(input, descs1), descs2) => WithDescriptionTypedPipe(input, combine(descs1, descs2)) @@ -603,7 +602,7 @@ object OptimizationRules { WithDescriptionTypedPipe(FlatMapped(in, fn), descs) case FlatMapValues(WithDescriptionTypedPipe(in, descs), fn) => WithDescriptionTypedPipe(FlatMapValues(in, fn), descs) - case f@Filter(WithDescriptionTypedPipe(_, _), _) => + case f @ Filter(WithDescriptionTypedPipe(_, _), _) => def go[A](f: Filter[A]): TypedPipe[A] = f match { case Filter(WithDescriptionTypedPipe(in, descs), fn) => @@ -632,7 +631,7 @@ object OptimizationRules { */ object DiamondToFlatMap extends Rule[TypedPipe] { def apply[T](on: Dag[TypedPipe]) = { - case m@MergedTypedPipe(_, _) => + case m @ MergedTypedPipe(_, _) => val pipes = unrollMerge(m) val flatMapped = dedupMerge(pipes) @@ -724,7 +723,7 @@ object OptimizationRules { Mapper.unmapped(tp) :: Nil case FilterKeys(p, fn) => toMappers(p).map(_.combine(FlatMappedFn.fromFilter(FilterKeysToFilter(fn)))) - case f@Filter(_, _) => + case f @ Filter(_, _) => // type inference needs hand holding on this one def go[A1 <: A](p: TypedPipe[A1], fn: A1 => Boolean): List[Mapper[A]] = { val fn1: FlatMappedFn[A1, A] = @@ -754,7 +753,7 @@ object OptimizationRules { case h :: Nil => // there is only one Mapper, just convert back to a TypedPipe h.toTypedPipe - case all@(h :: t) => + case all @ (h :: t) => // we have several merged back from a common point // we don't know what that previous type was, but // we cast it to Any, we know the values in there @@ -769,12 +768,12 @@ object OptimizationRules { def apply[A](on: Dag[TypedPipe]) = { // here are the trailing mappers of this pipe - case fork@Fork(inner) if on.hasSingleDependent(fork) => + case fork @ Fork(inner) if on.hasSingleDependent(fork) => // due to a previous application of this rule, // this fork may have been reduced to only one // downstream, in that case, we can remove the fork Some(inner) - case m@MergedTypedPipe(_, _) => + case m @ MergedTypedPipe(_, _) => // this rule only applies to merged pipes // let's see if we have any duplicated inputs: val mapperGroups = toMappers(m).groupBy(_.input) @@ -815,11 +814,11 @@ object OptimizationRules { */ object RemoveUselessFork extends PartialRule[TypedPipe] { def applyWhere[T](on: Dag[TypedPipe]) = { - case fork@Fork(t) if on.hasSingleDependent(fork) => t - case Fork(src@SourcePipe(_)) => src - case Fork(iter@IterablePipe(_)) => iter - case ForceToDisk(src@SourcePipe(_)) => src - case ForceToDisk(iter@IterablePipe(_)) => iter + case fork @ Fork(t) if on.hasSingleDependent(fork) => t + case Fork(src @ SourcePipe(_)) => src + case Fork(iter @ IterablePipe(_)) => iter + case ForceToDisk(src @ SourcePipe(_)) => src + case ForceToDisk(iter @ IterablePipe(_)) => iter } } @@ -871,7 +870,7 @@ object OptimizationRules { MergedTypedPipe(MapValues(a, fn), MapValues(b, fn)) case FlatMapValues(MergedTypedPipe(a, b), fn) => MergedTypedPipe(FlatMapValues(a, fn), FlatMapValues(b, fn)) - case f@Filter(_, _) if handleFilter.isDefinedAt(f) => handleFilter(f) + case f @ Filter(_, _) if handleFilter.isDefinedAt(f) => handleFilter(f) case FilterKeys(MergedTypedPipe(a, b), fn) => MergedTypedPipe(FilterKeys(a, fn), FilterKeys(b, fn)) } @@ -1007,7 +1006,7 @@ object OptimizationRules { */ object FilterLocally extends Rule[TypedPipe] { def apply[T](on: Dag[TypedPipe]) = { - case f@Filter(_, _) => + case f @ Filter(_, _) => def go[T1 <: T](f: Filter[T1]): Option[TypedPipe[T]] = f match { case Filter(IterablePipe(iter), fn) => @@ -1015,7 +1014,7 @@ object OptimizationRules { case _ => None } go(f) - case f@FilterKeys(_, _) => + case f @ FilterKeys(_, _) => def go[K, V, T >: (K, V)](f: FilterKeys[K, V]): Option[TypedPipe[T]] = f match { case FilterKeys(IterablePipe(iter), fn) => @@ -1054,17 +1053,17 @@ object OptimizationRules { def apply[T](on: Dag[TypedPipe]) = { case HashCoGroup(left, right: HashJoinable[a, b], joiner) => val newRight: HashJoinable[a, b] = right match { - case step@IdentityReduce(_, _, _, _, _) => + case step @ IdentityReduce(_, _, _, _, _) => step.copy(mapped = maybeForce(step.mapped)) - case step@UnsortedIdentityReduce(_, _, _, _, _) => + case step @ UnsortedIdentityReduce(_, _, _, _, _) => step.copy(mapped = maybeForce(step.mapped)) - case step@IteratorMappedReduce(_, _, _, _, _) => + case step @ IteratorMappedReduce(_, _, _, _, _) => step.copy(mapped = maybeForce(step.mapped)) } if (newRight != right) Some(HashCoGroup(left, newRight, joiner)) else None - case (cp@CrossPipe(_, _)) => Some(cp.viaHashJoin) - case (cv@CrossValue(_, _)) => Some(cv.viaHashJoin) + case (cp @ CrossPipe(_, _)) => Some(cp.viaHashJoin) + case (cv @ CrossValue(_, _)) => Some(cv.viaHashJoin) case _ => None } } @@ -1078,13 +1077,12 @@ object OptimizationRules { val leftg = Grouped(left)(right.keyOrdering) val joiner2 = Joiner.toCogroupJoiner2(joiner) Some(CoGroupedPipe(CoGrouped.Pair(leftg, right, joiner2))) - case (cp@CrossPipe(_, _)) => Some(cp.viaHashJoin) - case (cv@CrossValue(_, _)) => Some(cv.viaHashJoin) + case (cp @ CrossPipe(_, _)) => Some(cp.viaHashJoin) + case (cv @ CrossValue(_, _)) => Some(cv.viaHashJoin) case _ => None } } - /** * Prefer to do mapValues/flatMapValues in a Reduce/Join * so we can avoid some boxing in-and-out of cascading @@ -1109,7 +1107,7 @@ object OptimizationRules { CoGroupedPipe(CoGrouped.MapGroup(cg, MapGroupMapValues(fn))) case FlatMapValues(CoGroupedPipe(cg), fn) => CoGroupedPipe(CoGrouped.MapGroup(cg, MapGroupFlatMapValues(fn))) - case f@Filter(_, _) if handleFilter(f).isDefined => handleFilter(f).getOrElse(sys.error("unreachable: already checked isDefined")) + case f @ Filter(_, _) if handleFilter(f).isDefined => handleFilter(f).getOrElse(sys.error("unreachable: already checked isDefined")) case SumByLocalKeys(ReduceStepPipe(rs), sg) => ReduceStepPipe(ReduceStep.mapGroup(rs)(MapValueStream(SumAll(sg)))) case SumByLocalKeys(CoGroupedPipe(cg), sg) => diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/PartitionedDelimitedSource.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/PartitionedDelimitedSource.scala index 516b8dd86a..f990da072c 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/PartitionedDelimitedSource.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/PartitionedDelimitedSource.scala @@ -47,7 +47,7 @@ import cascading.tuple.Fields case class PartitionedDelimitedSource[P, T]( path: String, template: String, separator: String, fields: Fields, skipHeader: Boolean = false, writeHeader: Boolean = false, quote: String = "\"", strict: Boolean = true, safe: Boolean = true)(implicit mt: Manifest[T], val valueSetter: TupleSetter[T], val valueConverter: TupleConverter[T], - val partitionSetter: TupleSetter[P], val partitionConverter: TupleConverter[P]) extends PartitionSchemed[P, T] with Serializable { + val partitionSetter: TupleSetter[P], val partitionConverter: TupleConverter[P]) extends PartitionSchemed[P, T] with Serializable { assert( fields.size == valueSetter.arity, "The number of fields needs to be the same as the arity of the value setter") diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/PartitionedTextLine.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/PartitionedTextLine.scala index c35a1c6908..a1dd039cd2 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/PartitionedTextLine.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/PartitionedTextLine.scala @@ -53,7 +53,7 @@ import cascading.tuple.Fields */ case class PartitionedTextLine[P]( path: String, template: String, encoding: String = TextLine.DEFAULT_CHARSET)(implicit val valueSetter: TupleSetter[String], val valueConverter: TupleConverter[(Long, String)], - val partitionSetter: TupleSetter[P], val partitionConverter: TupleConverter[P]) extends SchemedSource with TypedSink[(P, String)] with Mappable[(P, (Long, String))] with HfsTapProvider + val partitionSetter: TupleSetter[P], val partitionConverter: TupleConverter[P]) extends SchemedSource with TypedSink[(P, String)] with Mappable[(P, (Long, String))] with HfsTapProvider with java.io.Serializable { // The partition fields, offset by the value arity. diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/Sketched.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/Sketched.scala index de74afd394..ee3c1aa3c1 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/Sketched.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/Sketched.scala @@ -26,12 +26,14 @@ import com.twitter.algebird.CMSMonoid * This class is generally only created by users * with the TypedPipe.sketch method */ -case class Sketched[K, V](pipe: TypedPipe[(K, V)], +case class Sketched[K, V]( + pipe: TypedPipe[(K, V)], numReducers: Int, delta: Double, eps: Double, - seed: Int)(implicit val serialization: K => Array[Byte], - ordering: Ordering[K]) + seed: Int)(implicit + val serialization: K => Array[Byte], + ordering: Ordering[K]) extends MustHaveReducers { def reducers = Some(numReducers) @@ -78,7 +80,8 @@ case class Sketched[K, V](pipe: TypedPipe[(K, V)], cogroup(right)(Joiner.hashLeft2) } -case class SketchJoined[K: Ordering, V, V2, R](left: Sketched[K, V], +case class SketchJoined[K: Ordering, V, V2, R]( + left: Sketched[K, V], right: TypedPipe[(K, V2)], numReducers: Int)(joiner: (K, V, Iterable[V2]) => Iterator[R]) extends MustHaveReducers { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/TypedPipe.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/TypedPipe.scala index afe23cc625..824ceb9c30 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/TypedPipe.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/TypedPipe.scala @@ -26,7 +26,7 @@ import com.twitter.scalding.serialization.{ OrderedSerialization, UnitOrderedSer import com.twitter.scalding.serialization.OrderedSerialization.Result import com.twitter.scalding.serialization.macros.impl.BinaryOrdering import com.twitter.scalding.serialization.macros.impl.BinaryOrdering._ -import com.stripe.dagon.{Memoize, RefPair} +import com.stripe.dagon.{ Memoize, RefPair } import scala.util.Try import scala.util.hashing.MurmurHash3 @@ -48,7 +48,8 @@ object TypedPipe extends Serializable { * This could be in TypedSource, but we don't want to encourage users * to work directly with Pipe */ - case class WrappingSource[T](pipe: Pipe, + case class WrappingSource[T]( + pipe: Pipe, fields: Fields, @transient localFlow: FlowDef, // FlowDef is not serializable. We shouldn't need to, but being paranoid mode: Mode, @@ -60,7 +61,8 @@ object TypedPipe extends Serializable { def read(implicit fd: FlowDef, m: Mode): Pipe = { // This check is not likely to fail unless someone does something really strange. // for historical reasons, it is not checked by the typed system - require(m == mode, + require( + m == mode, s"Cannot switch Mode between TypedPipe.from and toPipe calls. Pipe: $pipe, pipe mode: $m, outer mode: $mode") Dsl.flowDefToRichFlowDef(fd).mergeFrom(localFlow) pipe @@ -134,8 +136,8 @@ object TypedPipe extends Serializable { if (sz == 0) TypedPipe.empty else if (sz == 1) ps(0) else { - val left = combine(ps.take(sz/2)) - val right = combine(ps.drop(sz/2)) + val left = combine(ps.take(sz / 2)) + val right = combine(ps.drop(sz / 2)) left ++ right } } @@ -156,88 +158,88 @@ object TypedPipe extends Serializable { } } - final case class CoGroupedPipe[K, V](@transient cogrouped: CoGrouped[K, V]) extends TypedPipe[(K, V)] - final case class CounterPipe[A](pipe: TypedPipe[(A, Iterable[((String, String), Long)])]) extends TypedPipe[A] - final case class CrossPipe[T, U](left: TypedPipe[T], right: TypedPipe[U]) extends TypedPipe[(T, U)] { - def viaHashJoin: TypedPipe[(T, U)] = - left.withKey(()).hashJoin(right.withKey(())).values - } - final case class CrossValue[T, U](left: TypedPipe[T], right: ValuePipe[U]) extends TypedPipe[(T, U)] { - def viaHashJoin: TypedPipe[(T, U)] = - right match { - case EmptyValue => - EmptyTypedPipe - case LiteralValue(v) => - left.map(WithConstant(v)) - case ComputedValue(pipe) => - CrossPipe(left, pipe) - } - } - final case class DebugPipe[T](input: TypedPipe[T]) extends TypedPipe[T] - final case class FilterKeys[K, V](input: TypedPipe[(K, V)], @transient fn: K => Boolean) extends TypedPipe[(K, V)] - final case class Filter[T](input: TypedPipe[T], @transient fn: T => Boolean) extends TypedPipe[T] - final case class FlatMapValues[K, V, U](input: TypedPipe[(K, V)], @transient fn: V => TraversableOnce[U]) extends TypedPipe[(K, U)] - final case class FlatMapped[T, U](input: TypedPipe[T], @transient fn: T => TraversableOnce[U]) extends TypedPipe[U] - final case class ForceToDisk[T](input: TypedPipe[T]) extends TypedPipe[T] - final case class Fork[T](input: TypedPipe[T]) extends TypedPipe[T] - final case class HashCoGroup[K, V, W, R](left: TypedPipe[(K, V)], @transient right: HashJoinable[K, W], @transient joiner: (K, V, Iterable[W]) => Iterator[R]) extends TypedPipe[(K, R)] - final case class IterablePipe[T](iterable: Iterable[T]) extends TypedPipe[T] - final case class MapValues[K, V, U](input: TypedPipe[(K, V)], @transient fn: V => U) extends TypedPipe[(K, U)] - final case class Mapped[T, U](input: TypedPipe[T], @transient fn: T => U) extends TypedPipe[U] - final case class MergedTypedPipe[T](left: TypedPipe[T], right: TypedPipe[T]) extends TypedPipe[T] - final case class ReduceStepPipe[K, V1, V2](@transient reduce: ReduceStep[K, V1, V2]) extends TypedPipe[(K, V2)] - final case class SourcePipe[T](@transient source: TypedSource[T]) extends TypedPipe[T] - final case class SumByLocalKeys[K, V](input: TypedPipe[(K, V)], @transient semigroup: Semigroup[V]) extends TypedPipe[(K, V)] - final case class TrappedPipe[T](input: TypedPipe[T], @transient sink: Source with TypedSink[T], @transient conv: TupleConverter[T]) extends TypedPipe[T] - /** - * descriptions carry a boolean that is true if we should deduplicate the message. - * This is used for line numbers which are otherwise often duplicated - */ - final case class WithDescriptionTypedPipe[T](input: TypedPipe[T], descriptions: List[(String, Boolean)]) extends TypedPipe[T] - final case class WithOnComplete[T](input: TypedPipe[T], @transient fn: () => Unit) extends TypedPipe[T] - - case object EmptyTypedPipe extends TypedPipe[Nothing] { - // we can't let the default TypedPipe == go here, it will stack overflow on a pattern match - override def equals(that: Any): Boolean = - that match { - case e: EmptyTypedPipe.type => true - case _ => false - } - } - - implicit class InvariantTypedPipe[T](val pipe: TypedPipe[T]) extends AnyVal { - /** - * Returns the set of distinct elements in the TypedPipe - * This is the same as: .map((_, ())).group.sum.keys - * If you want a distinct while joining, consider: - * instead of: - * {@code - * a.join(b.distinct.asKeys) - * } - * manually do the distinct: - * {@code - * a.join(b.asKeys.sum) - * } - * The latter creates 1 map/reduce phase rather than 2 - */ - @annotation.implicitNotFound(msg = "For distinct method to work, the type in TypedPipe must have an Ordering.") - def distinct(implicit ord: Ordering[T]): TypedPipe[T] = - pipe.asKeys.sum.keys - - /** - * If any errors happen below this line, but before a groupBy, write to a TypedSink - */ - @deprecated("semantics of addTrap are hard to follow, prefer to use Either and manually write out error branchs", "0.18.0") - def addTrap(trapSink: Source with TypedSink[T])(implicit conv: TupleConverter[T]): TypedPipe[T] = - TypedPipe.TrappedPipe[T](pipe, trapSink, conv).withLine - } - - /** - * This is where all the methods that require TypedPipe[(K, V)] live. - * - * previously, these were directly on TypedPipe with the use of T <:< (K, V) - * however that complicates type inference on many functions. - */ + final case class CoGroupedPipe[K, V](@transient cogrouped: CoGrouped[K, V]) extends TypedPipe[(K, V)] + final case class CounterPipe[A](pipe: TypedPipe[(A, Iterable[((String, String), Long)])]) extends TypedPipe[A] + final case class CrossPipe[T, U](left: TypedPipe[T], right: TypedPipe[U]) extends TypedPipe[(T, U)] { + def viaHashJoin: TypedPipe[(T, U)] = + left.withKey(()).hashJoin(right.withKey(())).values + } + final case class CrossValue[T, U](left: TypedPipe[T], right: ValuePipe[U]) extends TypedPipe[(T, U)] { + def viaHashJoin: TypedPipe[(T, U)] = + right match { + case EmptyValue => + EmptyTypedPipe + case LiteralValue(v) => + left.map(WithConstant(v)) + case ComputedValue(pipe) => + CrossPipe(left, pipe) + } + } + final case class DebugPipe[T](input: TypedPipe[T]) extends TypedPipe[T] + final case class FilterKeys[K, V](input: TypedPipe[(K, V)], @transient fn: K => Boolean) extends TypedPipe[(K, V)] + final case class Filter[T](input: TypedPipe[T], @transient fn: T => Boolean) extends TypedPipe[T] + final case class FlatMapValues[K, V, U](input: TypedPipe[(K, V)], @transient fn: V => TraversableOnce[U]) extends TypedPipe[(K, U)] + final case class FlatMapped[T, U](input: TypedPipe[T], @transient fn: T => TraversableOnce[U]) extends TypedPipe[U] + final case class ForceToDisk[T](input: TypedPipe[T]) extends TypedPipe[T] + final case class Fork[T](input: TypedPipe[T]) extends TypedPipe[T] + final case class HashCoGroup[K, V, W, R](left: TypedPipe[(K, V)], @transient right: HashJoinable[K, W], @transient joiner: (K, V, Iterable[W]) => Iterator[R]) extends TypedPipe[(K, R)] + final case class IterablePipe[T](iterable: Iterable[T]) extends TypedPipe[T] + final case class MapValues[K, V, U](input: TypedPipe[(K, V)], @transient fn: V => U) extends TypedPipe[(K, U)] + final case class Mapped[T, U](input: TypedPipe[T], @transient fn: T => U) extends TypedPipe[U] + final case class MergedTypedPipe[T](left: TypedPipe[T], right: TypedPipe[T]) extends TypedPipe[T] + final case class ReduceStepPipe[K, V1, V2](@transient reduce: ReduceStep[K, V1, V2]) extends TypedPipe[(K, V2)] + final case class SourcePipe[T](@transient source: TypedSource[T]) extends TypedPipe[T] + final case class SumByLocalKeys[K, V](input: TypedPipe[(K, V)], @transient semigroup: Semigroup[V]) extends TypedPipe[(K, V)] + final case class TrappedPipe[T](input: TypedPipe[T], @transient sink: Source with TypedSink[T], @transient conv: TupleConverter[T]) extends TypedPipe[T] + /** + * descriptions carry a boolean that is true if we should deduplicate the message. + * This is used for line numbers which are otherwise often duplicated + */ + final case class WithDescriptionTypedPipe[T](input: TypedPipe[T], descriptions: List[(String, Boolean)]) extends TypedPipe[T] + final case class WithOnComplete[T](input: TypedPipe[T], @transient fn: () => Unit) extends TypedPipe[T] + + case object EmptyTypedPipe extends TypedPipe[Nothing] { + // we can't let the default TypedPipe == go here, it will stack overflow on a pattern match + override def equals(that: Any): Boolean = + that match { + case e: EmptyTypedPipe.type => true + case _ => false + } + } + + implicit class InvariantTypedPipe[T](val pipe: TypedPipe[T]) extends AnyVal { + /** + * Returns the set of distinct elements in the TypedPipe + * This is the same as: .map((_, ())).group.sum.keys + * If you want a distinct while joining, consider: + * instead of: + * {@code + * a.join(b.distinct.asKeys) + * } + * manually do the distinct: + * {@code + * a.join(b.asKeys.sum) + * } + * The latter creates 1 map/reduce phase rather than 2 + */ + @annotation.implicitNotFound(msg = "For distinct method to work, the type in TypedPipe must have an Ordering.") + def distinct(implicit ord: Ordering[T]): TypedPipe[T] = + pipe.asKeys.sum.keys + + /** + * If any errors happen below this line, but before a groupBy, write to a TypedSink + */ + @deprecated("semantics of addTrap are hard to follow, prefer to use Either and manually write out error branchs", "0.18.0") + def addTrap(trapSink: Source with TypedSink[T])(implicit conv: TupleConverter[T]): TypedPipe[T] = + TypedPipe.TrappedPipe[T](pipe, trapSink, conv).withLine + } + + /** + * This is where all the methods that require TypedPipe[(K, V)] live. + * + * previously, these were directly on TypedPipe with the use of T <:< (K, V) + * however that complicates type inference on many functions. + */ implicit class Keyed[K, V](val kvpipe: TypedPipe[(K, V)]) extends AnyVal { /** @@ -323,7 +325,8 @@ object TypedPipe extends Serializable { * This will generally only be beneficial if you have really heavy skew, where without * this you have 1 or 2 reducers taking hours longer than the rest. */ - def sketch(reducers: Int, + def sketch( + reducers: Int, eps: Double = 1.0E-5, //272k width = 1MB per row delta: Double = 0.01, //5 rows (= 5 hashes) seed: Int = 12345)(implicit serialization: K => Array[Byte], ordering: Ordering[K]): Sketched[K, V] = @@ -718,7 +721,6 @@ sealed abstract class TypedPipe[+T] extends Serializable with Product { def forceToDisk: TypedPipe[T] = TypedPipe.ForceToDisk(this).withLine - /** Send all items to a single reducer */ def groupAll: Grouped[Unit, T] = groupBy(Constant(()))(UnitOrderedSerialization).withReducers(1) @@ -874,7 +876,6 @@ sealed abstract class TypedPipe[+T] extends Serializable with Product { } } - /** * ValuePipe may be empty, so, this attaches it as an Option * cross is the same as leftCross(p).collect { case (t, Some(v)) => (t, v) } @@ -932,7 +933,6 @@ sealed abstract class TypedPipe[+T] extends Serializable with Product { def filterWithValue[U](value: ValuePipe[U])(f: (T, Option[U]) => Boolean): TypedPipe[T] = leftCross(value).filter(TuplizeFunction(f)).map(GetKey()) - /** * For each element, do a map-side (hash) left join to look up a value */ diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/TypedPipeDiff.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/TypedPipeDiff.scala index 0c648d1a53..b763e9ad05 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/TypedPipeDiff.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/TypedPipeDiff.scala @@ -36,7 +36,8 @@ object TypedPipeDiff { * which has the correct hashCode and equals needed. This does not involve * copying the arrays, just wrapping them, and is specialized for primitive arrays. */ - def diffArrayPipes[T: ClassTag](left: TypedPipe[Array[T]], + def diffArrayPipes[T: ClassTag]( + left: TypedPipe[Array[T]], right: TypedPipe[Array[T]], reducers: Option[Int] = None): TypedPipe[(Array[T], (Long, Long))] = { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/WritePartitioner.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/WritePartitioner.scala index 2755c5c1ae..5c42d30cce 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/WritePartitioner.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/WritePartitioner.scala @@ -82,7 +82,8 @@ object WritePartitioner { mat.sequence_(toSeq) } - def materialize1[M[+_], S[_]](phases: Seq[Rule[TypedPipe]], + def materialize1[M[+_], S[_]]( + phases: Seq[Rule[TypedPipe]], ws: List[PairK[TypedPipe, S, _]])(implicit mat: Materializer[M]): List[PairK[mat.TP, S, _]] = { val e = Dag.empty(OptimizationRules.toLiteral) @@ -97,24 +98,24 @@ object WritePartitioner { val optDag = finalDag.applySeq(phases :+ OptimizationRules.RemoveUselessFork) logger.info(s"optimized graph hash size: ${optDag.allNodes.size}") - import TypedPipe.{ReduceStepPipe, HashCoGroup} + import TypedPipe.{ ReduceStepPipe, HashCoGroup } def handleHashCoGroup[K, V, V2, R](hj: HashCoGroup[K, V, V2, R], recurse: FunctionK[TypedPipe, mat.TP]): mat.TP[(K, R)] = { import TypedPipe._ val exright: M[HashJoinable[K, V2]] = hj.right match { - case step@IdentityReduce(_, _, _, _, _) => + case step @ IdentityReduce(_, _, _, _, _) => type TK[+Z] = TypedPipe[(K, Z)] val mappedV2 = step.evidence.subst[TK](step.mapped) mat.map(recurse(mappedV2)) { (tp: TypedPipe[(K, V2)]) => IdentityReduce[K, V2, V2](step.keyOrdering, tp, step.reducers, step.descriptions, implicitly) } - case step@UnsortedIdentityReduce(_, _, _, _, _) => + case step @ UnsortedIdentityReduce(_, _, _, _, _) => type TK[+Z] = TypedPipe[(K, Z)] val mappedV2 = step.evidence.subst[TK](step.mapped) mat.map(recurse(mappedV2)) { (tp: TypedPipe[(K, V2)]) => UnsortedIdentityReduce[K, V2, V2](step.keyOrdering, tp, step.reducers, step.descriptions, implicitly) } - case step@IteratorMappedReduce(_, _, _, _, _) => + case step @ IteratorMappedReduce(_, _, _, _, _) => def go[A, B, C](imr: IteratorMappedReduce[A, B, C]) = mat.map(recurse(imr.mapped)) { (tp: TypedPipe[(A, B)]) => imr.copy(mapped = tp) } @@ -122,8 +123,9 @@ object WritePartitioner { } val zipped = mat.zip(recurse(hj.left), exright) - mat.map(zipped) { case (left, right) => - HashCoGroup(left, right, hj.joiner) + mat.map(zipped) { + case (left, right) => + HashCoGroup(left, right, hj.joiner) } } @@ -152,17 +154,18 @@ object WritePartitioner { } cg match { - case p@Pair(_, _, _) => + case p @ Pair(_, _, _) => def go[A, B, C](pair: Pair[K, A, B, C]): mat.TP[(K, C)] = { val mleft = handleCoGrouped(pair.larger, recurse) val mright = handleCoGrouped(pair.smaller, recurse) val both = mat.zip(mleft, mright) - mat.map(both) { case (l, r) => - CoGroupedPipe(Pair(pipeToCG(l), pipeToCG(r), pair.fn)) + mat.map(both) { + case (l, r) => + CoGroupedPipe(Pair(pipeToCG(l), pipeToCG(r), pair.fn)) } } widen(go(p)) - case wr@WithReducers(_, _) => + case wr @ WithReducers(_, _) => def go[V1 <: V](wr: WithReducers[K, V1]): mat.TP[(K, V)] = { val reds = wr.reds mat.map(handleCoGrouped(wr.on, recurse)) { (tp: TypedPipe[(K, V1)]) => @@ -178,7 +181,7 @@ object WritePartitioner { } } go(wr) - case wd@WithDescription(_, _) => + case wd @ WithDescription(_, _) => def go[V1 <: V](wd: WithDescription[K, V1]): mat.TP[(K, V)] = { val desc = wd.description mat.map(handleCoGrouped(wd.on, recurse)) { (tp: TypedPipe[(K, V1)]) => @@ -193,7 +196,7 @@ object WritePartitioner { } } go(wd) - case fk@CoGrouped.FilterKeys(_, _) => + case fk @ CoGrouped.FilterKeys(_, _) => def go[V1 <: V](fk: CoGrouped.FilterKeys[K, V1]): mat.TP[(K, V)] = { val fn = fk.fn mat.map(handleCoGrouped(fk.on, recurse)) { (tp: TypedPipe[(K, V1)]) => @@ -210,7 +213,7 @@ object WritePartitioner { } } go(fk) - case mg@MapGroup(_, _) => + case mg @ MapGroup(_, _) => def go[V1, V2 <: V](mg: MapGroup[K, V1, V2]): mat.TP[(K, V)] = { val fn = mg.fn mat.map(handleCoGrouped(mg.on, recurse)) { (tp: TypedPipe[(K, V1)]) => @@ -226,11 +229,11 @@ object WritePartitioner { } } go(mg) - case step@IdentityReduce(_, _, _, _, _) => + case step @ IdentityReduce(_, _, _, _, _) => widen(handleReduceStep(step, recurse)) // the widen trick sidesteps GADT bugs - case step@UnsortedIdentityReduce(_, _, _, _, _) => + case step @ UnsortedIdentityReduce(_, _, _, _, _) => widen(handleReduceStep(step, recurse)) - case step@IteratorMappedReduce(_, _, _, _, _) => + case step @ IteratorMappedReduce(_, _, _, _, _) => widen(handleReduceStep(step, recurse)) } } @@ -246,8 +249,8 @@ object WritePartitioner { tp match { case EmptyTypedPipe | IterablePipe(_) | SourcePipe(_) => false case CounterPipe(a) => isLogicalReduce(a) - case cp@CrossPipe(_, _) => isLogicalReduce(cp.viaHashJoin) - case cp@CrossValue(_, _) => isLogicalReduce(cp.viaHashJoin) + case cp @ CrossPipe(_, _) => isLogicalReduce(cp.viaHashJoin) + case cp @ CrossValue(_, _) => isLogicalReduce(cp.viaHashJoin) case DebugPipe(p) => isLogicalReduce(p) case FilterKeys(p, _) => isLogicalReduce(p) case Filter(p, _) => isLogicalReduce(p) @@ -306,7 +309,7 @@ object WritePartitioner { mat.map(rec((cp.pipe, bs)))(CounterPipe(_: TypedPipe[(a, Iterable[((String, String), Long)])])) case ((c: CrossPipe[a, b], bs), rec) => rec((c.viaHashJoin, bs)) - case ((cv@CrossValue(_, _), bs), rec) => + case ((cv @ CrossValue(_, _), bs), rec) => rec((cv.viaHashJoin, bs)) case ((p: DebugPipe[a], bs), rec) => mat.map(rec((p.input, bs)))(DebugPipe(_: TypedPipe[a])) @@ -322,10 +325,10 @@ object WritePartitioner { mat.map(rec((p.input, bs | OnlyMapping)))(FlatMapValues(_: TypedPipe[(a, b)], p.fn)) case ((p: FlatMapped[a, b], bs), rec) => mat.map(rec((p.input, bs | OnlyMapping)))(FlatMapped(_: TypedPipe[a], p.fn)) - case ((ForceToDisk(src@IterablePipe(_)), bs), rec) => + case ((ForceToDisk(src @ IterablePipe(_)), bs), rec) => // no need to put a checkpoint here: rec((src, bs)) - case ((ForceToDisk(src@SourcePipe(_)), bs), rec) => + case ((ForceToDisk(src @ SourcePipe(_)), bs), rec) => // no need to put a checkpoint here: rec((src, bs)) case ((p: ForceToDisk[a], bs), rec) => @@ -339,7 +342,7 @@ object WritePartitioner { matP case _ => mat.materialize(matP) } - case ((it@IterablePipe(_), _), _) => + case ((it @ IterablePipe(_), _), _) => mat.pure(it) case ((p: MapValues[a, b, c], bs), rec) => mat.map(rec((p.input, bs | OnlyMapping)))(MapValues(_: TypedPipe[(a, b)], p.fn)) @@ -350,7 +353,7 @@ object WritePartitioner { val mright = rec((p.right, bs)) val both = mat.zip(mleft, mright) mat.map(both) { case (l, r) => MergedTypedPipe(l, r) } - case ((src@SourcePipe(_), _), _) => + case ((src @ SourcePipe(_), _), _) => mat.pure(src) case ((p: SumByLocalKeys[a, b], bs), rec) => mat.map(rec((p.input, bs | OnlyMapping)))(SumByLocalKeys(_: TypedPipe[(a, b)], p.semigroup)) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/AsyncFlowDefRunner.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/AsyncFlowDefRunner.scala index 66bd90e0ad..ae0e07c6e5 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/AsyncFlowDefRunner.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/AsyncFlowDefRunner.scala @@ -36,7 +36,8 @@ object AsyncFlowDefRunner { * We send messages from other threads into the submit thread here */ private sealed trait FlowDefAction - private final case class RunFlowDef(conf: Config, + private final case class RunFlowDef( + conf: Config, fd: FlowDef, result: CPromise[(Long, JobStats)]) extends FlowDefAction private final case class StopFlow(flow: Flow[_], result: Promise[Unit]) extends FlowDefAction @@ -119,14 +120,16 @@ class AsyncFlowDefRunner(mode: CascadingMode) extends Writer { * Returns true if we actually add this optimized pipe. We do this * because we don't want to take the side effect twice. */ - def addForce[T](c: Config, + def addForce[T]( + c: Config, init: TypedPipe[T], opt: TypedPipe[T], p: Future[TypedPipe[T]]): (State, Boolean) = forcedPipes.get((c, opt)) match { case None => - (copy(forcedPipes = forcedPipes + ((c, opt) -> p), + (copy( + forcedPipes = forcedPipes + ((c, opt) -> p), initToOpt = initToOpt + ((c, init) -> opt)), true) case Some(_) => (copy(initToOpt = initToOpt + ((c, init) -> opt)), false) @@ -255,7 +258,8 @@ class AsyncFlowDefRunner(mode: CascadingMode) extends Writer { * calls runFlowDef, then clears the FlowStateMap */ def validateAndRun(conf: Config)(fn: Config => FlowDef)( - implicit cec: ConcurrentExecutionContext): CFuture[(Long, ExecutionCounters)] = { + implicit + cec: ConcurrentExecutionContext): CFuture[(Long, ExecutionCounters)] = { val tFlowDef = Try(fn(conf)).map { flowDef => FlowStateMap.validateSources(flowDef, mode) flowDef @@ -394,7 +398,7 @@ class AsyncFlowDefRunner(mode: CascadingMode) extends Writer { uuid: UUID, conf: Config, pipe: TypedPipe[T] // note, we don't use this, but it fixes the type T - ): (typed.TypedSink[T], () => TypedPipe[T], Option[String]) = + ): (typed.TypedSink[T], () => TypedPipe[T], Option[String]) = mode match { case _: CascadingLocal => // Local or Test mode diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/CascadingBackend.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/CascadingBackend.scala index 894fe7d4d2..82a343cf88 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/CascadingBackend.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/CascadingBackend.scala @@ -8,9 +8,27 @@ import com.stripe.dagon.{ FunctionK, HCache, Id, Rule, Dag } import com.twitter.scalding.TupleConverter.{ singleConverter, tuple2Converter } import com.twitter.scalding.TupleSetter.{ singleSetter, tup2Setter } import com.twitter.scalding.{ - CleanupIdentityFunction, Config, Dsl, Execution, Field, FlowState, FlowStateMap, GroupBuilder, - HadoopMode, IncrementCounters, IterableSource, MapsideReduce, Mode, RichFlowDef, - RichPipe, TupleConverter, TupleGetter, TupleSetter, TypedBufferOp, WrappedJoiner, Write + CleanupIdentityFunction, + Config, + Dsl, + Execution, + Field, + FlowState, + FlowStateMap, + GroupBuilder, + HadoopMode, + IncrementCounters, + IterableSource, + MapsideReduce, + Mode, + RichFlowDef, + RichPipe, + TupleConverter, + TupleGetter, + TupleSetter, + TypedBufferOp, + WrappedJoiner, + Write } import com.twitter.scalding.typed._ import com.twitter.scalding.typed.functions.{ FilterKeysToFilter, MapValuesToMap, FlatMapValuesToFlatMap } @@ -84,7 +102,8 @@ object CascadingBackend { val (boxfn, cls) = Boxed.nextCached[K](if (ordser.isInstanceOf[EquivSerialization[_]]) Some(ordser) else None) val boxordSer = BoxedOrderedSerialization(boxfn, ordser) - WrappedSerialization.rawSetBinary(List((cls, boxordSer)), + WrappedSerialization.rawSetBinary( + List((cls, boxordSer)), { case (k: String, v: String) => FlowStateMap.merge(flowDef, FlowState.withConfigSetting(k + cls, v)) @@ -115,7 +134,8 @@ object CascadingBackend { // the toPipe function directly, so we don't actually create the pipe until // the TupleSetter comes in. With this, we can make sure to use the right // TupleSetter on the final pipe - private case class CascadingPipe[+T](pipe: Pipe, + private case class CascadingPipe[+T]( + pipe: Pipe, fields: Fields, @transient localFlowDef: FlowDef, // not serializable. converter: TupleConverter[_ <: T]) { @@ -207,13 +227,13 @@ object CascadingBackend { private def withCachePolicy[U]: TypedPipe[U] => CascadingPipe[U] = { // Don't cache `CrossPipe`, but cache `left` and `right` side of it - case cp@CrossPipe(left, right) => + case cp @ CrossPipe(left, right) => notCached(excludes = Set(left, right))(cp) // Don't cache `Fork` and `WithDescriptionTypedPipe` // since if we do cache them `CrossPipe` will end up being cached as well - case tp@Fork(_) => + case tp @ Fork(_) => transform(tp, this) - case tp@WithDescriptionTypedPipe(_, _) => + case tp @ WithDescriptionTypedPipe(_, _) => transform(tp, this) // Cache all other typed pipes case tp => @@ -229,9 +249,8 @@ object CascadingBackend { private def transform[T]( pipe: TypedPipe[T], - rec: FunctionK[TypedPipe, CascadingPipe] - ): CascadingPipe[T] = pipe match { - case cp@CounterPipe(_) => + rec: FunctionK[TypedPipe, CascadingPipe]): CascadingPipe[T] = pipe match { + case cp @ CounterPipe(_) => def go[A](cp: CounterPipe[A]): CascadingPipe[A] = { val CascadingPipe(pipe0, initF, fd, conv) = rec(cp.pipe) val cpipe = RichPipe(pipe0) @@ -241,9 +260,9 @@ object CascadingBackend { } go(cp) - case cp@CrossPipe(_, _) => + case cp @ CrossPipe(_, _) => rec(cp.viaHashJoin) - case cv@CrossValue(_, _) => + case cv @ CrossValue(_, _) => rec(cv.viaHashJoin) case DebugPipe(p) => val inner = rec(p) @@ -251,14 +270,14 @@ object CascadingBackend { case EmptyTypedPipe => // just use an empty iterable pipe. rec(IterablePipe(List.empty[T])) - case fk@FilterKeys(_, _) => + case fk @ FilterKeys(_, _) => def go[K, V](node: FilterKeys[K, V]): CascadingPipe[(K, V)] = { val rewrite = Filter[(K, V)](node.input, FilterKeysToFilter(node.fn)) rec(rewrite) } go(fk) - case f@Filter(_, _) => + case f @ Filter(_, _) => // hand holding for type inference def go[T1 <: T](f: Filter[T1]): CascadingPipe[T] = { val Filter(input, fn) = f @@ -269,12 +288,12 @@ object CascadingBackend { } go(f) - case f@FlatMapValues(_, _) => + case f @ FlatMapValues(_, _) => def go[K, V, U](node: FlatMapValues[K, V, U]): CascadingPipe[T] = rec(FlatMapped[(K, V), (K, U)](node.input, FlatMapValuesToFlatMap(node.fn))) go(f) - case fm@FlatMapped(_, _) => + case fm @ FlatMapped(_, _) => // TODO we can optimize a flatmapped input directly and skip some tupleconverters def go[A, B <: T](fm: FlatMapped[A, B]): CascadingPipe[T] = { val CascadingPipe(pipe, initF, fd, conv) = rec(fm.input) @@ -295,12 +314,12 @@ object CascadingBackend { val fd = new FlowDef val pipe = IterableSource[T](iter, f0)(singleSetter, singleConverter).read(fd, mode) CascadingPipe.single[T](pipe, fd) - case f@MapValues(_, _) => + case f @ MapValues(_, _) => def go[K, A, B](fn: MapValues[K, A, B]): CascadingPipe[_ <: (K, B)] = rec(Mapped[(K, A), (K, B)](fn.input, MapValuesToMap(fn.fn))) go(f) - case m@Mapped(_, _) => + case m @ Mapped(_, _) => def go[A, B <: T](m: Mapped[A, B]): CascadingPipe[T] = { val Mapped(input, fn) = m val CascadingPipe(pipe, initF, fd, conv) = rec(input) @@ -311,7 +330,7 @@ object CascadingBackend { go(m) - case m@MergedTypedPipe(_, _) => + case m @ MergedTypedPipe(_, _) => OptimizationRules.unrollMerge(m) match { case Nil => rec(EmptyTypedPipe) case h :: Nil => rec(h) @@ -331,7 +350,7 @@ object CascadingBackend { val fd = new FlowDef val pipe = typedSrc.read(fd, mode) CascadingPipe[T](pipe, typedSrc.sourceFields, fd, typedSrc.converter[T]) - case sblk@SumByLocalKeys(_, _) => + case sblk @ SumByLocalKeys(_, _) => def go[K, V](sblk: SumByLocalKeys[K, V]): CascadingPipe[(K, V)] = { val cp = rec(sblk.input) val localFD = new FlowDef @@ -364,8 +383,7 @@ object CascadingBackend { @annotation.tailrec def loop[A]( t: TypedPipe[A], - acc: List[(String, Boolean)] - ): (TypedPipe[A], List[(String, Boolean)]) = + acc: List[(String, Boolean)]): (TypedPipe[A], List[(String, Boolean)]) = t match { case WithDescriptionTypedPipe(i, descs) => loop(i, descs ::: acc) @@ -381,9 +399,10 @@ object CascadingBackend { val next = new Each(cp.pipe, Fields.ALL, new CleanupIdentityFunction(fn)) cp.copy(pipe = next) - case hcg@HashCoGroup(_, _, _) => + case hcg @ HashCoGroup(_, _, _) => def go[K, V1, V2, R](hcg: HashCoGroup[K, V1, V2, R]): CascadingPipe[(K, R)] = - planHashJoin(hcg.left, + planHashJoin( + hcg.left, hcg.right, hcg.joiner, rec) @@ -419,8 +438,7 @@ object CascadingBackend { // add any explicit forces to the optimized graph Rule.orElse(List( forceHash, - OptimizationRules.RemoveDuplicateForceFork) - ))) + OptimizationRules.RemoveDuplicateForceFork)))) config.getOptimizationPhases match { case Some(tryPhases) => tryPhases.get.phases @@ -462,10 +480,11 @@ object CascadingBackend { def doWrites(writes: List[FlowStateMap.TypedWrite[_]]): Unit = { val empty = Dag.empty(OptimizationRules.toLiteral) type ToDo[A] = (Id[A], TypedSink[A]) - val (rootedDag, todos) = writes.foldLeft((empty, List.empty[ToDo[_]])) { case ((dag, items), tw) => - val (nextDag, id) = dag.addRoot(tw.pipe) - require(tw.mode == mode, s"${tw.mode} should be equal to $mode") - (nextDag, (id, tw.sink) :: items) + val (rootedDag, todos) = writes.foldLeft((empty, List.empty[ToDo[_]])) { + case ((dag, items), tw) => + val (nextDag, id) = dag.addRoot(tw.pipe) + require(tw.mode == mode, s"${tw.mode} should be equal to $mode") + (nextDag, (id, tw.sink) :: items) } val phases = defaultOptimizationRules( mode match { @@ -541,8 +560,7 @@ object CascadingBackend { // TypedPipe Some(Execution.failed(new Exception(s"expected empty FlowState other than TypedWrites, found: $fs"))) } - } - else Some(Execution.failed(new Exception(s"We can only convert Typed-API Jobs to Execution. Found non-empty FlowDef: $fd"))) + } else Some(Execution.failed(new Exception(s"We can only convert Typed-API Jobs to Execution. Found non-empty FlowDef: $fd"))) } /** @@ -551,7 +569,8 @@ object CascadingBackend { * cases where you want more direct control of the TypedPipe than * the default method gives you. */ - final def toPipeUnoptimized[U](input: TypedPipe[U], + final def toPipeUnoptimized[U]( + input: TypedPipe[U], fieldNames: Fields)(implicit flowDef: FlowDef, mode: Mode, setter: TupleSetter[U]): Pipe = { val compiler = cache.get(flowDef, mode) @@ -560,7 +579,8 @@ object CascadingBackend { * These rules are not optimizations, but actually required for Cascading to not * throw. Cascading requires certain shapes of the graphs */ - val p = OptimizationRules(input, + val p = OptimizationRules( + input, OptimizationRules.DescribeLater .orElse(OptimizationRules.DeferMerge) .orElse(OptimizationRules.DiamondToFlatMap)) @@ -580,7 +600,7 @@ object CascadingBackend { // before we need to call them below val inputsCR = cg.inputs.map(rec(_)) - import cg.{inputs, joinFunction} + import cg.{ inputs, joinFunction } // Cascading handles the first item in join differently, we have to see if it is repeated val firstCount = inputs.count(_ == inputs.head) @@ -613,7 +633,8 @@ object CascadingBackend { * not repeated. That case is below */ val NUM_OF_SELF_JOINS = firstCount - 1 - new CoGroup(assignName(toPipe[K, Any](inputs.head, kvFields, tupset)), + new CoGroup( + assignName(toPipe[K, Any](inputs.head, kvFields, tupset)), ordKeyField, NUM_OF_SELF_JOINS, outFields(firstCount), @@ -703,7 +724,8 @@ object CascadingBackend { * But the optimization is somewhat general: we often want a checkpoint * before a hashjoin is replicated */ - private def planHashJoin[K, V1, V2, R](left: TypedPipe[(K, V1)], + private def planHashJoin[K, V1, V2, R]( + left: TypedPipe[(K, V1)], right: HashJoinable[K, V2], joiner: (K, V1, Iterable[V2]) => Iterator[R], rec: FunctionK[TypedPipe, CascadingPipe]): CascadingPipe[(K, R)] = { @@ -769,11 +791,11 @@ object CascadingBackend { } rs match { - case ir@IdentityReduce(_, _, None, descriptions, _) => + case ir @ IdentityReduce(_, _, None, descriptions, _) => type CP[V] = CascadingPipe[_ <: (K, V)] // Not doing anything ir.evidence.subst[CP](mapped.copy(pipe = RichPipe.setPipeDescriptions(mapped.pipe, descriptions))) - case uir@UnsortedIdentityReduce(_, _, None, descriptions, _) => + case uir @ UnsortedIdentityReduce(_, _, None, descriptions, _) => type CP[V] = CascadingPipe[_ <: (K, V)] // Not doing anything uir.evidence.subst[CP](mapped.copy(pipe = RichPipe.setPipeDescriptions(mapped.pipe, descriptions))) @@ -782,7 +804,7 @@ object CascadingBackend { case UnsortedIdentityReduce(_, _, Some(reds), descriptions, _) => // This is weird, but it is sometimes used to force a partition groupOp { _.reducers(reds).setDescriptions(descriptions) } - case ivsr@IdentityValueSortedReduce(_, _, _, _, _, _) => + case ivsr @ IdentityValueSortedReduce(_, _, _, _, _, _) => groupOpWithValueSort(Some(ivsr.valueSort)) { gb => // If its an ordered serialization we need to unbox val mappedGB = @@ -797,20 +819,21 @@ object CascadingBackend { .reducers(ivsr.reducers.getOrElse(-1)) .setDescriptions(ivsr.descriptions) } - case vsr@ValueSortedReduce(_, _, _, _, _, _) => + case vsr @ ValueSortedReduce(_, _, _, _, _, _) => val optVOrdering = Some(vsr.valueSort) groupOpWithValueSort(optVOrdering) { // If its an ordered serialization we need to unbox // the value before handing it to the users operation _.every(new cascading.pipe.Every(_, valueField, - new TypedBufferOp[K, V1, V2](keyConverter(vsr.keyOrdering), + new TypedBufferOp[K, V1, V2]( + keyConverter(vsr.keyOrdering), valueConverter(optVOrdering), vsr.reduceFn, valueField), Fields.REPLACE)) .reducers(vsr.reducers.getOrElse(-1)) .setDescriptions(vsr.descriptions) } - case imr@IteratorMappedReduce(_, _, _, _, _) => + case imr @ IteratorMappedReduce(_, _, _, _, _) => groupOp { _.every(new cascading.pipe.Every(_, valueField, new TypedBufferOp(keyConverter(imr.keyOrdering), TupleConverter.singleConverter[V1], imr.reduceFn, valueField), Fields.REPLACE)) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/CoGroupJoiner.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/CoGroupJoiner.scala index 828aac8312..b30a4c673a 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/CoGroupJoiner.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/CoGroupJoiner.scala @@ -7,7 +7,8 @@ import com.twitter.scalding.serialization.Externalizer import scala.collection.JavaConverters._ import com.twitter.scalding.typed.MultiJoinFunction -abstract class CoGroupedJoiner[K](inputSize: Int, +abstract class CoGroupedJoiner[K]( + inputSize: Int, getter: TupleGetter[K], @transient inJoinFunction: MultiJoinFunction[K, Any]) extends CJoiner { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/DistinctCoGroupJoiner.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/DistinctCoGroupJoiner.scala index 4d420af669..5338cc9bdc 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/DistinctCoGroupJoiner.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/cascading_backend/DistinctCoGroupJoiner.scala @@ -4,7 +4,8 @@ import com.twitter.scalding.TupleGetter import com.twitter.scalding.typed.MultiJoinFunction // If all the input pipes are unique, this works: -class DistinctCoGroupJoiner[K](count: Int, +class DistinctCoGroupJoiner[K]( + count: Int, getter: TupleGetter[K], @transient joinF: MultiJoinFunction[K, Any]) extends CoGroupedJoiner[K](count, getter, joinF) { diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/functions/FlatMappedFn.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/functions/FlatMappedFn.scala index ba858e3bb0..68e77f5cb5 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/functions/FlatMappedFn.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/functions/FlatMappedFn.scala @@ -103,7 +103,7 @@ object FlatMappedFn extends Serializable { } def asFilter[A, B](f: FlatMappedFn[A, B]): Option[(A => Boolean, EqTypes[(_ >: A), (_ <: B)])] = f match { - case Single(filter@FlatMapping.Filter(_, _)) => Some((filter.fn, filter.ev)) + case Single(filter @ FlatMapping.Filter(_, _)) => Some((filter.fn, filter.ev)) case _ => None } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/functions/SubTypes.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/functions/SubTypes.scala index e1b8a8ad6f..0a322f3cdd 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/functions/SubTypes.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/functions/SubTypes.scala @@ -19,7 +19,8 @@ sealed abstract class SubTypes[-A, +B] extends java.io.Serializable { type G[-T] = SubTypes[F[T], F[B]] subst[G](SubTypes.fromSubType[F[B], F[B]]) } - /** create a new evidence for a contravariant type F[_] + /** + * create a new evidence for a contravariant type F[_] */ def liftContra[F[-_]]: SubTypes[F[B], F[A]] = { type G[-T] = SubTypes[F[B], F[T]] @@ -56,8 +57,8 @@ object SubTypes extends java.io.Serializable { } def compose[A, B, C](sub0: SubTypes[A, B], sub1: SubTypes[B, C]): SubTypes[A, C] = { - type SubC[-X] = SubTypes[X, C] - sub0.subst[SubC](sub1) + type SubC[-X] = SubTypes[X, C] + sub0.subst[SubC](sub1) } } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/memory_backend/MemoryBackend.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/memory_backend/MemoryBackend.scala index 8b8971fa08..94864c7396 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/memory_backend/MemoryBackend.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/memory_backend/MemoryBackend.scala @@ -5,7 +5,7 @@ import cascading.pipe.Pipe import com.twitter.scalding.typed._ import com.twitter.scalding.Mode import scala.concurrent.{ Future, ExecutionContext => ConcurrentExecutionContext, Promise } -import scala.util.{Failure, Success} +import scala.util.{ Failure, Success } trait MemorySource[A] { def read()(implicit ec: ConcurrentExecutionContext): Future[Iterator[A]] diff --git a/scalding-core/src/main/scala/com/twitter/scalding/typed/memory_backend/Op.scala b/scalding-core/src/main/scala/com/twitter/scalding/typed/memory_backend/Op.scala index 676b0b11db..87e500d418 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/typed/memory_backend/Op.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/typed/memory_backend/Op.scala @@ -13,7 +13,7 @@ sealed trait Op[+O] { transform { in: IndexedSeq[O] => val res = ArrayBuffer[O1]() val it = in.iterator - while(it.hasNext) { + while (it.hasNext) { val i = it.next fn(i).foreach(res += _) } @@ -51,7 +51,7 @@ object Op { case None => val promise = Promise[ArrayBuffer[_ <: O]]() (Some(promise), Right(promise)) - case s@Some(promise) => + case s @ Some(promise) => (s, Left(promise)) } @@ -72,9 +72,10 @@ object Op { def result(implicit cec: ConcurrentExecutionContext) = { val f1 = left.result val f2 = right.result - f1.zip(f2).map { case (l, r) => - if (l.size > r.size) l.asInstanceOf[ArrayBuffer[O]] ++= r - else r.asInstanceOf[ArrayBuffer[O]] ++= l + f1.zip(f2).map { + case (l, r) => + if (l.size > r.size) l.asInstanceOf[ArrayBuffer[O]] ++= r + else r.asInstanceOf[ArrayBuffer[O]] ++= l } } } @@ -85,7 +86,7 @@ object Op { input.result.map { array => val res: ArrayBuffer[O] = array.asInstanceOf[ArrayBuffer[O]] var pos = 0 - while(pos < array.length) { + while (pos < array.length) { res.update(pos, fn(array(pos))) pos = pos + 1 } @@ -99,7 +100,7 @@ object Op { val array = array0.asInstanceOf[ArrayBuffer[I]] var pos = 0 var writePos = 0 - while(pos < array.length) { + while (pos < array.length) { val item = array(pos) if (fn(item)) { array(writePos) = item @@ -129,8 +130,7 @@ object Op { final case class Reduce[K, V1, V2]( input: Op[(K, V1)], fn: (K, Iterator[V1]) => Iterator[V2], - ord: Option[Ordering[V1]] - ) extends Op[(K, V2)] { + ord: Option[Ordering[V1]]) extends Op[(K, V2)] { def result(implicit cec: ConcurrentExecutionContext): Future[ArrayBuffer[(K, V2)]] = input.result.map { kvs => @@ -146,15 +146,16 @@ object Op { * the keys into as many groups as there are CPUs and process that way */ val res = ArrayBuffer[(K, V2)]() - valuesByKey.foreach { case (k, vs) => - ord.foreach(Collections.sort[V1](vs, _)) - val v2iter = fn(k, vs.iterator.asScala) - while(v2iter.hasNext) { - res += ((k, v2iter.next)) - } + valuesByKey.foreach { + case (k, vs) => + ord.foreach(Collections.sort[V1](vs, _)) + val v2iter = fn(k, vs.iterator.asScala) + while (v2iter.hasNext) { + res += ((k, v2iter.next)) + } } res - } + } } final case class Join[A, B, C]( diff --git a/scalding-date/src/main/scala/com/twitter/scalding/AbsoluteDuration.scala b/scalding-date/src/main/scala/com/twitter/scalding/AbsoluteDuration.scala index 23fc883f3a..77a3b2cbf6 100644 --- a/scalding-date/src/main/scala/com/twitter/scalding/AbsoluteDuration.scala +++ b/scalding-date/src/main/scala/com/twitter/scalding/AbsoluteDuration.scala @@ -32,7 +32,8 @@ object AbsoluteDuration extends java.io.Serializable { val SEC_IN_MS = 1000 val MIN_IN_MS = 60 * SEC_IN_MS val HOUR_IN_MS = 60 * MIN_IN_MS - val UTC_UNITS = List[TimeCons]((Hours, HOUR_IN_MS), + val UTC_UNITS = List[TimeCons]( + (Hours, HOUR_IN_MS), (Minutes, MIN_IN_MS), (Seconds, SEC_IN_MS), (Millisecs, 1)).reverse @@ -79,7 +80,8 @@ object AbsoluteDuration extends java.io.Serializable { // We can't go any further, try to jam the rest into this unit: val (fn, cnt) = tc val theseUnits = diffInMs / cnt - require((theseUnits <= Int.MaxValue) && (theseUnits >= Int.MinValue), + require( + (theseUnits <= Int.MaxValue) && (theseUnits >= Int.MinValue), "diff not representable in an Int: " + theseUnits + AbsoluteDurationList(acc) + "total: " + (diffInMs + AbsoluteDurationList(acc).toMillisecs)) val thisPart = fn(theseUnits.toInt) diff --git a/scalding-date/src/main/scala/com/twitter/scalding/DateRange.scala b/scalding-date/src/main/scala/com/twitter/scalding/DateRange.scala index 7d2c6be508..72dadb5a7c 100644 --- a/scalding-date/src/main/scala/com/twitter/scalding/DateRange.scala +++ b/scalding-date/src/main/scala/com/twitter/scalding/DateRange.scala @@ -38,7 +38,8 @@ object DateRange extends java.io.Serializable { * could be construed as matching the string passed, e.g. * ("2011-01-02T04", "2011-01-02T05") includes two full hours (all of 4 and all of 5) */ - def parse(iso8601start: String, + def parse( + iso8601start: String, iso8601inclusiveUpper: String)(implicit tz: TimeZone, dp: DateParser): DateRange = { val start = RichDate(iso8601start) diff --git a/scalding-db/src/main/scala/com/twitter/scalding/db/ColumnDefinition.scala b/scalding-db/src/main/scala/com/twitter/scalding/db/ColumnDefinition.scala index 5dbed942d2..e26ad9aa04 100644 --- a/scalding-db/src/main/scala/com/twitter/scalding/db/ColumnDefinition.scala +++ b/scalding-db/src/main/scala/com/twitter/scalding/db/ColumnDefinition.scala @@ -21,7 +21,8 @@ import com.twitter.scalding.TupleConverter case class ColumnName(toStr: String) extends AnyVal case class SqlTypeName(toStr: String) extends AnyVal -case class ColumnDefinition(jdbcType: SqlType, +case class ColumnDefinition( + jdbcType: SqlType, name: ColumnName, nullable: IsNullable, sizeOpt: Option[Int], diff --git a/scalding-db/src/main/scala/com/twitter/scalding/db/DBColumnTransformer.scala b/scalding-db/src/main/scala/com/twitter/scalding/db/DBColumnTransformer.scala index fd3c7dc9d6..4bae0c8c05 100644 --- a/scalding-db/src/main/scala/com/twitter/scalding/db/DBColumnTransformer.scala +++ b/scalding-db/src/main/scala/com/twitter/scalding/db/DBColumnTransformer.scala @@ -20,7 +20,8 @@ package com.twitter.scalding.db case class Definition(toStr: String) extends AnyVal object DBColumnDefinition { - def apply(col: ColumnDefinition): DBColumnDefinition = DBColumnDefinition(col.jdbcType, + def apply(col: ColumnDefinition): DBColumnDefinition = DBColumnDefinition( + col.jdbcType, col.name, col.nullable, col.sizeOpt, @@ -28,7 +29,8 @@ object DBColumnDefinition { SqlTypeName(col.jdbcType.toString)) } -case class DBColumnDefinition(jdbcType: SqlType, +case class DBColumnDefinition( + jdbcType: SqlType, name: ColumnName, nullable: IsNullable, sizeOpt: Option[Int], @@ -36,7 +38,8 @@ case class DBColumnDefinition(jdbcType: SqlType, sqlType: SqlTypeName) object DBColumnTransformer { - def columnDefnToDefinition(col: ColumnDefinition, + def columnDefnToDefinition( + col: ColumnDefinition, columnMutator: PartialFunction[DBColumnDefinition, DBColumnDefinition]): Definition = { val preparedCol = columnMutator(DBColumnDefinition(col)) val sizeStr = preparedCol.sizeOpt.map { siz => s"($siz)" }.getOrElse("") @@ -55,11 +58,13 @@ object DBColumnTransformer { case t => t } - def mutateColumns(columnMutator: PartialFunction[DBColumnDefinition, DBColumnDefinition], + def mutateColumns( + columnMutator: PartialFunction[DBColumnDefinition, DBColumnDefinition], columns: Iterable[ColumnDefinition]): Iterable[DBColumnDefinition] = columns.map(c => columnMutator.orElse(defaultColumnMutator)(DBColumnDefinition(c))) - def columnDefnsToCreate(columnMutator: PartialFunction[DBColumnDefinition, DBColumnDefinition], + def columnDefnsToCreate( + columnMutator: PartialFunction[DBColumnDefinition, DBColumnDefinition], columns: Iterable[ColumnDefinition]): Iterable[Definition] = columns.map(c => columnDefnToDefinition(c, columnMutator.orElse(defaultColumnMutator))) diff --git a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/ColumnDefinitionProviderImpl.scala b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/ColumnDefinitionProviderImpl.scala index 5bf97a8c00..7e1a0861a1 100644 --- a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/ColumnDefinitionProviderImpl.scala +++ b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/ColumnDefinitionProviderImpl.scala @@ -49,7 +49,8 @@ object ColumnDefinitionProviderImpl { // Field To JDBCColumn @tailrec - def matchField(accessorTree: List[MethodSymbol], + def matchField( + accessorTree: List[MethodSymbol], oTpe: Type, fieldName: FieldName, defaultValOpt: Option[c.Expr[String]], @@ -141,7 +142,8 @@ object ColumnDefinitionProviderImpl { .keys if (duplicateFields.nonEmpty) { - c.abort(c.enclosingPosition, + c.abort( + c.enclosingPosition, s""" Duplicate field names found: ${duplicateFields.mkString(",")}. Please check your nested case classes. diff --git a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/JdbcStatementSetterImpl.scala b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/JdbcStatementSetterImpl.scala index 9d5a621721..81b4be3a05 100644 --- a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/JdbcStatementSetterImpl.scala +++ b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/JdbcStatementSetterImpl.scala @@ -25,7 +25,8 @@ import com.twitter.scalding.db.JdbcStatementSetter */ private[macros] object JdbcStatementSetterImpl { - def caseClassJdbcSetterCommonImpl[T](c: Context, + def caseClassJdbcSetterCommonImpl[T]( + c: Context, allowUnknownTypes: Boolean)(implicit T: c.WeakTypeTag[T]): c.Expr[JdbcStatementSetter[T]] = { import c.universe._ diff --git a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/ColumnFormat.scala b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/ColumnFormat.scala index 2ed8dbca40..8d8e7ad6d7 100644 --- a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/ColumnFormat.scala +++ b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/ColumnFormat.scala @@ -5,7 +5,8 @@ import scala.reflect.macros.Context import com.twitter.scalding.db.macros.impl.FieldName object ColumnFormat { - def apply(c: Context)(fAccessor: List[c.universe.MethodSymbol], fType: String, size: Option[Int])(implicit fName: FieldName, + def apply(c: Context)(fAccessor: List[c.universe.MethodSymbol], fType: String, size: Option[Int])(implicit + fName: FieldName, isNullable: Boolean, defaultV: Option[c.Expr[String]]): ColumnFormat[c.type] = { new ColumnFormat[c.type](c) { diff --git a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/DateTypeHandler.scala b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/DateTypeHandler.scala index 736ae6cdda..3cf1d9c47f 100644 --- a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/DateTypeHandler.scala +++ b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/DateTypeHandler.scala @@ -7,7 +7,8 @@ import com.twitter.scalding.db.macros.impl.FieldName object DateTypeHandler { - def apply[T](c: Context)(implicit accessorTree: List[c.universe.MethodSymbol], + def apply[T](c: Context)(implicit + accessorTree: List[c.universe.MethodSymbol], fieldName: FieldName, defaultValue: Option[c.Expr[String]], annotationInfo: List[(c.universe.Type, Option[Int])], diff --git a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/NumericTypeHandler.scala b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/NumericTypeHandler.scala index 43058feae6..69532cd051 100644 --- a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/NumericTypeHandler.scala +++ b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/NumericTypeHandler.scala @@ -6,7 +6,8 @@ import scala.util.{ Success, Failure } import com.twitter.scalding.db.macros.impl.FieldName object NumericTypeHandler { - def apply[T](c: Context)(implicit accessorTree: List[c.universe.MethodSymbol], + def apply[T](c: Context)(implicit + accessorTree: List[c.universe.MethodSymbol], fieldName: FieldName, defaultValue: Option[c.Expr[String]], annotationInfo: List[(c.universe.Type, Option[Int])], diff --git a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/StringTypeHandler.scala b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/StringTypeHandler.scala index 83864fcc3e..10ae008020 100644 --- a/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/StringTypeHandler.scala +++ b/scalding-db/src/main/scala/com/twitter/scalding/db/macros/impl/handler/StringTypeHandler.scala @@ -6,7 +6,8 @@ import scala.util.{ Success, Failure } import com.twitter.scalding.db.macros.impl.FieldName object StringTypeHandler { - def apply[T](c: Context)(implicit accessorTree: List[c.universe.MethodSymbol], + def apply[T](c: Context)(implicit + accessorTree: List[c.universe.MethodSymbol], fieldName: FieldName, defaultValue: Option[c.Expr[String]], annotationInfo: List[(c.universe.Type, Option[Int])], diff --git a/scalding-hraven/src/main/scala/com/twitter/scalding/hraven/estimation/HRavenHistoryService.scala b/scalding-hraven/src/main/scala/com/twitter/scalding/hraven/estimation/HRavenHistoryService.scala index 084e5ad18b..ae8e801bbd 100644 --- a/scalding-hraven/src/main/scala/com/twitter/scalding/hraven/estimation/HRavenHistoryService.scala +++ b/scalding-hraven/src/main/scala/com/twitter/scalding/hraven/estimation/HRavenHistoryService.scala @@ -23,7 +23,8 @@ object HRavenClient { def apply(conf: JobConf): Try[HRavenRestClient] = conf.getFirstKey(apiHostnameKey) - .map(new HRavenRestClient(_, + .map(new HRavenRestClient( + _, conf.getInt(clientConnectTimeoutKey, clientConnectTimeoutDefault), conf.getInt(clientReadTimeoutKey, clientReadTimeoutDefault))) } diff --git a/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/Parquet346ScroogeScheme.scala b/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/Parquet346ScroogeScheme.scala index afc15ebc6b..bcbcafe5e3 100644 --- a/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/Parquet346ScroogeScheme.scala +++ b/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/Parquet346ScroogeScheme.scala @@ -31,7 +31,8 @@ import scala.util.control.NonFatal class Parquet346ScroogeScheme[T <: ThriftStruct](config: ParquetValueScheme.Config[T]) extends ParquetScroogeScheme[T](config) { - override def sourceConfInit(fp: FlowProcess[JobConf], + override def sourceConfInit( + fp: FlowProcess[JobConf], tap: Tap[JobConf, RecordReader[_, _], OutputCollector[_, _]], jobConf: JobConf): Unit = { @@ -67,7 +68,8 @@ object Parquet346ScroogeRecordConverter { * ThriftSchemaConverter to not throw, but we can put dummy data in there because it's not actually * used. */ -class Parquet346ScroogeRecordConverter[T <: ThriftStruct](thriftClass: Class[T], +class Parquet346ScroogeRecordConverter[T <: ThriftStruct]( + thriftClass: Class[T], parquetSchema: MessageType, thriftType: StructType) extends ThriftRecordConverter[T]( // this is a little confusing because it's all being passed to the super constructor diff --git a/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/PartitionedParquetScroogeSource.scala b/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/PartitionedParquetScroogeSource.scala index 4f958a61a0..13f1ba5d59 100644 --- a/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/PartitionedParquetScroogeSource.scala +++ b/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/PartitionedParquetScroogeSource.scala @@ -29,7 +29,8 @@ import scala.reflect.ClassTag * }}} * */ -case class PartitionedParquetScroogeSource[P, T <: ThriftStruct](path: String, template: String)(implicit val ct: ClassTag[T], +case class PartitionedParquetScroogeSource[P, T <: ThriftStruct](path: String, template: String)(implicit + val ct: ClassTag[T], val valueSetter: TupleSetter[T], val valueConverter: TupleConverter[T], val partitionSetter: TupleSetter[P], val partitionConverter: TupleConverter[P]) extends FixedPathSource(path) with ParquetThriftBase[T] with PartitionSchemed[P, T] with Serializable { diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/HasColumnProjection.scala b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/HasColumnProjection.scala index 3a14111a48..615a76beeb 100644 --- a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/HasColumnProjection.scala +++ b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/HasColumnProjection.scala @@ -42,7 +42,8 @@ trait HasColumnProjection { val deprecated = withColumns val strict = withColumnProjections - require(deprecated.isEmpty || strict.isEmpty, + require( + deprecated.isEmpty || strict.isEmpty, "Cannot provide both withColumns and withColumnProjections") deprecated.foreach(requireNoSemiColon) diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/Parquet346TBaseScheme.scala b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/Parquet346TBaseScheme.scala index d71623dcd4..65f0deb5b2 100644 --- a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/Parquet346TBaseScheme.scala +++ b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/Parquet346TBaseScheme.scala @@ -34,7 +34,8 @@ import scala.collection.JavaConverters._ class Parquet346TBaseScheme[T <: TBase[_, _]](config: ParquetValueScheme.Config[T]) extends ParquetTBaseScheme[T](config) { - override def sourceConfInit(fp: FlowProcess[JobConf], + override def sourceConfInit( + fp: FlowProcess[JobConf], tap: Tap[JobConf, RecordReader[_, _], OutputCollector[_, _]], jobConf: JobConf): Unit = { @@ -52,7 +53,8 @@ class Parquet346TBaseScheme[T <: TBase[_, _]](config: ParquetValueScheme.Config[ * ThriftSchemaConverter to not throw, but we can put dummy data in there because it's not actually * used. */ -class Parquet346TBaseRecordConverter[T <: TBase[_, _]](thriftClass: Class[T], +class Parquet346TBaseRecordConverter[T <: TBase[_, _]]( + thriftClass: Class[T], requestedParquetSchema: MessageType, thriftType: ThriftType.StructType) extends ThriftRecordConverter[T]( // this is a little confusing because it's all being passed to the super constructor diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/PartitionedParquetThriftSource.scala b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/PartitionedParquetThriftSource.scala index b24ca3cc3f..6e88a6d878 100644 --- a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/PartitionedParquetThriftSource.scala +++ b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/PartitionedParquetThriftSource.scala @@ -27,7 +27,8 @@ import scala.reflect.ClassTag * }}} * */ -case class PartitionedParquetThriftSource[P, T <: ParquetThrift.ThriftBase](path: String, template: String)(implicit val ct: ClassTag[T], +case class PartitionedParquetThriftSource[P, T <: ParquetThrift.ThriftBase](path: String, template: String)(implicit + val ct: ClassTag[T], val valueSetter: TupleSetter[T], val valueConverter: TupleConverter[T], val partitionSetter: TupleSetter[P], val partitionConverter: TupleConverter[P]) extends FixedPathSource(path) with ParquetThriftBase[T] with PartitionSchemed[P, T] with Serializable { diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/macros/impl/ParquetReadSupportProvider.scala b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/macros/impl/ParquetReadSupportProvider.scala index 3c7cf49d97..137e252549 100644 --- a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/macros/impl/ParquetReadSupportProvider.scala +++ b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/macros/impl/ParquetReadSupportProvider.scala @@ -18,7 +18,8 @@ class ParquetReadSupportProvider(schemaProvider: ParquetSchemaProvider) { import ctx.universe._ if (!IsCaseClassImpl.isCaseClassType(ctx)(T.tpe)) - ctx.abort(ctx.enclosingPosition, + ctx.abort( + ctx.enclosingPosition, s"""We cannot enforce ${T.tpe} is a case class, either it is not a case class or this macro call is possibly enclosed in a class. This will mean the macro is operating on a non-resolved type.""") diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/macros/impl/WriteSupportProvider.scala b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/macros/impl/WriteSupportProvider.scala index 2414907d35..2c469bcb26 100644 --- a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/macros/impl/WriteSupportProvider.scala +++ b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/macros/impl/WriteSupportProvider.scala @@ -11,7 +11,8 @@ class WriteSupportProvider(schemaProvider: ParquetSchemaProvider) { import ctx.universe._ if (!IsCaseClassImpl.isCaseClassType(ctx)(T.tpe)) - ctx.abort(ctx.enclosingPosition, + ctx.abort( + ctx.enclosingPosition, s"""We cannot enforce ${T.tpe} is a case class, either it is not a case class or this macro call is possibly enclosed in a class. This will mean the macro is operating on a non-resolved type.""") diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/scheme/TypedParquetTupleScheme.scala b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/scheme/TypedParquetTupleScheme.scala index a5ec064b7e..0ea5c756ed 100644 --- a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/scheme/TypedParquetTupleScheme.scala +++ b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/scheme/TypedParquetTupleScheme.scala @@ -168,7 +168,8 @@ class TypedParquetTupleScheme[T](val readSupport: ParquetReadSupport[T], val wri override def sink(flowProcess: FlowProcess[JobConf], sinkCall: SinkCallType): Unit = { val tuple = sinkCall.getOutgoingEntry - require(tuple.size == 1, + require( + tuple.size == 1, "TypedParquetTupleScheme expects tuple with an arity of exactly 1, but found " + tuple.getFields) val value = tuple.getObject(0).asInstanceOf[T] val outputCollector = sinkCall.getOutput diff --git a/scalding-quotation/src/main/scala/com/twitter/scalding/quotation/TextMacro.scala b/scalding-quotation/src/main/scala/com/twitter/scalding/quotation/TextMacro.scala index f5538c9969..befcbf4efe 100644 --- a/scalding-quotation/src/main/scala/com/twitter/scalding/quotation/TextMacro.scala +++ b/scalding-quotation/src/main/scala/com/twitter/scalding/quotation/TextMacro.scala @@ -64,7 +64,7 @@ trait TextMacro { '[' -> ']') /* - * Reads the parameters block. It takes in consideration nested blocks like `map(v => { ... })` + * Reads the parameters block. It takes in consideration nested blocks like `map(v => { ... })` */ def readParams(chars: List[Char], open: List[Char], acc: List[Char] = Nil): (List[Char], List[Char]) = chars match { diff --git a/scalding-repl/src/main/scala/com/twitter/scalding/ReplImplicits.scala b/scalding-repl/src/main/scala/com/twitter/scalding/ReplImplicits.scala index 33006b447d..19757865af 100644 --- a/scalding-repl/src/main/scala/com/twitter/scalding/ReplImplicits.scala +++ b/scalding-repl/src/main/scala/com/twitter/scalding/ReplImplicits.scala @@ -244,8 +244,9 @@ object ReplImplicits extends FieldConversions { * @return a Source backed by the specified iterable. */ implicit def iterableToSource[T]( - iterable: Iterable[T])(implicit setter: TupleSetter[T], - converter: TupleConverter[T]): Source = { + iterable: Iterable[T])(implicit + setter: TupleSetter[T], + converter: TupleConverter[T]): Source = { IterableSource[T](iterable)(setter, converter) } @@ -258,8 +259,9 @@ object ReplImplicits extends FieldConversions { * @return a Pipe backed by the specified iterable. */ implicit def iterableToPipe[T]( - iterable: Iterable[T])(implicit setter: TupleSetter[T], - converter: TupleConverter[T], flowDef: FlowDef, mode: Mode): Pipe = { + iterable: Iterable[T])(implicit + setter: TupleSetter[T], + converter: TupleConverter[T], flowDef: FlowDef, mode: Mode): Pipe = { iterableToSource(iterable)(setter, converter).read } @@ -273,8 +275,9 @@ object ReplImplicits extends FieldConversions { * @return a RichPipe backed by the specified iterable. */ implicit def iterableToRichPipe[T]( - iterable: Iterable[T])(implicit setter: TupleSetter[T], - converter: TupleConverter[T], flowDef: FlowDef, mode: Mode): RichPipe = { + iterable: Iterable[T])(implicit + setter: TupleSetter[T], + converter: TupleConverter[T], flowDef: FlowDef, mode: Mode): RichPipe = { RichPipe(iterableToPipe(iterable)(setter, converter, flowDef, mode)) } diff --git a/scalding-repl/src/main/scala/com/twitter/scalding/ScaldingShell.scala b/scalding-repl/src/main/scala/com/twitter/scalding/ScaldingShell.scala index cd312010ea..42e0a35c33 100644 --- a/scalding-repl/src/main/scala/com/twitter/scalding/ScaldingShell.scala +++ b/scalding-repl/src/main/scala/com/twitter/scalding/ScaldingShell.scala @@ -138,7 +138,8 @@ trait BaseScaldingShell extends MainGenericRunner { private[scalding] def createReplCodeJar(): Option[File] = { scaldingREPL.map { repl => val virtualDirectory = repl.virtualDirectory - val tempJar = new File(Files.createTempDir(), + val tempJar = new File( + Files.createTempDir(), "scalding-repl-session-" + System.currentTimeMillis() + ".jar") createJar(virtualDirectory.asInstanceOf[VirtualDirectory], tempJar) } diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Boxed.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Boxed.scala index b386cbd780..1c7a0a9f89 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Boxed.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Boxed.scala @@ -530,7 +530,8 @@ class Boxed249[K](override val get: K) extends Boxed[K] class Boxed250[K](override val get: K) extends Boxed[K] -case class BoxedOrderedSerialization[K](box: K => Boxed[K], +case class BoxedOrderedSerialization[K]( + box: K => Boxed[K], ord: OrderedSerialization[K]) extends OrderedSerialization[Boxed[K]] { override def compare(a: Boxed[K], b: Boxed[K]) = ord.compare(a.get, b.get) diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/OrderedSerialization.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/OrderedSerialization.scala index 3b9be5bc91..d8de41d923 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/OrderedSerialization.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/OrderedSerialization.scala @@ -101,7 +101,8 @@ object OrderedSerialization { case NonFatal(e) => CompareFailure(e) } - private[this] def internalTransformer[T, U, V](packFn: T => U, + private[this] def internalTransformer[T, U, V]( + packFn: T => U, unpackFn: U => V, presentFn: Try[V] => Try[T])(implicit otherOrdSer: OrderedSerialization[U]): OrderedSerialization[T] = { @@ -152,7 +153,8 @@ object OrderedSerialization { * The the serialized comparison matches the unserialized comparison */ def compareBinaryMatchesCompare[T](implicit ordb: OrderedSerialization[T]): Law2[T] = - Law2("compare(a, b) == compareBinary(aBin, bBin)", + Law2( + "compare(a, b) == compareBinary(aBin, bBin)", { (a: T, b: T) => resultFrom(ordb.compare(a, b)) == writeThenCompare(a, b) }) /** @@ -160,7 +162,8 @@ object OrderedSerialization { * will be broken */ def orderingTransitive[T](implicit ordb: OrderedSerialization[T]): Law3[T] = - Law3("transitivity", + Law3( + "transitivity", { (a: T, b: T, c: T) => if (ordb.lteq(a, b) && ordb.lteq(b, c)) { ordb.lteq(a, c) } else true @@ -170,7 +173,8 @@ object OrderedSerialization { * will be broken */ def orderingAntisymmetry[T](implicit ordb: OrderedSerialization[T]): Law2[T] = - Law2("antisymmetry", + Law2( + "antisymmetry", { (a: T, b: T) => if (ordb.lteq(a, b) && ordb.lteq(b, a)) { ordb.equiv(a, b) } else true @@ -183,7 +187,8 @@ object OrderedSerialization { Law2("totality", { (a: T, b: T) => (ordb.lteq(a, b) || ordb.lteq(b, a)) }) def allLaws[T: OrderedSerialization]: Iterable[Law[T]] = - Serialization.allLaws ++ List[Law[T]](compareBinaryMatchesCompare[T], + Serialization.allLaws ++ List[Law[T]]( + compareBinaryMatchesCompare[T], orderingTransitive[T], orderingAntisymmetry[T], orderingTotality[T]) @@ -197,7 +202,8 @@ object OrderedSerialization { * Note: it is your responsibility that the hash in serialization is consistent * with the ordering (if equivalent in the ordering, the hash must match). */ -final case class DeserializingOrderedSerialization[T](serialization: Serialization[T], +final case class DeserializingOrderedSerialization[T]( + serialization: Serialization[T], ordering: Ordering[T]) extends OrderedSerialization[T] { final override def read(i: InputStream) = serialization.read(i) diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Serialization.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Serialization.scala index 5930260999..1e08deabc7 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Serialization.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Serialization.scala @@ -146,7 +146,8 @@ object Serialization { * The sizes must match and be correct if they are present */ def sizeLaw[T: Serialization]: Law1[T] = - Law1("staticSize.orElse(dynamicSize(t)).map { _ == toBytes(t).length }", + Law1( + "staticSize.orElse(dynamicSize(t)).map { _ == toBytes(t).length }", { (t: T) => val ser = implicitly[Serialization[T]] (ser.staticSize, ser.dynamicSize(t)) match { @@ -158,13 +159,15 @@ object Serialization { }) def transitivity[T: Serialization]: Law3[T] = - Law3("equiv(a, b) && equiv(b, c) => equiv(a, c)", + Law3( + "equiv(a, b) && equiv(b, c) => equiv(a, c)", { (t1: T, t2: T, t3: T) => !(equiv(t1, t2) && equiv(t2, t3)) || equiv(t1, t3) }) def allLaws[T: Serialization]: Iterable[Law[T]] = - List[Law[T]](roundTripLaw, + List[Law[T]]( + roundTripLaw, serializationIsEquivalence, hashCodeImpliesEquality, reflexivity, diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Serialization2.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Serialization2.scala index 27e75a7a67..c7b55ad772 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Serialization2.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Serialization2.scala @@ -68,7 +68,8 @@ object OrderedSerialization2 { } } -class OrderedSerialization2[A, B](val ordA: OrderedSerialization[A], +class OrderedSerialization2[A, B]( + val ordA: OrderedSerialization[A], val ordB: OrderedSerialization[B]) extends Serialization2[A, B](ordA, ordB) with OrderedSerialization[(A, B)] { override def compare(x: (A, B), y: (A, B)) = { val ca = ordA.compare(x._1, y._1) diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/StringOrderedSerialization.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/StringOrderedSerialization.scala index 23ab371c4d..f18e32ddaf 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/StringOrderedSerialization.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/StringOrderedSerialization.scala @@ -33,16 +33,19 @@ object StringOrderedSerialization { def compareBytes(count: Int): Int = if ((count & 2) == 2) { // there are 2 or 3 bytes to read - val cmp = Integer.compare(seekingLeft.readUnsignedShort, + val cmp = Integer.compare( + seekingLeft.readUnsignedShort, seekingRight.readUnsignedShort) if (cmp != 0) cmp - else if (count == 3) Integer.compare(seekingLeft.readUnsignedByte, + else if (count == 3) Integer.compare( + seekingLeft.readUnsignedByte, seekingRight.readUnsignedByte) else 0 } else { // there are 0 or 1 bytes to read if (count == 0) 0 - else Integer.compare(seekingLeft.readUnsignedByte, + else Integer.compare( + seekingLeft.readUnsignedByte, seekingRight.readUnsignedByte) } diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Writer.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Writer.scala index c99a4134dc..ebdd9376f0 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Writer.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Writer.scala @@ -29,7 +29,8 @@ trait Writer[@specialized(Boolean, Byte, Short, Int, Long, Float, Double) -T] { object Writer { import JavaStreamEnrichments._ - def write[@specialized(Boolean, Byte, Short, Int, Long, Float, Double) T](os: OutputStream, + def write[@specialized(Boolean, Byte, Short, Int, Long, Float, Double) T]( + os: OutputStream, t: T)(implicit w: Writer[T]): Unit = w.write(os, t) /* diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/ProductLike.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/ProductLike.scala index f246d8547f..652b067b52 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/ProductLike.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/ProductLike.scala @@ -101,12 +101,14 @@ object ProductLike { maybeLength, noLength) case f: FastLengthCalculation[_] => - (constantLength, + ( + constantLength, dynamicLength :+ f.asInstanceOf[FastLengthCalculation[c.type]].t, maybeLength, noLength) case m: MaybeLengthCalculation[_] => - (constantLength, + ( + constantLength, dynamicLength, maybeLength :+ m.asInstanceOf[MaybeLengthCalculation[c.type]].t, noLength) diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/TreeOrderedBuf.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/TreeOrderedBuf.scala index 231c9c6d28..32a2dd6f2f 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/TreeOrderedBuf.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/TreeOrderedBuf.scala @@ -34,7 +34,8 @@ object CommonCompareBinary { * potentially complex logic in binary comparators */ @SuppressWarnings(Array("org.wartremover.warts.Return")) - final def earlyEqual(inputStreamA: InputStream, + final def earlyEqual( + inputStreamA: InputStream, lenA: Int, inputStreamB: InputStream, lenB: Int): Boolean = @@ -55,8 +56,7 @@ object CommonCompareBinary { inputStreamB.reset() // yeah, return sucks, but trying to optimize here return false - } - else if (a < 0) return JavaStreamEnrichments.eof + } else if (a < 0) return JavaStreamEnrichments.eof // a == b, but may be eof } // we consumed all the bytes, and they were all equal diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/CaseClassOrderedBuf.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/CaseClassOrderedBuf.scala index 2d751feba3..ef71067ed0 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/CaseClassOrderedBuf.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/CaseClassOrderedBuf.scala @@ -29,7 +29,8 @@ object CaseClassOrderedBuf { CaseClassOrderedBuf(c)(buildDispatcher, tpe) } - def apply(c: Context)(buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], + def apply(c: Context)( + buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], outerType: c.Type): TreeOrderedBuf[c.type] = { import c.universe._ def freshT(id: String) = TermName(c.freshName(id)) diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/EitherOrderedBuf.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/EitherOrderedBuf.scala index 9120f60453..bb47e2f31a 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/EitherOrderedBuf.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/EitherOrderedBuf.scala @@ -29,7 +29,8 @@ object EitherOrderedBuf { EitherOrderedBuf(c)(buildDispatcher, tpe) } - def apply(c: Context)(buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], + def apply(c: Context)( + buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], outerType: c.Type): TreeOrderedBuf[c.type] = { import c.universe._ def freshT(id: String) = TermName(c.freshName(id)) diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/OptionOrderedBuf.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/OptionOrderedBuf.scala index 88548a1e60..2f211cc074 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/OptionOrderedBuf.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/OptionOrderedBuf.scala @@ -29,7 +29,8 @@ object OptionOrderedBuf { OptionOrderedBuf(c)(buildDispatcher, tpe) } - def apply(c: Context)(buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], + def apply(c: Context)( + buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], outerType: c.Type): TreeOrderedBuf[c.type] = { import c.universe._ def freshT(id: String) = TermName(c.freshName(id)) diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/ProductOrderedBuf.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/ProductOrderedBuf.scala index cffe87354e..cea89a171c 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/ProductOrderedBuf.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/ProductOrderedBuf.scala @@ -18,13 +18,12 @@ package com.twitter.scalding.serialization.macros.impl.ordered_serialization.pro import scala.reflect.macros.blackbox.Context import com.twitter.scalding.serialization.macros.impl.ordered_serialization.{ -ProductLike, + ProductLike, TreeOrderedBuf } object ProductOrderedBuf { - def dispatch(c: Context)(buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]]) - : PartialFunction[c.Type, TreeOrderedBuf[c.type]] = { + def dispatch(c: Context)(buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]]): PartialFunction[c.Type, TreeOrderedBuf[c.type]] = { import c.universe._ val validTypes: List[Type] = List( typeOf[Product1[Any]], @@ -42,132 +41,13 @@ object ProductOrderedBuf { typeOf[Product13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], typeOf[Product14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], typeOf[Product15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], - typeOf[ - Product16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], - typeOf[ - Product17[Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any]], - typeOf[ - Product18[Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any]], - typeOf[ - Product19[Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any]], - typeOf[ - Product20[Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any]], - typeOf[ - Product21[Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any]], - typeOf[ - Product22[Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any, - Any]] - ) + typeOf[Product16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], + typeOf[Product17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], + typeOf[Product18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], + typeOf[Product19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], + typeOf[Product20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], + typeOf[Product21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]], + typeOf[Product22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]]) def validType(curType: Type): Boolean = validTypes.exists { t => @@ -191,9 +71,10 @@ object ProductOrderedBuf { pf } - def apply(c: Context)(buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], - originalType: c.Type, - outerType: c.Type): TreeOrderedBuf[c.type] = { + def apply(c: Context)( + buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], + originalType: c.Type, + outerType: c.Type): TreeOrderedBuf[c.type] = { import c.universe._ def freshT(id: String) = TermName(c.freshName(id)) diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/SealedTraitOrderedBuf.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/SealedTraitOrderedBuf.scala index 56695ee484..310f462ef0 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/SealedTraitOrderedBuf.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/SealedTraitOrderedBuf.scala @@ -29,7 +29,8 @@ object SealedTraitOrderedBuf { pf } - def apply(c: Context)(buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], + def apply(c: Context)( + buildDispatcher: => PartialFunction[c.Type, TreeOrderedBuf[c.type]], outerType: c.Type): TreeOrderedBuf[c.type] = { import c.universe._ def freshT(id: String) = TermName(c.freshName(s"$id")) @@ -66,7 +67,8 @@ object SealedTraitOrderedBuf { .zipWithIndex .map { case ((tpe, tbuf), idx) => (idx, tpe, tbuf) } - require(subData.nonEmpty, + require( + subData.nonEmpty, "Unable to parse any subtypes for the sealed trait, error. This must be an error.") new TreeOrderedBuf[c.type] { diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/TraversablesOrderedBuf.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/TraversablesOrderedBuf.scala index a2ee208727..a643572e4c 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/TraversablesOrderedBuf.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/providers/TraversablesOrderedBuf.scala @@ -101,7 +101,8 @@ object TraversablesOrderedBuf { // When dealing with a map we have 2 type args, and need to generate the tuple type // it would correspond to if we .toList the Map. val innerType = if (outerType.asInstanceOf[TypeRefApi].args.size == 2) { - val (tpe1, tpe2) = (outerType.asInstanceOf[TypeRefApi].args.head, + val (tpe1, tpe2) = ( + outerType.asInstanceOf[TypeRefApi].args.head, outerType.asInstanceOf[TypeRefApi].args(1)) // linter:ignore val containerType = typeOf[Tuple2[Any, Any]].asInstanceOf[TypeRef] import compat._ diff --git a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/runtime_helpers/TraversableHelpers.scala b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/runtime_helpers/TraversableHelpers.scala index a754de1f82..459b0a6d7f 100644 --- a/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/runtime_helpers/TraversableHelpers.scala +++ b/scalding-serialization/src/main/scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/runtime_helpers/TraversableHelpers.scala @@ -39,7 +39,8 @@ object TraversableHelpers { } final def iteratorCompare[T](iteratorA: Iterator[T], iteratorB: Iterator[T])( - implicit ord: Ordering[T]): Int = { + implicit + ord: Ordering[T]): Int = { @annotation.tailrec def result: Int = if (iteratorA.isEmpty) { @@ -58,7 +59,8 @@ object TraversableHelpers { } final def iteratorEquiv[T](iteratorA: Iterator[T], iteratorB: Iterator[T])( - implicit eq: Equiv[T]): Boolean = { + implicit + eq: Equiv[T]): Boolean = { @annotation.tailrec def result: Boolean = if (iteratorA.isEmpty) iteratorB.isEmpty @@ -79,7 +81,8 @@ object TraversableHelpers { * sort case */ final def sortedCompare[T](travA: Iterable[T], travB: Iterable[T])( - implicit ord: Ordering[T]): Int = { + implicit + ord: Ordering[T]): Int = { def compare(startA: Int, endA: Int, a: Buffer[T], startB: Int, endB: Int, b: Buffer[T]): Int = if (startA == endA) { if (startB == endB) 0 // both empty @@ -87,7 +90,8 @@ object TraversableHelpers { } else if (startB == endB) 1 // non-empty is bigger than empty else { @annotation.tailrec - def partition(pivot: T, + def partition( + pivot: T, pivotStart: Int, pivotEnd: Int, endX: Int, diff --git a/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/Op.scala b/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/Op.scala index 3dbf17acb5..82f12172e2 100644 --- a/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/Op.scala +++ b/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/Op.scala @@ -1,17 +1,17 @@ package com.twitter.scalding.spark_backend -import org.apache.spark.{HashPartitioner, Partitioner} -import org.apache.spark.rdd.{RDD, UnionRDD} +import org.apache.spark.{ HashPartitioner, Partitioner } +import org.apache.spark.rdd.{ RDD, UnionRDD } import org.apache.spark.sql.SparkSession import org.apache.spark.storage.StorageLevel -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.{ ExecutionContext, Future } import scala.reflect.ClassTag -import com.twitter.scalding.{Config, FutureCache} +import com.twitter.scalding.{ Config, FutureCache } import com.twitter.scalding.typed.TypedSource import SparkPlanner.PartitionComputer sealed abstract class Op[+A] { - import Op.{Transformed, fakeClassTag} + import Op.{ Transformed, fakeClassTag } def run(session: SparkSession)(implicit ec: ExecutionContext): Future[RDD[_ <: A]] diff --git a/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/SparkMode.scala b/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/SparkMode.scala index f5f67374d2..42f1736933 100644 --- a/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/SparkMode.scala +++ b/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/SparkMode.scala @@ -1,6 +1,6 @@ package com.twitter.scalding.spark_backend -import com.twitter.scalding.{Config, Mode, WritableSequenceFile, TextLine} +import com.twitter.scalding.{ Config, Mode, WritableSequenceFile, TextLine } import com.twitter.scalding.typed.{ Resolver, TypedSource, TypedSink } import org.apache.spark.rdd.RDD import org.apache.spark.storage.StorageLevel @@ -73,7 +73,8 @@ object SparkSource extends Serializable { } } - def writableSequenceFile[K <: Writable, V <: Writable](path: String, + def writableSequenceFile[K <: Writable, V <: Writable]( + path: String, kclass: Class[K], vclass: Class[V]): SparkSource[(K, V)] = new SparkSource[(K, V)] { override def read(session: SparkSession, config: Config)(implicit ec: ExecutionContext): Future[RDD[_ <: (K, V)]] = { @@ -92,7 +93,7 @@ object SparkSource extends Serializable { def apply[A](i: TypedSource[A]): Option[SparkSource[A]] = { i match { case ws @ WritableSequenceFile(path, _, _) => - Some(writableSequenceFile(path, ws.keyType, ws.valueType)) + Some(writableSequenceFile(path, ws.keyType, ws.valueType)) case tl: TextLine => // actually only one path: Some(textLine(tl.localPaths.head, None)) diff --git a/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/SparkWriter.scala b/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/SparkWriter.scala index 776c32d7e4..d77ba8f21d 100644 --- a/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/SparkWriter.scala +++ b/scalding-spark/src/main/scala/com/twitter/scalding/spark_backend/SparkWriter.scala @@ -5,7 +5,7 @@ import com.stripe.dagon.{ HMap, Rule } import com.twitter.scalding.typed._ import com.twitter.scalding.Mode import com.twitter.scalding.typed.memory_backend.AtomicBox -import com.twitter.scalding.{ Config, Execution, ExecutionCounters, CancellationHandler, CFuture} +import com.twitter.scalding.{ Config, Execution, ExecutionCounters, CancellationHandler, CFuture } import org.apache.spark.sql.SparkSession import org.apache.spark.rdd.RDD import org.apache.spark.storage.StorageLevel @@ -186,8 +186,7 @@ class SparkWriter(val sparkMode: SparkMode) extends Writer { val rddF = op.run(session) promise.completeWith(rddF) rddF.map(_ => ()) - } - else Future.successful(()) + } else Future.successful(()) rddF.flatMap(_ => newState.write(conf, keyPipe, sink)) } @@ -200,10 +199,10 @@ class SparkWriter(val sparkMode: SparkMode) extends Writer { */ val (id: Long, acts) = state.update { s => val (nextState, acts) = optimizedWrites.foldLeft((s, List.empty[Action])) { - case (old@(state, acts), OptimizedWrite(pipe, Force(opt))) => + case (old @ (state, acts), OptimizedWrite(pipe, Force(opt))) => val (st, a) = force(opt, pipe, state) (st, a :: acts) - case (old@(state, acts), OptimizedWrite(pipe, ToIterable(opt))) => + case (old @ (state, acts), OptimizedWrite(pipe, ToIterable(opt))) => val (st, a) = force(opt, pipe, state) (st, a :: acts) case ((state, acts), OptimizedWrite(pipe, ToWrite.SimpleWrite(opt, sink))) =>