Merge pull request #9 from ryan-williams/ug

spark-util upgrade
hammerlab · Jun 17, 2018 · 14c5bda · 14c5bda
2 parents a1dd929 + 0bfaad9
commit 14c5bda
Show file tree

Hide file tree

Showing 12 changed files with 95 additions and 25 deletions.
diff --git a/base/src/main/scala/org/hammerlab/cli/base/app/App.scala b/base/src/main/scala/org/hammerlab/cli/base/app/App.scala
@@ -5,8 +5,8 @@ import org.hammerlab.cli.base.close._
 abstract class App[Opts](protected val _args: Args[Opts])(
   implicit val container: Closeable
 )
-  extends CloseableProxy
-     with Serializable {
+extends CloseableProxy
+   with Serializable {
 
   implicit protected val opts = _args.opts
   implicit protected val _iargs: Args[Opts] = _args

diff --git a/base/src/main/scala/org/hammerlab/cli/base/app/OutPathApp.scala b/base/src/main/scala/org/hammerlab/cli/base/app/OutPathApp.scala
@@ -24,10 +24,10 @@ object OutPathApp {
  * [[App]] superclass where the second argument is the output path to write to; default: stdout
  */
 abstract class ArgsOutPathApp[Opts](args: Args[Opts])(
-    implicit c: Closeable
+  implicit c: Closeable
 )
-  extends PathApp[Opts](args)
-    with OutPathApp {
+extends PathApp[Opts](args)
+   with OutPathApp {
   override implicit val outPath =
     if (args.size > 1)
       Some(
@@ -69,10 +69,10 @@ abstract class RequiredArgOutPathApp[Opts](args: Args[Opts])(
  * field named `outPath` of type [[Option]][[Path]]
  */
 abstract class OptsOutPathApp[Opts: HasOutPath](args: Args[Opts])(
-    implicit c: Closeable
+  implicit c: Closeable
 )
-  extends PathApp[Opts](args)
-    with OutPathApp {
+extends PathApp[Opts](args)
+   with OutPathApp {
   override implicit val outPath = GetOutPath(args)
 }
 
diff --git a/base/src/test/scala/org/hammerlab/caseapp/ParsersTest.scala b/base/src/test/scala/org/hammerlab/caseapp/ParsersTest.scala
@@ -20,12 +20,14 @@ class ParsersTest
       parser(None, "1,2.3,4.56"),
       Right(Array(1, 2.3, 4.56))
     )
+
     ==(
       parser(None, "1,2.a"),
       Left(
         MalformedValue("double float", "2.a")
       )
     )
+
     ==(
       parser(None, "1,2.a,3.4,d"),
       Left(

diff --git a/base/src/test/scala/org/hammerlab/cli/base/app/MainSuite.scala b/base/src/test/scala/org/hammerlab/cli/base/app/MainSuite.scala
@@ -47,7 +47,7 @@ trait MainSuiteI {
       be(expected.stripMargin)
     )
 
-  def run(args: Seq[Arg]): Path = {
+  def run(args: Arg*): Path = {
     val outPath =
       if (outBasename.nonEmpty)
         tmpPath(outBasename)
@@ -64,7 +64,7 @@ trait MainSuiteI {
   }
 
   def shouldMatch[T](args: Seq[Arg], actual: Path ⇒ T, matcher: Matcher[T]): Unit = {
-    val outPath = run(args)
+    val outPath = run(args: _*)
     actual(outPath) should matcher
   }
 }

diff --git a/build.sbt b/build.sbt
@@ -6,7 +6,7 @@ default(
        hammerlab.io → "5.1.1",
               paths → "1.5.0",
     shapeless_utils → "1.3.0",
-         spark_util → "2.0.4"
+         spark_util → "3.0.0"
   ),
   `2.11`.only
 )
@@ -20,21 +20,24 @@ lazy val base = project.settings(
     shapeless,
     shapeless_utils
   ),
+  emptyDocJar,  // compiling tests for docs causes compiler stack-overflow in scala 2.12 😔
   publishTestJar  // `MainSuite` is useful in downstream libraries' tests
 )
 
+import Spark.autoImport.{ spark ⇒ sprk }
+
 lazy val spark = project.settings(
   dep(
     case_app,
     paths,
     slf4j,
     spark_util
   ),
+  sprk,
   testDeps ++= Seq(
     cats,
-    magic_rdds % "4.2.2"
+    magic_rdds % "4.2.3"
   ),
-  Spark.autoImport.spark,
   publishTestJar  // `MainSuite` is useful in downstream libraries' tests
 ).dependsOn(
   base andTest

diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1 +1 @@
-addSbtPlugin("org.hammerlab.sbt" % "base" % "4.6.1")
+addSbtPlugin("org.hammerlab.sbt" % "base" % "4.6.2")
diff --git a/spark/src/main/scala/hammerlab/cli/spark/package.scala b/spark/src/main/scala/hammerlab/cli/spark/package.scala
@@ -2,6 +2,7 @@ package hammerlab.cli
 
 import org.hammerlab.cli.{spark ⇒ s}
 package object spark {
+  type App[Opts] = s.App[Opts]
   type PathApp[Opts] = s.PathApp[Opts]
   type Registrar = s.Registrar
 }
diff --git a/spark/src/main/scala/org/hammerlab/cli/spark/HasSparkContext.scala b/spark/src/main/scala/org/hammerlab/cli/spark/HasSparkContext.scala
@@ -1,7 +1,7 @@
 package org.hammerlab.cli.spark
 
 import grizzled.slf4j.Logging
-import hammerlab.cli.App
+import hammerlab.cli
 import hammerlab.path._
 import org.apache.spark.SparkContext
 import org.hammerlab.hadoop.Configuration
@@ -18,7 +18,7 @@ trait HasSparkContext
   extends HasSparkConf
     with Logging {
 
-  self: App[_] ⇒
+  self: cli.App[_] ⇒
 
   @transient private var _sc: SparkContext = _
 

diff --git a/spark/src/main/scala/org/hammerlab/cli/spark/PathApp.scala b/spark/src/main/scala/org/hammerlab/cli/spark/PathApp.scala
@@ -1,21 +1,37 @@
 package org.hammerlab.cli.spark
 
 import Registrar.noop
+import org.hammerlab.cli.base
 import org.hammerlab.cli.base.app.{ Args, ArgsOutPathApp, HasPrintLimit }
 import org.hammerlab.cli.base.close.Closeable
 import org.hammerlab.spark.confs
 
+/**
+ * Generic Spark [[App]]
+ */
+abstract class App[Opts](
+  _args: Args[Opts],
+  reg: Registrar = noop
+)(
+  implicit c: Closeable
+)
+extends base.app.App[Opts](_args)
+   with HasSparkContext
+   with confs.Kryo {
+  reg.apply(this)
+}
+
 /**
  * [[HasSparkContext]] that takes an input path and prints some information to stdout or a path, with optional truncation of
  * such output.
  */
 abstract class PathApp[Opts](_args: Args[Opts],
                              reg: Registrar = noop)(
-    implicit c: Closeable
+  implicit c: Closeable
 )
-  extends ArgsOutPathApp[Opts](_args)
-    with HasSparkContext
-    with HasPrintLimit
-    with confs.Kryo {
+extends ArgsOutPathApp[Opts](_args)
+   with HasSparkContext
+   with HasPrintLimit
+   with confs.Kryo {
   reg.apply(this)
 }
diff --git a/spark/src/test/scala/org/hammerlab/cli/spark/ConfigTest.scala b/spark/src/test/scala/org/hammerlab/cli/spark/ConfigTest.scala
@@ -1,11 +1,12 @@
 package org.hammerlab.cli.spark
 
-import org.hammerlab.cli.base.app.{ App, Cmd }
+import org.hammerlab.cli.base
+import org.hammerlab.cli.base.app.Cmd
 import org.hammerlab.cli.base.args.PrintLimitArgs
 import org.hammerlab.kryo.spark
 
 /**
- * [[App]] that sets various configuration options and verifies their propagation.
+ * [[base.app.App]] that sets various configuration options and verifies their propagation.
  */
 object ConfigTest
   extends Cmd.With[PrintLimitArgs] {

diff --git a/spark/src/test/scala/org/hammerlab/cli/spark/NoopAppTest.scala b/spark/src/test/scala/org/hammerlab/cli/spark/NoopAppTest.scala
@@ -1,6 +1,6 @@
 package org.hammerlab.cli.spark
 
-import org.hammerlab.cli.base.app.{ App, Cmd }
+import org.hammerlab.cli.base.app.Cmd
 import org.hammerlab.cli.base.args.PrintLimitArgs
 
 class NoopAppTest
@@ -15,7 +15,7 @@ class NoopAppTest
 }
 
 /**
- * [[App]] that exercises some error and no-op code paths.
+ * [[PathApp]] that exercises some error and no-op code paths.
  */
 object NoopAppTest
   extends Cmd.With[PrintLimitArgs] {

diff --git a/spark/src/test/scala/org/hammerlab/cli/spark/PathOptTest.scala b/spark/src/test/scala/org/hammerlab/cli/spark/PathOptTest.scala
@@ -0,0 +1,47 @@
+package org.hammerlab.cli.spark
+
+import hammerlab.cli._
+import hammerlab.indent.spaces
+import hammerlab.path._
+import hammerlab.print._
+import hammerlab.show._
+import org.hammerlab.kryo
+
+class PathOptTest
+  extends MainSuite(PathOptTest) {
+  test("run") {
+    val out = tmpPath()
+    appContainer.main(
+      "-n", "100",
+      "-o", out
+    )
+    ==(
+      out.read,
+      "5050\n"
+    )
+  }
+}
+
+/**
+ * Test-[[Cmd]] similar to [[SumNumbersTest]], but taking its output [[Path]] as an option instead of as a positional
+ * argument
+ */
+object PathOptTest
+  extends Cmd {
+  case class Opts(
+    @O("n") n: Int,
+    @O("o") out: Path
+  )
+
+  case class Reg() extends kryo.spark.Registrar(classOf[Range])
+
+  val main = Main(
+    new spark.App(_, Reg) {
+      val out = opts.out
+      out.mkdirs
+      implicit val printer = Printer(out)
+      val sum = sc.parallelize(1 to opts.n).reduce(_ + _)
+      echo(sum)
+    }
+  )
+}