Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
96 lines (82 sloc) 1.52 KB
package org.hammerlab.cli.spark
import hammerlab.cli._
import hammerlab.lines.limit._
import magic_rdds._
class SumNumbersTest
extends MainSuite(SumNumbers) {
/**
* Test dummy [[SumNumbers]] app below, which prints the sum of some numbers as well as its
* [[org.apache.spark.serializer.KryoRegistrator]].
*/
test("print all") {
check(
path("numbers")
)(
"""10 numbers:
| 1
| 2
| 3
| 4
| 5
| 6
| 7
| 8
| 9
| 10
|
|Sum: 55
|"""
)
}
test("print limit 5") {
check(
"-l", "5",
path("numbers")
)(
"""5 of 10 numbers:
| 1
| 2
| 3
| 4
| 5
| …
|
|Sum: 55
|"""
)
}
}
/**
* Simple [[App]] and [[Runner]] that reads some numbers and prints them and their sum.
*
* Exercises [[HasPrintLimit]], among other things.
*/
object SumNumbers extends Cmd {
case class Opts(@R output: PrintLimitArgs)
val main = Main(
new PathApp(_) {
val rdd =
sc
.textFile(path.toString)
.map(_.toInt)
import cats.implicits._
val (sum, count) =
rdd
.map(_ 1L)
.reduce(_ |+| _)
val sampledInts = rdd.sample(count)
echo(
Limited(
sampledInts,
count,
s"$count numbers:",
s"$limit of $count numbers:"
)
)
echo(
"",
s"Sum: $sum"
)
}
)
}
You can’t perform that action at this time.