Skip to content

Commit

Permalink
Code cleanup, more sugar
Browse files Browse the repository at this point in the history
  • Loading branch information
@pomadchin @echeipesh
pomadchin authored and echeipesh committed Jan 15, 2018
1 parent 177d609 commit 0d63ceb
Show file tree
Hide file tree
Showing 43 changed files with 92 additions and 389 deletions.
69 changes: 0 additions & 69 deletions spark-pipeline/src/main/scala/geotrellis/spark/pipeline/Main.scala
View file

This file was deleted.

4 changes: 0 additions & 4 deletions spark-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/Node.scala
View file
Expand Up @@ -8,9 +8,5 @@ import org.apache.spark.SparkContext
trait Node[T] {
def get(implicit sc: SparkContext): T
def arg: PipelineExpr
def validation: (Boolean, String) = {
if (arg == null) (false, s"null")
else (true, "")
}
def asJson: List[Json]
}
6 changes: 0 additions & 6 deletions ...ne/src/main/scala/geotrellis/spark/pipeline/ast/multiband/spatial/BufferedReproject.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class BufferedReproject(
) extends Transform[MultibandTileLayerRDD[SpatialKey], MultibandTileLayerRDD[SpatialKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = Transform.bufferedReproject(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...k-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/spatial/FileWrite.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class FileWrite(
) extends Write[Stream[(Int, MultibandTileLayerRDD[SpatialKey])]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, MultibandTileLayerRDD[SpatialKey])] = Write.eval(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
1 change: 0 additions & 1 deletion ...-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/spatial/HadoopRead.scala
View file
Expand Up @@ -14,5 +14,4 @@ case class HadoopRead(arg: read.JsonRead) extends Read[RDD[(ProjectedExtent, Mul
def asJson = arg.asJson :: Nil
def get(implicit sc: SparkContext): RDD[(ProjectedExtent, MultibandTile)] =
Read.evalSpatialMultibandHadoop(arg)
def validate: (Boolean, String) = validation
}
6 changes: 0 additions & 6 deletions ...pipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/spatial/HadoopWrite.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class HadoopWrite(
) extends Write[Stream[(Int, MultibandTileLayerRDD[SpatialKey])]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, MultibandTileLayerRDD[SpatialKey])] = Write.eval(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...ine/src/main/scala/geotrellis/spark/pipeline/ast/multiband/spatial/PerTileReproject.scala
View file
Expand Up @@ -16,10 +16,4 @@ case class PerTileReproject(
) extends Transform[RDD[(ProjectedExtent, MultibandTile)], RDD[(ProjectedExtent, MultibandTile)]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): RDD[(ProjectedExtent, MultibandTile)] = Transform.perTileReproject(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions spark-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/spatial/Pyramid.scala
View file
Expand Up @@ -18,10 +18,4 @@ case class Pyramid(
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, MultibandTileLayerRDD[SpatialKey])] =
Transform.pyramid(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...eline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/spatial/RetileToLayout.scala
View file
Expand Up @@ -16,10 +16,4 @@ case class RetileToLayout(
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] =
Transform.retileToLayoutSpatial(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...ipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/spatial/TileToLayout.scala
View file
Expand Up @@ -17,10 +17,4 @@ case class TileToLayout(
) extends Transform[RDD[(ProjectedExtent, MultibandTile)], MultibandTileLayerRDD[SpatialKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = Transform.tileToLayout(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...e/src/main/scala/geotrellis/spark/pipeline/ast/multiband/temporal/BufferedReproject.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class BufferedReproject(
) extends Transform[MultibandTileLayerRDD[SpaceTimeKey], MultibandTileLayerRDD[SpaceTimeKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): MultibandTileLayerRDD[SpaceTimeKey] = Transform.bufferedReproject(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/temporal/FileWrite.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class FileWrite(
) extends Write[Stream[(Int, MultibandTileLayerRDD[SpaceTimeKey])]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, MultibandTileLayerRDD[SpaceTimeKey])] = Write.eval(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
1 change: 0 additions & 1 deletion ...pipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/temporal/HadoopRead.scala
View file
Expand Up @@ -13,5 +13,4 @@ case class HadoopRead(arg: read.JsonRead) extends Read[RDD[(TemporalProjectedExt
def asJson = arg.asJson :: Nil
def get(implicit sc: SparkContext): RDD[(TemporalProjectedExtent, MultibandTile)] =
Read.evalTemporalMultibandHadoop(arg)
def validate: (Boolean, String) = validation
}
6 changes: 0 additions & 6 deletions ...ipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/temporal/HadoopWrite.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class HadoopWrite(
) extends Write[Stream[(Int, MultibandTileLayerRDD[SpaceTimeKey])]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, MultibandTileLayerRDD[SpaceTimeKey])] = Write.eval(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...ne/src/main/scala/geotrellis/spark/pipeline/ast/multiband/temporal/PerTileReproject.scala
View file
Expand Up @@ -16,10 +16,4 @@ case class PerTileReproject(
) extends Transform[RDD[(TemporalProjectedExtent, MultibandTile)], RDD[(TemporalProjectedExtent, MultibandTile)]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): RDD[(TemporalProjectedExtent, MultibandTile)] = Transform.perTileReproject(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions spark-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/temporal/Pyramid.scala
View file
Expand Up @@ -17,10 +17,4 @@ case class Pyramid(
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, MultibandTileLayerRDD[SpaceTimeKey])] =
Transform.pyramid(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...line/src/main/scala/geotrellis/spark/pipeline/ast/multiband/temporal/RetileToLayout.scala
View file
Expand Up @@ -14,10 +14,4 @@ case class RetileToLayout(
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): MultibandTileLayerRDD[SpaceTimeKey] =
Transform.retileToLayoutTemporal(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...peline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/temporal/TileToLayout.scala
View file
Expand Up @@ -16,10 +16,4 @@ case class TileToLayout(
) extends Transform[RDD[(TemporalProjectedExtent, MultibandTile)], MultibandTileLayerRDD[SpaceTimeKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): MultibandTileLayerRDD[SpaceTimeKey] = Transform.tileToLayout(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...e/src/main/scala/geotrellis/spark/pipeline/ast/singleband/spatial/BufferedReproject.scala
View file
Expand Up @@ -17,10 +17,4 @@ case class BufferedReproject(
) extends Transform[TileLayerRDD[SpatialKey], TileLayerRDD[SpatialKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): TileLayerRDD[SpatialKey] = Transform.bufferedReproject(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/singleband/spatial/FileWrite.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class FileWrite(
) extends Write[Stream[(Int, TileLayerRDD[SpatialKey])]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, TileLayerRDD[SpatialKey])] = Write.eval(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
1 change: 0 additions & 1 deletion ...pipeline/src/main/scala/geotrellis/spark/pipeline/ast/singleband/spatial/HadoopRead.scala
View file
Expand Up @@ -13,5 +13,4 @@ import org.apache.spark.rdd.RDD
case class HadoopRead(arg: read.JsonRead) extends Read[RDD[(ProjectedExtent, Tile)]] {
def asJson = arg.asJson :: Nil
def get(implicit sc: SparkContext): RDD[(ProjectedExtent, Tile)] = Read.evalSpatialHadoop(arg)
def validate: (Boolean, String) = validation
}
6 changes: 0 additions & 6 deletions ...ipeline/src/main/scala/geotrellis/spark/pipeline/ast/singleband/spatial/HadoopWrite.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class HadoopWrite(
) extends Write[Stream[(Int, TileLayerRDD[SpatialKey])]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, TileLayerRDD[SpatialKey])] = Write.eval(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...ne/src/main/scala/geotrellis/spark/pipeline/ast/singleband/spatial/PerTileReproject.scala
View file
Expand Up @@ -16,10 +16,4 @@ case class PerTileReproject(
) extends Transform[RDD[(ProjectedExtent, Tile)], RDD[(ProjectedExtent, Tile)]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): RDD[(ProjectedExtent, Tile)] = Transform.perTileReproject(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions spark-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/singleband/spatial/Pyramid.scala
View file
Expand Up @@ -18,10 +18,4 @@ case class Pyramid(
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, TileLayerRDD[SpatialKey])] =
Transform.pyramid(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...line/src/main/scala/geotrellis/spark/pipeline/ast/singleband/spatial/RetileToLayout.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class RetileToLayout(
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): TileLayerRDD[SpatialKey] =
Transform.retileToLayoutSpatial(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...peline/src/main/scala/geotrellis/spark/pipeline/ast/singleband/spatial/TileToLayout.scala
View file
Expand Up @@ -17,10 +17,4 @@ case class TileToLayout(
) extends Transform[RDD[(ProjectedExtent, Tile)], TileLayerRDD[SpatialKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): TileLayerRDD[SpatialKey] = Transform.tileToLayout(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions .../src/main/scala/geotrellis/spark/pipeline/ast/singleband/temporal/BufferedReproject.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class BufferedReproject(
) extends Transform[TileLayerRDD[SpaceTimeKey], TileLayerRDD[SpaceTimeKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): TileLayerRDD[SpaceTimeKey] = Transform.bufferedReproject(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...pipeline/src/main/scala/geotrellis/spark/pipeline/ast/singleband/temporal/FileWrite.scala
View file
Expand Up @@ -15,10 +15,4 @@ case class FileWrite(
) extends Write[Stream[(Int, TileLayerRDD[SpaceTimeKey])]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, TileLayerRDD[SpaceTimeKey])] = Write.eval(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
1 change: 0 additions & 1 deletion ...ipeline/src/main/scala/geotrellis/spark/pipeline/ast/singleband/temporal/HadoopRead.scala
View file
Expand Up @@ -13,5 +13,4 @@ import org.apache.spark.rdd.RDD
case class HadoopRead(arg: read.JsonRead) extends Read[RDD[(TemporalProjectedExtent, Tile)]] {
def asJson = arg.asJson :: Nil
def get(implicit sc: SparkContext): RDD[(TemporalProjectedExtent, Tile)] = Read.evalTemporalHadoop(arg)
def validate: (Boolean, String) = validation
}
5 changes: 0 additions & 5 deletions ...peline/src/main/scala/geotrellis/spark/pipeline/ast/singleband/temporal/HadoopWrite.scala
View file
Expand Up @@ -15,9 +15,4 @@ case class HadoopWrite(
) extends Write[Stream[(Int, TileLayerRDD[SpaceTimeKey])]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, TileLayerRDD[SpaceTimeKey])] = Write.eval(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node") else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...e/src/main/scala/geotrellis/spark/pipeline/ast/singleband/temporal/PerTileReproject.scala
View file
Expand Up @@ -16,10 +16,4 @@ case class PerTileReproject(
) extends Transform[RDD[(TemporalProjectedExtent, Tile)], RDD[(TemporalProjectedExtent, Tile)]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): RDD[(TemporalProjectedExtent, Tile)] = Transform.perTileReproject(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}
6 changes: 0 additions & 6 deletions ...k-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/singleband/temporal/Pyramid.scala
View file
Expand Up @@ -17,10 +17,4 @@ case class Pyramid(
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): Stream[(Int, TileLayerRDD[SpaceTimeKey])] =
Transform.pyramid(arg)(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
else node.validation
val (fs, msgs) = validation
(f && fs, msgs ++ msg)
}
}

0 comments on commit 0d63ceb

Please sign in to comment.