Skip to content

Commit

Permalink
Add json encode function
Browse files Browse the repository at this point in the history
  • Loading branch information
pomadchin authored and echeipesh committed Jan 15, 2018
1 parent 2ab672c commit 7821ead
Show file tree
Hide file tree
Showing 38 changed files with 270 additions and 157 deletions.
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
package geotrellis.spark.pipeline.ast

import geotrellis.spark.pipeline.json.PipelineExpr

import io.circe.Json
import org.apache.spark.SparkContext

trait Node[T] {
def get(implicit sc: SparkContext): T
def arg: PipelineExpr
def validation = {
def validation: (Boolean, String) = {
if (arg == null) (false, s"null")
else (true, "")
}
def asJson: List[Json]
}

trait Read[T] extends Node[T]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -11,6 +13,7 @@ case class BufferedReproject(
node: Node[MultibandTileLayerRDD[SpatialKey]],
arg: transform.BufferedReproject
) extends Transform[MultibandTileLayerRDD[SpatialKey], (Int, MultibandTileLayerRDD[SpatialKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, MultibandTileLayerRDD[SpatialKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.spatial

import io.circe.syntax._

import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.pipeline.ast._
Expand All @@ -11,6 +13,7 @@ case class FileWrite(
node: Node[(Int, MultibandTileLayerRDD[SpatialKey])],
arg: write.File
) extends Write[(Int, MultibandTileLayerRDD[SpatialKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, MultibandTileLayerRDD[SpatialKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark.pipeline.ast.Read
import geotrellis.spark.pipeline.json.read
Expand All @@ -9,6 +11,7 @@ import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD

case class HadoopRead(arg: read.SpatialMultibandHadoop) extends Read[RDD[(ProjectedExtent, MultibandTile)]] {
def asJson = arg.asJson :: Nil
def get(implicit sc: SparkContext): RDD[(ProjectedExtent, MultibandTile)] = arg.eval
def validate: (Boolean, String) = validation
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.spatial

import io.circe.syntax._

import geotrellis.spark.pipeline.ast._
import geotrellis.spark._
import geotrellis.spark.io._
Expand All @@ -11,6 +13,7 @@ case class HadoopWrite(
node: Node[(Int, MultibandTileLayerRDD[SpatialKey])],
arg: write.Hadoop
) extends Write[(Int, MultibandTileLayerRDD[SpatialKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, MultibandTileLayerRDD[SpatialKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark.pipeline.ast._
import geotrellis.spark.pipeline.json.transform
Expand All @@ -12,6 +14,7 @@ case class PerTileReproject(
node: Node[RDD[(ProjectedExtent, MultibandTile)]],
arg: transform.PerTileReproject
) extends Transform[RDD[(ProjectedExtent, MultibandTile)], RDD[(ProjectedExtent, MultibandTile)]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): RDD[(ProjectedExtent, MultibandTile)] = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -13,6 +15,7 @@ case class TileToLayout(
node: Node[RDD[(ProjectedExtent, MultibandTile)]],
arg: transform.TileToLayout
) extends Transform[RDD[(ProjectedExtent, MultibandTile)], MultibandTileLayerRDD[SpatialKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -13,6 +15,7 @@ case class TileToLayoutWithZoom(
node: Node[RDD[(ProjectedExtent, MultibandTile)]],
arg: transform.TileToLayoutWithZoom
) extends Transform[RDD[(ProjectedExtent, MultibandTile)],(Int, MultibandTileLayerRDD[SpatialKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, MultibandTileLayerRDD[SpatialKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.temporal

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -11,6 +13,7 @@ case class BufferedReproject(
node: Node[MultibandTileLayerRDD[SpaceTimeKey]],
arg: transform.BufferedReproject
) extends Transform[MultibandTileLayerRDD[SpaceTimeKey], (Int, MultibandTileLayerRDD[SpaceTimeKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, MultibandTileLayerRDD[SpaceTimeKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.temporal

import io.circe.syntax._

import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.pipeline.ast._
Expand All @@ -11,6 +13,7 @@ case class FileWrite(
node: Node[(Int, MultibandTileLayerRDD[SpaceTimeKey])],
arg: write.File
) extends Write[(Int, MultibandTileLayerRDD[SpaceTimeKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, MultibandTileLayerRDD[SpaceTimeKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.temporal

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark.TemporalProjectedExtent
import geotrellis.spark.pipeline.ast.Read
Expand All @@ -8,6 +10,7 @@ import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD

case class HadoopRead(arg: read.TemporalMultibandHadoop) extends Read[RDD[(TemporalProjectedExtent, MultibandTile)]] {
def asJson = arg.asJson :: Nil
def get(implicit sc: SparkContext): RDD[(TemporalProjectedExtent, MultibandTile)] = arg.eval
def validate: (Boolean, String) = validation
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.temporal

import io.circe.syntax._

import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.pipeline.ast._
Expand All @@ -11,6 +13,7 @@ case class HadoopWrite(
node: Node[(Int, MultibandTileLayerRDD[SpaceTimeKey])],
arg: write.Hadoop
) extends Write[(Int, MultibandTileLayerRDD[SpaceTimeKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, MultibandTileLayerRDD[SpaceTimeKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.temporal

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -12,6 +14,7 @@ case class PerTileReproject(
node: Node[RDD[(TemporalProjectedExtent, MultibandTile)]],
arg: transform.PerTileReproject
) extends Transform[RDD[(TemporalProjectedExtent, MultibandTile)], RDD[(TemporalProjectedExtent, MultibandTile)]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): RDD[(TemporalProjectedExtent, MultibandTile)] = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.temporal

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -12,6 +14,7 @@ case class TileToLayout(
node: Node[RDD[(TemporalProjectedExtent, MultibandTile)]],
arg: transform.TileToLayout
) extends Transform[RDD[(TemporalProjectedExtent, MultibandTile)], MultibandTileLayerRDD[SpaceTimeKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): MultibandTileLayerRDD[SpaceTimeKey] = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.multiband.temporal

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -12,6 +14,7 @@ case class TileToLayoutWithZoom(
node: Node[RDD[(TemporalProjectedExtent, MultibandTile)]],
arg: transform.TileToLayoutWithZoom
) extends Transform[RDD[(TemporalProjectedExtent, MultibandTile)], (Int, MultibandTileLayerRDD[SpaceTimeKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, MultibandTileLayerRDD[SpaceTimeKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.singleband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -13,6 +15,7 @@ case class BufferedReproject(
node: Node[TileLayerRDD[SpatialKey]],
arg: transform.BufferedReproject
) extends Transform[RDD[(ProjectedExtent, Tile)], (Int, TileLayerRDD[SpatialKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, TileLayerRDD[SpatialKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.singleband.spatial

import io.circe.syntax._

import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.pipeline.ast._
Expand All @@ -11,6 +13,7 @@ case class FileWrite(
node: Node[(Int, TileLayerRDD[SpatialKey])],
arg: write.File
) extends Write[(Int, TileLayerRDD[SpatialKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, TileLayerRDD[SpatialKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.singleband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark.pipeline.ast.Read
import geotrellis.spark.pipeline.json.read
Expand All @@ -9,6 +11,7 @@ import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD

case class HadoopRead(arg: read.SpatialHadoop) extends Read[RDD[(ProjectedExtent, Tile)]] {
def asJson = arg.asJson :: Nil
def get(implicit sc: SparkContext): RDD[(ProjectedExtent, Tile)] = arg.eval
def validate: (Boolean, String) = validation
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.singleband.spatial

import io.circe.syntax._

import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.pipeline.ast._
Expand All @@ -11,6 +13,7 @@ case class HadoopWrite(
node: Node[(Int, TileLayerRDD[SpatialKey])],
arg: write.Hadoop
) extends Write[(Int, TileLayerRDD[SpatialKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, TileLayerRDD[SpatialKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.singleband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark.pipeline.ast._
import geotrellis.spark.pipeline.json.transform
Expand All @@ -12,6 +14,7 @@ case class PerTileReproject(
node: Node[RDD[(ProjectedExtent, Tile)]],
arg: transform.PerTileReproject
) extends Transform[RDD[(ProjectedExtent, Tile)], RDD[(ProjectedExtent, Tile)]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): RDD[(ProjectedExtent, Tile)] = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.singleband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -13,6 +15,7 @@ case class TileToLayout(
node: Node[RDD[(ProjectedExtent, Tile)]],
arg: transform.TileToLayout
) extends Transform[RDD[(ProjectedExtent, Tile)], TileLayerRDD[SpatialKey]] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): TileLayerRDD[SpatialKey] = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package geotrellis.spark.pipeline.ast.singleband.spatial

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
Expand All @@ -13,6 +15,7 @@ case class TileToLayoutWithZoom(
node: Node[RDD[(ProjectedExtent, Tile)]],
arg: transform.TileToLayoutWithZoom
) extends Transform[RDD[(ProjectedExtent, Tile)], (Int, TileLayerRDD[SpatialKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, TileLayerRDD[SpatialKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
package geotrellis.spark.pipeline.ast.singleband.temporal

import io.circe.syntax._

import geotrellis.raster._
import geotrellis.spark._
import geotrellis.spark.pipeline.ast._
import geotrellis.spark.pipeline.json.transform

import org.apache.spark.SparkContext

case class BufferedReproject(
node: Node[TileLayerRDD[SpaceTimeKey]],
arg: transform.BufferedReproject
) extends Transform[TileLayerRDD[SpaceTimeKey], (Int, TileLayerRDD[SpaceTimeKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, TileLayerRDD[SpaceTimeKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
package geotrellis.spark.pipeline.ast.singleband.temporal

import io.circe.syntax._

import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.pipeline.ast._
import geotrellis.spark.pipeline.json.write

import org.apache.spark.SparkContext

case class FileWrite(
node: Node[(Int,TileLayerRDD[SpaceTimeKey])],
arg: write.File
) extends Write[(Int, TileLayerRDD[SpaceTimeKey])] {
def asJson = node.asJson :+ arg.asJson
def get(implicit sc: SparkContext): (Int, TileLayerRDD[SpaceTimeKey]) = arg.eval(node.get)
def validate: (Boolean, String) = {
val (f, msg) = if (node == null) (false, s"${this.getClass} has no node")
Expand Down
Loading

0 comments on commit 7821ead

Please sign in to comment.