Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
12 changed files
with
191 additions
and
113 deletions.
There are no files selected for viewing
7 changes: 7 additions & 0 deletions
7
spark-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/Exp.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| package geotrellis.spark.pipeline.ast | ||
|
|
||
| trait Exp[T] { | ||
| def lit(i: Int): T | ||
| def neg(t: T): T | ||
| def add(l: T, r: T): T | ||
| } |
16 changes: 16 additions & 0 deletions
16
spark-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/Group.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package geotrellis.spark.pipeline.ast | ||
|
|
||
| import geotrellis.spark.pipeline._ | ||
| import geotrellis.spark.pipeline.json.Transform | ||
| import org.apache.spark.rdd.RDD | ||
|
|
||
| case class Group[T]( | ||
| tags: List[String], | ||
| tag: String, | ||
| `type`: String = "transform.group" | ||
| ) extends Transform { | ||
| def eval(labeledRDDs: List[(String, RDD[T])]): List[(String, RDD[T])] = { | ||
| val (grdds, rdds) = labeledRDDs.partition { case (l, _) => tags.contains(l) } | ||
| grdds.map { case (_, rdd) => tag -> rdd } ::: rdds | ||
| } | ||
| } |
14 changes: 14 additions & 0 deletions
14
spark-pipeline/src/main/scala/geotrellis/spark/pipeline/ast/multiband/spatial/Group.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| package geotrellis.spark.pipeline.ast.multiband.spatial | ||
|
|
||
| import geotrellis.spark.pipeline | ||
|
|
||
| import geotrellis.raster.Tile | ||
| import geotrellis.vector.ProjectedExtent | ||
|
|
||
| class Group { | ||
| def apply(tags: List[String], | ||
| tag: String, | ||
| `type`: String = "transform.group"): Group = { | ||
|
|
||
| } | ||
| } |
81 changes: 81 additions & 0 deletions
81
spark-pipeline/src/main/scala/geotrellis/spark/pipeline/interpreter/Interpreter.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| package geotrellis.spark.pipeline.interpreter | ||
|
|
||
| import geotrellis.spark.pipeline.json._ | ||
| import org.apache.spark.rdd.RDD | ||
|
|
||
| object TypedObject { | ||
| def read(clazz: String, `type`: String) = { | ||
| val isSpatial = `type`.indexOfSlice("spatial") > 0 | ||
| val isSingleband = `type`.indexOfSlice("singleband") > 0 | ||
| } | ||
|
|
||
| } | ||
|
|
||
| object Interpreter { | ||
| sealed trait Tree | ||
| final case class Leaf(clazz: String) extends Tree | ||
| final case class Node(clazz: String, ts: List[Tree]) extends Tree | ||
|
|
||
| def interpretUntyped = { | ||
| // everything should keep ordering | ||
| val reads: List[Read] = List() | ||
| val transformations: List[Transform] = List() | ||
| val writes: List[Write] = List() | ||
|
|
||
| val inputs: List[(String, RDD[Any])] = reads.map { r => | ||
| // make instance of a class and typed; after that it's possible to erase types again. | ||
| // Class.forName(r.`type`).newInstance | ||
| // read => | ||
| null: (String, RDD[Any]) | ||
| } | ||
|
|
||
| // along with common transform operations there can be arguable aggregate functions, | ||
| // to rename multiple inputs | ||
| // or to merge them into multiband input | ||
|
|
||
| val reorogonizedInputs = transformations.flatMap { | ||
| case t: TransformGroup => | ||
| // make instance of a class and typed; after that it's possible to erase types again. | ||
| // Class.forName(r.`type`).newInstance | ||
| // List[(String, RDD[Any])] => List[(String, RDD[Any])] function applied | ||
| // casting of RDD can be incapsulated into this functions | ||
| null: List[(String, RDD[Any])] | ||
|
|
||
| case t: TransformMerge => | ||
| // make instance of a class and typed; after that it's possible to erase types again. | ||
| // Class.forName(r.`type`).newInstance | ||
| // List[(String, RDD[Any])] => List[(String, RDD[Any])] function applied | ||
| // casting of RDD can be incapsulated into this functions | ||
| null: List[(String, RDD[Any])] | ||
|
|
||
| // no transofmration steps applied | ||
| case _ => null: List[(String, RDD[Any])] | ||
| } | ||
|
|
||
| val generalTransformations: List[(String, RDD[Any])] = reorogonizedInputs.map { | ||
| case p @ (tag, rdd) => | ||
| transformations.foldLeft(p) { case (acc, tr: Transform) => | ||
| // make instance of a class and typed; after that it's possible to erase types again. | ||
| // Class.forName(r.`type`).newInstance | ||
| // List[(String, RDD[Any])] => List[(String, RDD[Any])] functions applied | ||
| // casting of RDD can be incapsulated into this functions | ||
| // String as a first tuple argument can be used to be sure that transformation can be applied | ||
| // runtime exceptions can happen: class not found, or type can't be casted | ||
| // shapeless.cast function can be used(?) | ||
|
|
||
| // tr.instance.apply(acc) | ||
|
|
||
| null: (String, RDD[Any]) | ||
| } | ||
| } | ||
|
|
||
| writes.collect { case w: Write => | ||
| // make instance of a class and typed; after that it's possible to erase types again. | ||
| // Class.forName(r.`type`).newInstance | ||
| // List[(String, RDD[Any])] => Boolean // Unit | ||
|
|
||
| () | ||
| } | ||
|
|
||
| } | ||
| } |
4 changes: 3 additions & 1 deletion
4
...trellis/spark/pipeline/PipelineExpr.scala → ...is/spark/pipeline/json/PipelineExpr.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
...ark/pipeline/PipelineKeyIndexMethod.scala → ...ipeline/json/PipelineKeyIndexMethod.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.