Skip to content
Switch branches/tags
Go to file
Cannot retrieve contributors at this time
44 lines (37 sloc) 1.24 KB
package com.vishnuviswanath.spark.streaming
import com.vishnuviswanath.spark.streaming.sources.netcat.NetcatSourceProvider
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.streaming.Trigger
* Created by vviswanath on 2/20/18.
* An example that uses CustomV2 source {@link NetcatSourceProvider}
object CustomV2SourceExample {
def main(args: Array[String]): Unit = {
val spark = SparkSession.builder()
.appName("CustomV2 source")
import spark.implicits._
val raw = spark
.option("port", 9999)
.option("host", "localhost")
.option("buffSize", 100)
val consoleQuery = raw
.selectExpr("cast(value as STRING)")
//.outputMode("complete") not supported since it requires an agg, and Continuous processing does not support aggregations.
.trigger(Trigger.Continuous("3 second")) //how often to checkpoint