-
Notifications
You must be signed in to change notification settings - Fork 3
/
Test.kt
28 lines (18 loc) · 814 Bytes
/
Test.kt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import org.apache.spark.SparkConf
import org.apache.spark.api.java.JavaSparkContext
import kotlin.reflect.KClass
fun main(args: Array<String>) {
class MyItem(val id: Int, val value: String)
val conf = SparkConf()
.setMaster("local")
.setAppName("Line Counter")
conf.registerKryoClasses(MyItem::class)
val sc = JavaSparkContext(conf)
val input = sc.parallelize(listOf(MyItem(1,"Alpha"), MyItem(2,"Beta")))
val letters = input.flatMap { it.value.split(Regex("(?<=.)")) }
.map(String::toUpperCase)
.filter { it.matches(Regex("[A-Z]")) }
println(letters.collect())
}
//extension function to register Kotlin classes
fun SparkConf.registerKryoClasses(vararg args: KClass<*>) = registerKryoClasses(args.map { it.java }.toTypedArray())