Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
33 lines (24 sloc) 743 Bytes
package chunbo.recommend
import org.apache.spark.SparkContext
//共同好友统计问题
//参考:http://www.cnblogs.com/charlesblc/p/6126346.html
object testCommendFriend {
def index(_spark_sc: SparkContext): Unit = {
//获取数据
val friendRDD = _spark_sc.textFile(Config.HDFS_HOSH + "test/common_friend")
//map
val friendKV = friendRDD.map(x => {
val fields = x.split(":")
val person = fields(0)
val friends = fields(1).split(",").toList
(person, friends)
})
val mapRDD = friendKV.flatMap(x => {
for (i <- 0 until x._2.length) yield (x._2(i), x._1)
})
//reduce
val reduceRDD = mapRDD.reduceByKey(_ + "::" + _)
//打印
reduceRDD.foreach(println)
}
}