Skip to content

Commit 69eef8e

Browse files
committed
committing changes related to porting the code to Scala
1 parent 66632e4 commit 69eef8e

File tree

3 files changed

+108
-4
lines changed

3 files changed

+108
-4
lines changed

code/chap06/scala/build.gradle

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,28 @@
11
apply plugin: 'scala'
22
apply plugin: 'application'
33

4-
ext.scalaClassifier = '2.13'
5-
ext.scalaVersion = '2.13.7'
4+
ext.scalaClassifier = '2.12'
5+
ext.scalaVersion = '2.12.15'
66
ext.sparkVersion = '3.2.0'
77

88
group 'org.data.algorithms.spark.ch03'
99
version '1.0-SNAPSHOT'
1010

1111
repositories {
12-
mavenLocal()
12+
// mavenLocal()
1313
mavenCentral()
14+
maven {
15+
url "https://repos.spark-packages.org"
16+
}
1417
}
1518

1619
dependencies {
1720
implementation "org.scala-lang:scala-library:$scalaVersion"
1821
implementation "org.apache.spark:spark-core_$scalaClassifier:$sparkVersion"
1922
implementation "org.apache.spark:spark-sql_$scalaClassifier:$sparkVersion"
23+
implementation "org.apache.spark:spark-graphx_$scalaClassifier:$sparkVersion"
24+
implementation 'graphframes:graphframes:0.8.2-spark3.2-s_2.12'
25+
2026
}
2127

2228
application {

code/chap06/scala/settings.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
rootProject.name = 'data-algos-with-spark-ch03'
1+
rootProject.name = 'data-algos-with-spark-ch06'
22

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,103 @@
1+
/*
2+
#-----------------------------------------------------
3+
# This program
4+
#
5+
# 1. Builds a graph using GraphFrames package.
6+
#
7+
# 2. Applies Breadth First Search (BFS) algorithm.
8+
# Breadth-First Search (BFS) finds the shortest
9+
# path(s) from one vertex (or a set of vertices)
10+
# to another vertex (or a set of vertices). The
11+
# beginning and end vertices are specified as Spark
12+
# DataFrame expressions.
13+
#------------------------------------------------------
14+
# Input Parameters:
15+
# none
16+
#-------------------------------------------------------
17+
# @author Sanjay Bheemasenarao
18+
#-------------------------------------------------------
19+
*/
20+
121
package org.data.algorithms.spark.ch06
222

23+
import org.apache.spark.sql.SparkSession
24+
import org.graphframes._
25+
326
object BreadthFirstSearchExample {
27+
def main(args: Array[String]): Unit = {
28+
29+
// create an instance of SparkSession
30+
// spark is an instance of a SparkSession
31+
val spark = SparkSession
32+
.builder
33+
.appName("breadth_first_search_example").master("local[*]")
34+
.getOrCreate()
35+
36+
// Step-1: create vertices:
37+
val vertices = List(("a", "Alice", 30),
38+
("b", "Bob", 31),
39+
("c", "Charlie", 32),
40+
("d", "David", 23),
41+
("e", "Emma", 24),
42+
("f", "Frank", 26))
43+
44+
import spark.implicits._
45+
val v = spark.createDataset(vertices).toDF("id", "name", "age")
46+
v.show()
47+
48+
//Step-2: Create an Edge DataFrame with "src" and "dst"
49+
val edges = List(("a", "b", "follow"),
50+
("b", "c", "follow"),
51+
("c", "d", "follow"),
52+
("d", "e", "follow"),
53+
("b", "e", "follow"),
54+
("c", "e", "follow"),
55+
("e", "f", "follow"))
56+
57+
val e = spark.createDataset(edges).toDF("src","dst","relationship")
58+
e.show()
59+
/*
60+
+---+---+------------+
61+
|src|dst|relationship|
62+
+---+---+------------+
63+
| a| b| follow|
64+
| b| c| follow|
65+
| c| d| follow|
66+
| d| e| follow|
67+
| b| e| follow|
68+
| c| e| follow|
69+
| e| f| follow|
70+
+---+---+------------+
71+
*/
72+
73+
/*
74+
Step-3: Create a GraphFrame. Using GraphFrames API, a graph
75+
is built as an instance of a GraphFrame, which is a pair of
76+
vertices (as `v`) and edges (as `e`):
77+
*/
78+
val graph = GraphFrame(v, e)
79+
print("graph=", graph)
80+
81+
// GraphFrame(v:sql.DataFrame, e:sql.DataFrame)
82+
83+
/*
84+
==============
85+
BFS Algorithm
86+
==============
87+
The following code snippets uses BFS to find path between
88+
vertex with name "Alice" to a vertex with age < 27.
89+
90+
Search from "Alice" for users of age < 27.
91+
*/
92+
93+
val paths = graph.bfs.fromExpr("name = 'Alice'").toExpr("age > 30").run()
94+
paths.show()
95+
96+
// Specify edge filters or max path lengths.
97+
val paths2 = graph.bfs.fromExpr("name = 'Alice'").toExpr("age > 30").edgeFilter("relationship == 'follow'").maxPathLength(4).run()
98+
paths2.show()
499

100+
//done!
101+
spark.stop()
102+
}
5103
}

0 commit comments

Comments
 (0)