Skip to content

Commit 28ab020

Browse files
marlessonmarlesson
committed
init
0 parents  commit 28ab020

15 files changed

+343716
-0
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.ipynb_checkpoints/
2+
log/

‎README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
## Criação de Grafos com a bilbioteca GraphX do Spark e GraphFrames
2+
3+
![Alt Text](./img/graphx-logo.svg)
4+
5+
Tutorial que utiliza as bibliotecas Spark GraphX (https://spark.apache.org/graphx/
6+
), GraphFrames (https://graphframes.github.io/) e D3 (https://d3js.org/) para criação, análise e visualização de Grafos no Spark.
7+
8+
Ver **tutorial.ipynb**
9+
10+
![Alt Text](./img/graph.png)
11+

‎build_graph.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
'''
2+
Spark Job Model Production
3+
4+
$ spark-submit --py-files graphframes.zip --jars jars/scala-logging-api_2.11-2.1.2.jar,jars/scala-logging-slf4j_2.11-2.1.2.jar build_graph.py
5+
6+
7+
Marlesson
8+
'''
9+
import os
10+
import sys
11+
import argparse
12+
import time
13+
import importlib
14+
from pyspark.sql import SparkSession
15+
16+
from graphframes import *
17+
18+
if __name__ == '__main__':
19+
# Session Spark
20+
spark = SparkSession\
21+
.builder\
22+
.appName("Graph")\
23+
.getOrCreate()
24+
25+
# Create a Vertex DataFrame with unique ID column "id"
26+
v = spark.createDataFrame([
27+
("a", "Alice", 34),
28+
("b", "Bob", 36),
29+
("c", "Charlie", 30),
30+
], ["id", "name", "age"])
31+
# Create an Edge DataFrame with "src" and "dst" columns
32+
e = spark.createDataFrame([
33+
("a", "b", "friend"),
34+
("b", "c", "follow"),
35+
("c", "b", "follow"),
36+
], ["src", "dst", "relationship"])
37+
# Create a GraphFrame
38+
from graphframes import *
39+
g = GraphFrame(v, e)
40+
41+
# Query: Get in-degree of each vertex.
42+
g.inDegrees.show()
43+
44+
# Query: Count the number of "follow" connections in the graph.
45+
g.edges.filter("relationship = 'follow'").count()
46+
47+
# # Run PageRank algorithm, and show results.
48+
# results = g.pageRank(resetProbability=0.01, maxIter=20)
49+
# results.vertices.select("id", "pagerank").show()

‎d3.html‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
<!DOCTYPE html>
2+
<meta charset="utf-8">
3+
<style>
4+
5+
.links line {
6+
stroke: #999;
7+
stroke-opacity: 0.6;
8+
}
9+
10+
.nodes circle {
11+
stroke: #fff;
12+
stroke-width: 1.5px;
13+
}
14+
15+
</style>
16+
<svg width="600" height="600"></svg>
17+
<script src="./d3.min.js"></script>
18+
<script>
19+
20+
var svg = d3.select("svg"),
21+
width = +svg.attr("width"),
22+
height = +svg.attr("height");
23+
24+
var color = d3.scaleOrdinal(d3.schemeCategory10);
25+
26+
var simulation = d3.forceSimulation()
27+
.force("link", d3.forceLink().id(function(d) { return d.id; }))
28+
.force("charge", d3.forceManyBody())
29+
.force("center", d3.forceCenter(width / 2, height / 2));
30+
// .force('chargeDistance', function(d) { return 19999});
31+
32+
d3.json("graph.json", function(error, graph) {
33+
if (error) throw error;
34+
35+
var link = svg.append("g")
36+
.attr("class", "links")
37+
.selectAll("line")
38+
.data(graph.links)
39+
.enter().append("line")
40+
.attr("stroke-width", function(d) { return 1 });
41+
42+
var node = svg.append("g")
43+
.attr("class", "nodes")
44+
.selectAll("circle")
45+
.data(graph.nodes)
46+
.enter().append("circle")
47+
.attr("r", 10)
48+
.attr("fill", function(d) { return color(d.group); })
49+
.call(d3.drag()
50+
.on("start", dragstarted)
51+
.on("drag", dragged)
52+
.on("end", dragended));
53+
54+
//Add the SVG Text Element to the svgContainer
55+
var text = svg.append("g")
56+
.attr("class", "text")
57+
.selectAll("text")
58+
.data(graph.nodes)
59+
.enter()
60+
.append("text")
61+
.text( function (d) { return d.name; })
62+
.attr("font-size", "15px")
63+
.attr("fill", "black");
64+
65+
node.append("title")
66+
.text(function(d) { return d.name; });
67+
68+
simulation
69+
.nodes(graph.nodes)
70+
.on("tick", ticked);
71+
72+
73+
simulation.force("link")
74+
.links(graph.links).distance(function(d) {return 100});
75+
76+
77+
function ticked() {
78+
link
79+
.attr("x1", function(d) { return d.source.x; })
80+
.attr("y1", function(d) { return d.source.y; })
81+
.attr("x2", function(d) { return d.target.x; })
82+
.attr("y2", function(d) { return d.target.y; });
83+
84+
text
85+
.attr("dx", function(d) { return d.x-20; })
86+
.attr("dy", function(d) { return d.y; });
87+
88+
node
89+
.attr("cx", function(d) { return d.x; })
90+
.attr("cy", function(d) { return d.y; });
91+
}
92+
});
93+
94+
function dragstarted(d) {
95+
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
96+
d.fx = d.x;
97+
d.fy = d.y;
98+
}
99+
100+
function dragged(d) {
101+
d.fx = d3.event.x;
102+
d.fy = d3.event.y;
103+
}
104+
105+
function dragended(d) {
106+
if (!d3.event.active) simulation.alphaTarget(0);
107+
d.fx = null;
108+
d.fy = null;
109+
}
110+
111+
</script>

‎d3.min.js‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)