This repository has been archived by the owner on Sep 20, 2022. It is now read-only.
/
EMR_INTEGRATION.mm
109 lines (109 loc) · 6.56 KB
/
EMR_INTEGRATION.mm
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
<map version="1.0.1">
<!-- To view this file, download free mind mapping software FreeMind from http://freemind.sourceforge.net -->
<node CREATED="1494945794550" ID="ID_1624326082" MODIFIED="1496749301308" TEXT="EMR Spark Integration">
<node CREATED="1494945843890" ID="ID_1215886071" MODIFIED="1498742558162" POSITION="right" TEXT="spark cluster">
<node CREATED="1494945924544" ID="ID_1008144698" MODIFIED="1494945929994" TEXT="master">
<node CREATED="1494946159437" ID="ID_1365128354" MODIFIED="1495021729523" TEXT="docker">
<node CREATED="1494946185076" ID="ID_1012994400" MODIFIED="1494946207558" TEXT="v3io-daemon">
<node CREATED="1494946212757" ID="ID_221238814" MODIFIED="1494946230979" TEXT="listen on 0.0.0.0:2000">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
<node CREATED="1494946257732" ID="ID_646384069" MODIFIED="1495021733970" TEXT="java process - spark-master">
<node CREATED="1494946351467" ID="ID_87818165" MODIFIED="1494946424193" TEXT="listen on 0.0.0.0:7077">
<icon BUILTIN="button_ok"/>
</node>
</node>
<node CREATED="1494946324675" ID="ID_280261053" MODIFIED="1495021736898" TEXT="Java process web-ui">
<node CREATED="1494946374180" ID="ID_239076490" MODIFIED="1494946427376" TEXT="listen on 0.0.0.0:8080">
<icon BUILTIN="button_ok"/>
</node>
</node>
<node CREATED="1494946334686" ID="ID_1523379741" MODIFIED="1494946343390" TEXT="java process spark-history"/>
</node>
<node CREATED="1494945930573" ID="ID_1251972489" MODIFIED="1494945939733" TEXT="worker1">
<node CREATED="1494946170164" ID="ID_745038308" MODIFIED="1494946172812" TEXT="docker">
<node CREATED="1494946185076" ID="ID_946897362" MODIFIED="1494946207558" TEXT="v3io-daemon">
<node CREATED="1494946212757" ID="ID_1035881344" MODIFIED="1494946230979" TEXT="listen on 0.0.0.0:2000">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
<node CREATED="1494946272269" ID="ID_616636133" MODIFIED="1494946282524" TEXT="java process - spark-worker"/>
</node>
<node CREATED="1494945940343" ID="ID_936643696" MODIFIED="1494945943322" TEXT="worker2">
<node CREATED="1494946176349" ID="ID_285588569" MODIFIED="1494946179418" TEXT="docker">
<node CREATED="1494946185076" ID="ID_496872463" MODIFIED="1494946207558" TEXT="v3io-daemon">
<node CREATED="1494946212757" ID="ID_519351131" MODIFIED="1494946230979" TEXT="listen on 0.0.0.0:2000">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
<node CREATED="1494946286555" ID="ID_959768151" MODIFIED="1494946296484" TEXT="java process - spark-worker"/>
</node>
</node>
<node CREATED="1494945877126" HGAP="32" ID="ID_928319639" MODIFIED="1498742568005" POSITION="right" TEXT="Node2" VSHIFT="52">
<node CREATED="1494946063310" ID="ID_245456572" MODIFIED="1494946082813" TEXT="RDMI interface(DISABLED)">
<icon BUILTIN="stop-sign"/>
</node>
<node CREATED="1494946088339" ID="ID_1208226851" MODIFIED="1494946099812" TEXT="Management interface 10G">
<node CREATED="1494946105231" ID="ID_1815453238" MODIFIED="1494946151515" TEXT="listen on port 0.0.0.0.1234">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
<node BACKGROUND_COLOR="#00ffff" CREATED="1494945905557" HGAP="42" ID="ID_464594781" MODIFIED="1498742317451" POSITION="left" TEXT="API" VSHIFT="10">
<node CREATED="1494946002503" ID="ID_1860569735" MODIFIED="1494946006851" TEXT="JDK"/>
<node CREATED="1494946007738" ID="ID_969142786" MODIFIED="1494946010927" TEXT="BOTO"/>
<node CREATED="1494946695621" ID="ID_1301545496" MODIFIED="1498742321413" TEXT="AWS cli">
<node CREATED="1496749125032" HGAP="30" ID="ID_6186809" MODIFIED="1498742607039" TEXT="aws config" VSHIFT="-4">
<icon BUILTIN="full-2"/>
<node CREATED="1496749146644" ID="ID_84215275" MODIFIED="1496749199491" TEXT="aws_access_key_id"/>
<node CREATED="1496749205652" ID="ID_1609669360" MODIFIED="1496749226433" TEXT="aws_secret_access_key"/>
<node CREATED="1496749214813" ID="ID_760136395" MODIFIED="1496749243395" TEXT="region"/>
<node CREATED="1496749244401" ID="ID_439029332" MODIFIED="1496749256359" TEXT="output = json"/>
</node>
<node CREATED="1498742334080" HGAP="60" ID="ID_155248191" MODIFIED="1498742516439" TEXT="emr_install" VSHIFT="-50">
<node CREATED="1498742385049" ID="ID_491971961" MODIFIED="1498742399662" TEXT="emr_install.sh"/>
<node CREATED="1498742400274" ID="ID_1031935820" MODIFIED="1498742405902" TEXT="emr_install.conf"/>
<node CREATED="1498742410905" ID="ID_352343107" MODIFIED="1498742420821" TEXT="emr_install.json"/>
</node>
<node CREATED="1498742373763" ID="ID_108094170" MODIFIED="1498742379364" TEXT="extras">
<node CREATED="1498742496335" ID="ID_1029428114" MODIFIED="1498742506384" TEXT="foxyproxy.configuration"/>
</node>
<node CREATED="1498742352502" HGAP="49" ID="ID_1457449815" MODIFIED="1498742533566" TEXT="s3_buckets" VSHIFT="33">
<node CREATED="1498742434401" ID="ID_727739914" MODIFIED="1498742437803" TEXT="jars"/>
<node CREATED="1498742438753" ID="ID_1413344172" MODIFIED="1498742440835" TEXT="rpms"/>
<node CREATED="1498742441406" ID="ID_596157873" MODIFIED="1498742449072" TEXT="download.sh script"/>
<node CREATED="1498742453254" ID="ID_1558626732" MODIFIED="1498742459918" TEXT="application.jar"/>
<node CREATED="1498742465991" ID="ID_112870155" MODIFIED="1498742474317" TEXT="post_install.sh script"/>
</node>
</node>
</node>
<node CREATED="1494946765152" HGAP="-7" ID="ID_115501610" MODIFIED="1498742543190" POSITION="left" TEXT="EQUINIX" VSHIFT="4">
<edge COLOR="#ff0033"/>
<node CREATED="1494946837757" HGAP="29" ID="ID_981727441" MODIFIED="1498742554581" TEXT="Node1" VSHIFT="34">
<node CREATED="1494946063310" ID="ID_1777156367" MODIFIED="1494946082813" TEXT="RDMI interface(DISABLED)">
<icon BUILTIN="stop-sign"/>
</node>
<node CREATED="1494946088339" ID="ID_1335374438" MODIFIED="1494946099812" TEXT="Management interface 10G">
<node CREATED="1494946105231" ID="ID_570907768" MODIFIED="1494946151515" TEXT="listen on port 0.0.0.0.1234">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
</node>
<node CREATED="1496749325930" HGAP="-21" ID="ID_971711127" MODIFIED="1498742577445" POSITION="right" TEXT="node-for-market-place" VSHIFT="31">
<node CREATED="1496749358515" ID="ID_409082581" MODIFIED="1496752161301" TEXT="micro-instance">
<node CREATED="1496749391171" ID="ID_1194256747" MODIFIED="1496749402722" TEXT="installation amazon linux micro"/>
<node CREATED="1496749403251" ID="ID_1780471522" MODIFIED="1496749492198" TEXT="aws config ">
<icon BUILTIN="full-2"/>
</node>
<node CREATED="1496749525371" ID="ID_882242134" MODIFIED="1496749532992" TEXT="script installation"/>
<node CREATED="1496749533952" ID="ID_685369630" MODIFIED="1496749547888" TEXT="config.json"/>
<node CREATED="1496749548528" ID="ID_246917442" MODIFIED="1496749624765" TEXT="emr-credentials.pem ssh keys"/>
</node>
</node>
</node>
</map>