/
76483f54.html
449 lines (416 loc) · 43.7 KB
/
76483f54.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
<!DOCTYPE html><html class="theme-next gemini use-motion" lang="zh-Hans"><head><meta name="generator" content="Hexo 3.8.0"><meta name="google-site-verification" content="SJDgyqjBwkGnbvvQE_jdQKhgkQfRh7WhtS0-E3egoyk"><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1"><meta name="theme-color" content="#222"><meta name="viewport" content="width=device-width"><script src="/lib/pace/pace.min.js?v=1.0.2"></script><link href="/lib/pace/pace-theme-minimal.min.css?v=1.0.2" rel="stylesheet"><meta http-equiv="Cache-Control" content="no-transform"><meta http-equiv="Cache-Control" content="no-siteapp"><link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css"><link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css"><link href="/css/main.css?v=5.1.4" rel="stylesheet" type="text/css"><link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=5.1.4"><link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=5.1.4"><link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=5.1.4"><link rel="mask-icon" href="/images/logo.svg?v=5.1.4" color="#222"><meta name="keywords" content="Hadoop,"><link rel="alternate" href="/atom.xml" title="从入门到放弃" type="application/atom+xml"><meta name="description" content="前言目前人工智能和大数据火热，使用的场景也越来越广，日常开发中也逐渐接触了更多与大数据相关的开发需求。因此对大数据知识也有必要进行一些学习理解 基础概念大数据的本质一、数据的存储：分布式文件系统（分布式存储） 二、数据的计算：分部署计算 基础知识学习大数据需要具备Java知识基础及Linux知识基础 学习路线Java基础和Linux基础Hadoop的学习：体系结构、原理、编程第一阶段： HDFS、"><meta name="keywords" content="Hadoop"><meta property="og:type" content="article"><meta property="og:title" content="【从入门到放弃-Hadoop】Hadoop基础学习"><meta property="og:url" content="https://nc2era.com/76483f54.html"><meta property="og:site_name" content="从入门到放弃"><meta property="og:description" content="前言目前人工智能和大数据火热，使用的场景也越来越广，日常开发中也逐渐接触了更多与大数据相关的开发需求。因此对大数据知识也有必要进行一些学习理解 基础概念大数据的本质一、数据的存储：分布式文件系统（分布式存储） 二、数据的计算：分部署计算 基础知识学习大数据需要具备Java知识基础及Linux知识基础 学习路线Java基础和Linux基础Hadoop的学习：体系结构、原理、编程第一阶段： HDFS、"><meta property="og:locale" content="zh-Hans"><meta property="og:image" content="https://nc2era.com/76483f54/1.png"><meta property="og:image" content="https://nc2era.com/76483f54/2.png"><meta property="og:image" content="https://nc2era.com/76483f54/3.png"><meta property="og:image" content="https://nc2era.com/76483f54/4.png"><meta property="og:image" content="https://nc2era.com/76483f54/5.png"><meta property="og:image" content="https://nc2era.com/76483f54/6.png"><meta property="og:image" content="https://nc2era.com/76483f54/7.png"><meta property="og:image" content="https://nc2era.com/76483f54/8.png"><meta property="og:image" content="https://nc2era.com/76483f54/9.png"><meta property="og:image" content="https://nc2era.com/76483f54/10.png"><meta property="og:image" content="https://nc2era.com/76483f54/11.png"><meta property="og:image" content="https://nc2era.com/76483f54/12.png"><meta property="og:image" content="https://nc2era.com/76483f54/13.png"><meta property="og:updated_time" content="2019-05-26T06:33:21.140Z"><meta name="twitter:card" content="summary"><meta name="twitter:title" content="【从入门到放弃-Hadoop】Hadoop基础学习"><meta name="twitter:description" content="前言目前人工智能和大数据火热，使用的场景也越来越广，日常开发中也逐渐接触了更多与大数据相关的开发需求。因此对大数据知识也有必要进行一些学习理解 基础概念大数据的本质一、数据的存储：分布式文件系统（分布式存储） 二、数据的计算：分部署计算 基础知识学习大数据需要具备Java知识基础及Linux知识基础 学习路线Java基础和Linux基础Hadoop的学习：体系结构、原理、编程第一阶段： HDFS、"><meta name="twitter:image" content="https://nc2era.com/76483f54/1.png"><script type="text/javascript" id="hexo.configurations">var NexT=window.NexT||{},CONFIG={root:"/",scheme:"Gemini",version:"5.1.4",sidebar:{position:"left",display:"post",offset:12,b2t:!1,scrollpercent:!1,onmobile:!1},fancybox:!0,tabs:!0,motion:{enable:!0,async:!1,transition:{post_block:"fadeIn",post_header:"slideDownIn",post_body:"slideDownIn",coll_header:"slideLeftIn",sidebar:"slideUpIn"}},duoshuo:{userId:"0",author:"博主"},algolia:{applicationID:"",apiKey:"",indexName:"",hits:{per_page:10},labels:{input_placeholder:"Search for Posts",hits_empty:"We didn't find any results for the search: ${query}",hits_stats:"${hits} results found in ${time} ms"}}}</script><link rel="canonical" href="https://nc2era.com/76483f54.html"><title>【从入门到放弃-Hadoop】Hadoop基础学习 | 从入门到放弃</title><script type="text/javascript">var _hmt=_hmt||[];!function(){var e=document.createElement("script");e.src="https://hm.baidu.com/hm.js?7f0f37da8af427b455867492ca709c92";var t=document.getElementsByTagName("script")[0];t.parentNode.insertBefore(e,t)}()</script></head><body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans"><div class="container sidebar-position-left page-post-detail"><div class="headband"></div><header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader"><div class="header-inner"><div class="site-brand-wrapper"><div class="site-meta"><div class="custom-logo-site-title"><a href="/" class="brand" rel="start"><span class="logo-line-before"><i></i></span> <span class="site-title">从入门到放弃</span><span class="logo-line-after"><i></i></span></a></div><h1 class="site-subtitle" itemprop="description">从入门到放弃-程序员进阶之路</h1></div><div class="site-nav-toggle"> <button><span class="btn-bar"></span><span class="btn-bar"></span><span class="btn-bar"></span></button></div></div><nav class="site-nav"><ul id="menu" class="menu"><li class="menu-item menu-item-home"><a href="/" rel="section"><i class="menu-item-icon fa fa-fw fa-home"></i><br> 首页</a></li><li class="menu-item menu-item-tags"><a href="/tags/" rel="section"><i class="menu-item-icon fa fa-fw fa-tags"></i><br> 标签</a></li><li class="menu-item menu-item-categories"><a href="/categories/" rel="section"><i class="menu-item-icon fa fa-fw fa-th"></i><br> 分类</a></li><li class="menu-item menu-item-archives"><a href="/archives/" rel="section"><i class="menu-item-icon fa fa-fw fa-archive"></i><br> 归档</a></li><li class="menu-item menu-item-search"><a href="javascript:;" class="popup-trigger"><i class="menu-item-icon fa fa-search fa-fw"></i><br> 搜索</a></li></ul><div class="site-search"><div class="popup search-popup local-search-popup"><div class="local-search-header clearfix"><span class="search-icon"><i class="fa fa-search"></i></span><span class="popup-btn-close"><i class="fa fa-times-circle"></i></span><div class="local-search-input-wrapper"> <input autocomplete="off" placeholder="搜索..." spellcheck="false" type="text" id="local-search-input"></div></div><div id="local-search-result"></div></div></div></nav></div></header><main id="main" class="main"><div class="main-inner"><div class="content-wrap"><div id="content" class="content"><div id="posts" class="posts-expand"><article class="post post-type-normal" itemscope itemtype="http://schema.org/Article"><div class="post-block"><link itemprop="mainEntityOfPage" href="https://nc2era.com/76483f54.html"><span hidden itemprop="author" itemscope itemtype="http://schema.org/Person"><meta itemprop="name" content="AloofJr"><meta itemprop="description" content><meta itemprop="image" content="/images/avatar.gif"></span><span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization"><meta itemprop="name" content="从入门到放弃"></span><header class="post-header"><h2 class="post-title" itemprop="name headline">【从入门到放弃-Hadoop】Hadoop基础学习</h2><div class="post-meta"><span class="post-time"><span class="post-meta-item-icon"><i class="fa fa-calendar-o"></i></span> <span class="post-meta-item-text">发表于</span> <time title="创建于" itemprop="dateCreated datePublished" datetime="2019-05-26T14:11:26+08:00">2019-05-26</time></span> <span class="post-category"><span class="post-meta-divider">|</span><span class="post-meta-item-icon"><i class="fa fa-folder-o"></i></span> <span class="post-meta-item-text">分类于</span> <span itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/categories/个人学习/" itemprop="url" rel="index"><span itemprop="name">个人学习</span></a></span></span> <span class="post-comments-count"><span class="post-meta-divider">|</span><span class="post-meta-item-icon"><i class="fa fa-comment-o"></i></span><a href="/76483f54.html#comments" itemprop="discussionUrl"><span class="post-comments-count valine-comment-count" data-xid="/76483f54.html" itemprop="commentCount"></span></a></span> <span class="post-meta-divider">|</span><span class="page-pv"><i class="fa fa-file-o"></i> 浏览<span class="busuanzi-value" id="busuanzi_value_page_pv"></span></span><div class="post-wordcount"><span class="post-meta-item-icon"><i class="fa fa-file-word-o"></i></span> <span class="post-meta-item-text">字数统计:</span> <span title="字数统计">1.5k</span> <span class="post-meta-divider">|</span><span class="post-meta-item-icon"><i class="fa fa-clock-o"></i></span> <span class="post-meta-item-text">阅读时长 ≈</span> <span title="阅读时长">6</span></div></div></header><div class="post-body" itemprop="articleBody"><h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2><p>目前人工智能和大数据火热，使用的场景也越来越广，日常开发中也逐渐接触了更多与大数据相关的开发需求。因此对大数据知识也有必要进行一些学习理解</p><h2 id="基础概念"><a href="#基础概念" class="headerlink" title="基础概念"></a>基础概念</h2><h3 id="大数据的本质"><a href="#大数据的本质" class="headerlink" title="大数据的本质"></a>大数据的本质</h3><p>一、数据的存储：分布式文件系统（分布式存储）</p><p>二、数据的计算：分部署计算</p><h3 id="基础知识"><a href="#基础知识" class="headerlink" title="基础知识"></a>基础知识</h3><p>学习大数据需要具备Java知识基础及Linux知识基础</p><h3 id="学习路线"><a href="#学习路线" class="headerlink" title="学习路线"></a>学习路线</h3><h4 id="Java基础和Linux基础"><a href="#Java基础和Linux基础" class="headerlink" title="Java基础和Linux基础"></a>Java基础和Linux基础</h4><h4 id="Hadoop的学习：体系结构、原理、编程"><a href="#Hadoop的学习：体系结构、原理、编程" class="headerlink" title="Hadoop的学习：体系结构、原理、编程"></a>Hadoop的学习：体系结构、原理、编程</h4><p>第一阶段：</p><pre><code>HDFS、MapReduce、HBase（NoSQL数据库）
</code></pre><p>第二阶段：</p><pre><code>数据分析引擎 -> Hive、Pig
数据采集引擎 -> Sqoop、Flume
</code></pre><p>第三阶段：</p><pre><code>HUE：Web管理工具
ZooKeeper：实现Hadoop的HA
Oozie：工作流引擎
</code></pre><h4 id="Spark的学习"><a href="#Spark的学习" class="headerlink" title="Spark的学习"></a>Spark的学习</h4><pre><code>第一阶段：Scala编程语言
第二阶段：Spark Core -> 基于内存、数据的计算
第三阶段：Spark SQL -> 类似于mysql 的sql语句
第四阶段：Spark Streaming ->进行流式计算：比如：自来水厂
</code></pre><h4 id="Apache-Storm-类似"><a href="#Apache-Storm-类似" class="headerlink" title="Apache Storm 类似"></a>Apache Storm 类似</h4><pre><code>Spark Streaming ->进行流式计算
</code></pre><h4 id="NoSQL"><a href="#NoSQL" class="headerlink" title="NoSQL"></a>NoSQL</h4><pre><code>Redis基于内存的数据库
</code></pre><h3 id="HDFS"><a href="#HDFS" class="headerlink" title="HDFS"></a>HDFS</h3><p>分布式文件系统 解决以下问题：</p><pre><code>1、硬盘不够大：多几块硬盘，理论上可以无限大
2、数据不够安全：冗余度，hdfs默认冗余为3 ，用水平复制提高效率，传输按照数据库为单位：Hadoop1.x 64M，Hadoop2.x 128M
</code></pre><p>管理员：NameNode 硬盘：DataNode</p> <img src="/76483f54/1.png"><h3 id="MapReduce"><a href="#MapReduce" class="headerlink" title="MapReduce"></a>MapReduce</h3><pre><code>基础编程模型：把一个大任务拆分成小任务，再进行汇总
MR任务：Job = Map + Reduce
Map的输出是Reduce的输入、MR的输入和输出都是在HDFS
</code></pre><p> MapReduce数据流程分析：</p><pre><code>Map的输出是Reduce的输入，Reduce的输入是Map的集合
</code></pre><img src="/76483f54/2.png"><h3 id="HBase"><a href="#HBase" class="headerlink" title="HBase"></a>HBase</h3><p>什么是BigTable？: 把所有的数据保存到一张表中，采用冗余 —> 好处：提高效率</p><pre><code>1、因为有了bigtable的思想：NoSQL：HBase数据库
2、HBase基于Hadoop的HDFS的
3、描述HBase的表结构
核心思想是：利用空间换效率
</code></pre><img src="/76483f54/3.png"><h2 id="Hadoop环境搭建"><a href="#Hadoop环境搭建" class="headerlink" title="Hadoop环境搭建"></a>Hadoop环境搭建</h2><h3 id="环境准备"><a href="#环境准备" class="headerlink" title="环境准备"></a>环境准备</h3><p>Linux环境、JDK、<a href="hadoop-3.0.0.tar.gz">http://mirrors.shu.edu.cn/apache/hadoop/common/hadoop-3.0.0/hadoop-3.0.0-src.tar.gz</a></p><h3 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h3><p>1、安装jdk、并配置环境变量</p><pre><code>vim /etc/profile 末尾添加
</code></pre><img src="/76483f54/4.png"><p>2、解压hadoop-3.0.0.tar.gz、并配置环境变量</p><pre><code>tar -zxvf hadoop-3.0.0.tar.gz -C /usr/local/
mv hadoop-3.0.0/ hadoop
</code></pre><img src="/76483f54/5.png"> <img src="/76483f54/6.png"><p>vim /etc/profile 末尾添加</p> <img src="/76483f54/7.png"><h3 id="配置"><a href="#配置" class="headerlink" title="配置"></a>配置</h3><p>Hadoop有三种安装模式：</p><pre><code>本地模式 ：
1台主机
不具备HDFS，只能测试MapReduce程序
伪分布模式：
1台主机
具备Hadoop的所有功能，在单机上模拟一个分布式的环境
（1）HDFS：主：NameNode，数据节点：DataNode
（2）Yarn：容器，运行MapReduce程序
主节点：ResourceManager
从节点：NodeManager
全分布模式：
至少3台
</code></pre><p>我们以伪分布模式为例配置：</p><h3 id="修改hdfs-site-xml：冗余度1、权限检查false"><a href="#修改hdfs-site-xml：冗余度1、权限检查false" class="headerlink" title="修改hdfs-site.xml：冗余度1、权限检查false"></a>修改hdfs-site.xml：冗余度1、权限检查false</h3><pre><code><!--配置冗余度为1-->
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<!--配置权限检查为false-->
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
</code></pre><h3 id="修改core-site-xml"><a href="#修改core-site-xml" class="headerlink" title="修改core-site.xml"></a>修改core-site.xml</h3><pre><code><!--配置HDFS的NameNode-->
<property>
<name>fs.defaultFS</name>
<value>hdfs://192.168.56.102:9000</value>
</property>
<!--配置DataNode保存数据的位置-->
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/tmp</value>
</property>
</code></pre><h3 id="修改mapred-site-xml"><a href="#修改mapred-site-xml" class="headerlink" title="修改mapred-site.xml"></a>修改mapred-site.xml</h3><pre><code><!--配置MR运行的框架-->
<property>
<name>mapreduce.framework.name</name>
<value>yar</value>
</property>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=/usr/local/hadoop</value>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=/usr/local/hadoop</value>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=/usr/local/hadoop</value>
</property>
<property>
<name>mapreduce.application.classpath</name>
<value>
/usr/local/hadoop/etc/hadoop,
/usr/local/hadoop/share/hadoop/common/*,
/usr/local/hadoop/share/hadoop/common/lib/*,
/usr/local/hadoop/share/hadoop/hdfs/*,
/usr/local/hadoop/share/hadoop/hdfs/lib/*,
/usr/local/hadoop/share/hadoop/mapreduce/*,
/usr/local/hadoop/share/hadoop/mapreduce/lib/*,
/usr/local/hadoop/share/hadoop/yarn/*,
/usr/local/hadoop/share/hadoop/yarn/lib/*,
</value>
</property>
</code></pre><h3 id="修改yarn-site-xml"><a href="#修改yarn-site-xml" class="headerlink" title="修改yarn-site.xml"></a>修改yarn-site.xml</h3><pre><code><!--配置ResourceManager地址-->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>192.168.56.102</value>
</property>
<!--配置NodeManager执行任务的方式-->
<property>
<name>yarn.nodemanager.aux-service</name>
<value>mapreduce_shuffle</value>
</property>
</code></pre><h3 id="格式化NameNode"><a href="#格式化NameNode" class="headerlink" title="格式化NameNode"></a>格式化NameNode</h3><p>hdfs namenode -format</p><p>看到common.Storage: Storage directory /usr/local/hadoop/tmp/dfs/name has been successfully formatted表示格式化成功</p><h3 id="启动"><a href="#启动" class="headerlink" title="启动"></a>启动</h3><p>start-all.sh</p><ul><li>HDFS：存储数据</li><li>YARN：</li></ul><h3 id="访问"><a href="#访问" class="headerlink" title="访问"></a>访问</h3><ul><li>命令行</li><li>Java Api</li><li>WEB Console</li></ul><p>HDFS: <a href="http://192.168.56.102:50070" rel="external nofollow noopener noreferrer" target="_blank">http://192.168.56.102:50070</a></p><p>Yarn: <a href="http://192.168.56.102:8088" rel="external nofollow noopener noreferrer" target="_blank">http://192.168.56.102:8088</a></p> <img src="/76483f54/8.png"><p>查看HDFS管理界面和yarn资源管理系统</p> <img src="/76483f54/9.png"> <img src="/76483f54/10.png"><h2 id="基本操作："><a href="#基本操作：" class="headerlink" title="基本操作："></a>基本操作：</h2><h3 id="HDFS相关命令"><a href="#HDFS相关命令" class="headerlink" title="HDFS相关命令"></a>HDFS相关命令</h3><pre><code>-mkdir 在HDFD创建目录 hdfs dfs -mkdir /data
-ls 查看目录 hdfs dfs -ls
-ls -R 查看目录与子目录 hdfs dfs -ls -R
-put 上传一个文件 hdfs dfs -put data.txt /data/input
-copyFromLocal 上传一个文件 与-put一样
-moveFromLocal 上传一个文件并删除本地文件
-copyToLocal 下载文件 hdfs dfs -copyTolocal /data/input/data.txt
-put 下载文件 hdfs dfs -put/data/input/data.txt
-rm 删除文件 hdfs dfs -rm
-getmerge 将目录所有文件先合并再下载
-cp 拷贝
-mv 移动
-count 统计目录下的文件个数
-text、-cat 查看文件
-balancer 平衡操作
</code></pre><img src="/76483f54/11.png"><h2 id="MapReduce示例"><a href="#MapReduce示例" class="headerlink" title="MapReduce示例"></a>MapReduce示例</h2><img src="/76483f54/12.png"><p>结果：<br><img src="/76483f54/13.png"></p><p>如上 一个最简单的MapReduce示例就执行成功了</p><h2 id="思考"><a href="#思考" class="headerlink" title="思考"></a>思考</h2><p>Hadoop是基于Java语言的，日常开发是用的PHP(写文章时，博主主要是用PHP，现在已经转Java了)，在使用、查找错误时还是蛮吃力的。工作之余还是需要多补充点其它语言的相关知识，编程语言是我们开发、学习的工具，而不应成为限制我们技术成长的瓶颈</p></div><footer class="post-footer"><div class="post-tags"> <a href="/tags/Hadoop/" rel="tag"># Hadoop</a></div><div class="post-nav"><div class="post-nav-next post-nav-item"><a href="/76b41bca.html" rel="next" title="【从入门到放弃-PHP】foreach 引用的坑"><i class="fa fa-chevron-left"></i> 【从入门到放弃-PHP】foreach 引用的坑</a></div><span class="post-nav-divider"></span><div class="post-nav-prev post-nav-item"> <a href="/6c0fa14d.html" rel="prev" title="【从入门到放弃-MySQL】数据库连接过程分析">【从入门到放弃-MySQL】数据库连接过程分析<i class="fa fa-chevron-right"></i></a></div></div></footer></div></article><div class="post-spread"></div></div></div><div class="comments" id="comments"></div></div><div class="sidebar-toggle"><div class="sidebar-toggle-line-wrap"><span class="sidebar-toggle-line sidebar-toggle-line-first"></span><span class="sidebar-toggle-line sidebar-toggle-line-middle"></span><span class="sidebar-toggle-line sidebar-toggle-line-last"></span></div></div><aside id="sidebar" class="sidebar"><div class="sidebar-inner"><ul class="sidebar-nav motion-element"><li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap"> 文章目录</li><li class="sidebar-nav-overview" data-target="site-overview-wrap"> 站点概览</li></ul><section class="site-overview-wrap sidebar-panel"><div class="site-overview"><div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person"><p class="site-author-name" itemprop="name">AloofJr</p><p class="site-description motion-element" itemprop="description"></p></div><nav class="site-state motion-element"><div class="site-state-item site-state-posts"> <a href="/archives/"><span class="site-state-item-count">47</span> <span class="site-state-item-name">日志</span></a></div><div class="site-state-item site-state-categories"> <a href="/categories/index.html"><span class="site-state-item-count">1</span> <span class="site-state-item-name">分类</span></a></div><div class="site-state-item site-state-tags"> <a href="/tags/index.html"><span class="site-state-item-count">50</span> <span class="site-state-item-name">标签</span></a></div></nav><div class="feed-link motion-element"><a href="/atom.xml" rel="alternate"><i class="fa fa-rss"></i> RSS</a></div><div class="links-of-author motion-element"><span class="links-of-author-item"><a href="https://github.com/AloofJr" target="_blank" title="GitHub" rel="external nofollow noopener noreferrer"><i class="fa fa-fw fa-github"></i> GitHub</a></span></div></div></section><section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active"><div class="post-toc"><div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#前言"><span class="nav-number">1.</span> <span class="nav-text">前言</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#基础概念"><span class="nav-number">2.</span> <span class="nav-text">基础概念</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#大数据的本质"><span class="nav-number">2.1.</span> <span class="nav-text">大数据的本质</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#基础知识"><span class="nav-number">2.2.</span> <span class="nav-text">基础知识</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#学习路线"><span class="nav-number">2.3.</span> <span class="nav-text">学习路线</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Java基础和Linux基础"><span class="nav-number">2.3.1.</span> <span class="nav-text">Java基础和Linux基础</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Hadoop的学习：体系结构、原理、编程"><span class="nav-number">2.3.2.</span> <span class="nav-text">Hadoop的学习：体系结构、原理、编程</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Spark的学习"><span class="nav-number">2.3.3.</span> <span class="nav-text">Spark的学习</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Apache-Storm-类似"><span class="nav-number">2.3.4.</span> <span class="nav-text">Apache Storm 类似</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#NoSQL"><span class="nav-number">2.3.5.</span> <span class="nav-text">NoSQL</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#HDFS"><span class="nav-number">2.4.</span> <span class="nav-text">HDFS</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#MapReduce"><span class="nav-number">2.5.</span> <span class="nav-text">MapReduce</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#HBase"><span class="nav-number">2.6.</span> <span class="nav-text">HBase</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Hadoop环境搭建"><span class="nav-number">3.</span> <span class="nav-text">Hadoop环境搭建</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#环境准备"><span class="nav-number">3.1.</span> <span class="nav-text">环境准备</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安装"><span class="nav-number">3.2.</span> <span class="nav-text">安装</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#配置"><span class="nav-number">3.3.</span> <span class="nav-text">配置</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#修改hdfs-site-xml：冗余度1、权限检查false"><span class="nav-number">3.4.</span> <span class="nav-text">修改hdfs-site.xml：冗余度1、权限检查false</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#修改core-site-xml"><span class="nav-number">3.5.</span> <span class="nav-text">修改core-site.xml</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#修改mapred-site-xml"><span class="nav-number">3.6.</span> <span class="nav-text">修改mapred-site.xml</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#修改yarn-site-xml"><span class="nav-number">3.7.</span> <span class="nav-text">修改yarn-site.xml</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#格式化NameNode"><span class="nav-number">3.8.</span> <span class="nav-text">格式化NameNode</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#启动"><span class="nav-number">3.9.</span> <span class="nav-text">启动</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#访问"><span class="nav-number">3.10.</span> <span class="nav-text">访问</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#基本操作："><span class="nav-number">4.</span> <span class="nav-text">基本操作：</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#HDFS相关命令"><span class="nav-number">4.1.</span> <span class="nav-text">HDFS相关命令</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#MapReduce示例"><span class="nav-number">5.</span> <span class="nav-text">MapReduce示例</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#思考"><span class="nav-number">6.</span> <span class="nav-text">思考</span></a></li></ol></div></div></section></div></aside></div></main><footer id="footer" class="footer"><div class="footer-inner"><div class="copyright">© <span itemprop="copyrightYear">2020</span><span class="with-love"><i class="fa fa-user"></i></span> <span class="author" itemprop="copyrightHolder">AloofJr</span> <span class="post-meta-divider">|</span><span class="post-meta-item-icon"><i class="fa fa-area-chart"></i></span> <span class="post-meta-item-text">Site words total count:</span> <span title="Site words total count">79.4k</span></div><script>!function(){var t=document.createElement("script"),e=window.location.protocol.split(":")[0];t.src="https"===e?"https://zz.bdstatic.com/linksubmit/push.js":"http://push.zhanzhang.baidu.com/push.js";var s=document.getElementsByTagName("script")[0];s.parentNode.insertBefore(t,s)}()</script><div class="busuanzi-count"><script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script><span class="site-uv"><i class="fa fa-user"></i><span class="busuanzi-value" id="busuanzi_value_site_uv"></span></span><span class="site-pv"><i class="fa fa-eye"></i><span class="busuanzi-value" id="busuanzi_value_site_pv"></span></span></div></div></footer><div class="back-to-top"><i class="fa fa-arrow-up"></i></div></div><script type="text/javascript">"[object Function]"!==Object.prototype.toString.call(window.Promise)&&(window.Promise=null)</script><script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script><script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script><script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script><script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script><script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script><script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script><script type="text/javascript" src="/js/src/utils.js?v=5.1.4"></script><script type="text/javascript" src="/js/src/motion.js?v=5.1.4"></script><script type="text/javascript" src="/js/src/affix.js?v=5.1.4"></script><script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.1.4"></script><script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.4"></script><script type="text/javascript" src="/js/src/post-details.js?v=5.1.4"></script><script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.4"></script><script src="//cdn1.lncld.net/static/js/3.0.4/av-min.js"></script><script src="//unpkg.com/valine/dist/Valine.min.js"></script><script type="text/javascript">
var GUEST = ['nick','mail','link'];
var guest = 'nick,mail';
guest = guest.split(',').filter(item=>{
return GUEST.indexOf(item)>-1;
});
new Valine({
el: '#comments' ,
verify: true,
notify: true,
appId: 'AtTFeTEomwo5ADIQTR8YFRcL-gzGzoHsz',
appKey: 'vKO8qLF0TzyALMhWTxgQ7GPh',
placeholder: '欢迎各路大神交流指正！',
avatar:'mm',
guest_info:guest,
pageSize:'10' || 10,
});
</script><script type="text/javascript">
// Popup Window;
var isfetched = false;
var isXml = true;
// Search DB path;
var search_path = "search.xml";
if (search_path.length === 0) {
search_path = "search.xml";
} else if (/json$/i.test(search_path)) {
isXml = false;
}
var path = "/" + search_path;
// monitor main search box;
var onPopupClose = function (e) {
$('.popup').hide();
$('#local-search-input').val('');
$('.search-result-list').remove();
$('#no-result').remove();
$(".local-search-pop-overlay").remove();
$('body').css('overflow', '');
}
function proceedsearch() {
$("body")
.append('<div class="search-popup-overlay local-search-pop-overlay"></div>')
.css('overflow', 'hidden');
$('.search-popup-overlay').click(onPopupClose);
$('.popup').toggle();
var $localSearchInput = $('#local-search-input');
$localSearchInput.attr("autocapitalize", "none");
$localSearchInput.attr("autocorrect", "off");
$localSearchInput.focus();
}
// search function;
var searchFunc = function(path, search_id, content_id) {
'use strict';
// start loading animation
$("body")
.append('<div class="search-popup-overlay local-search-pop-overlay">' +
'<div id="search-loading-icon">' +
'<i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>' +
'</div>' +
'</div>')
.css('overflow', 'hidden');
$("#search-loading-icon").css('margin', '20% auto 0 auto').css('text-align', 'center');
$.ajax({
url: path,
dataType: isXml ? "xml" : "json",
async: true,
success: function(res) {
// get the contents from search data
isfetched = true;
$('.popup').detach().appendTo('.header-inner');
var datas = isXml ? $("entry", res).map(function() {
return {
title: $("title", this).text(),
content: $("content",this).text(),
url: $("url" , this).text()
};
}).get() : res;
var input = document.getElementById(search_id);
var resultContent = document.getElementById(content_id);
var inputEventFunction = function() {
var searchText = input.value.trim().toLowerCase();
var keywords = searchText.split(/[\s\-]+/);
if (keywords.length > 1) {
keywords.push(searchText);
}
var resultItems = [];
if (searchText.length > 0) {
// perform local searching
datas.forEach(function(data) {
var isMatch = false;
var hitCount = 0;
var searchTextCount = 0;
var title = data.title.trim();
var titleInLowerCase = title.toLowerCase();
var content = data.content.trim().replace(/<[^>]+>/g,"");
var contentInLowerCase = content.toLowerCase();
var articleUrl = decodeURIComponent(data.url);
var indexOfTitle = [];
var indexOfContent = [];
// only match articles with not empty titles
if(title != '') {
keywords.forEach(function(keyword) {
function getIndexByWord(word, text, caseSensitive) {
var wordLen = word.length;
if (wordLen === 0) {
return [];
}
var startPosition = 0, position = [], index = [];
if (!caseSensitive) {
text = text.toLowerCase();
word = word.toLowerCase();
}
while ((position = text.indexOf(word, startPosition)) > -1) {
index.push({position: position, word: word});
startPosition = position + wordLen;
}
return index;
}
indexOfTitle = indexOfTitle.concat(getIndexByWord(keyword, titleInLowerCase, false));
indexOfContent = indexOfContent.concat(getIndexByWord(keyword, contentInLowerCase, false));
});
if (indexOfTitle.length > 0 || indexOfContent.length > 0) {
isMatch = true;
hitCount = indexOfTitle.length + indexOfContent.length;
}
}
// show search results
if (isMatch) {
// sort index by position of keyword
[indexOfTitle, indexOfContent].forEach(function (index) {
index.sort(function (itemLeft, itemRight) {
if (itemRight.position !== itemLeft.position) {
return itemRight.position - itemLeft.position;
} else {
return itemLeft.word.length - itemRight.word.length;
}
});
});
// merge hits into slices
function mergeIntoSlice(text, start, end, index) {
var item = index[index.length - 1];
var position = item.position;
var word = item.word;
var hits = [];
var searchTextCountInSlice = 0;
while (position + word.length <= end && index.length != 0) {
if (word === searchText) {
searchTextCountInSlice++;
}
hits.push({position: position, length: word.length});
var wordEnd = position + word.length;
// move to next position of hit
index.pop();
while (index.length != 0) {
item = index[index.length - 1];
position = item.position;
word = item.word;
if (wordEnd > position) {
index.pop();
} else {
break;
}
}
}
searchTextCount += searchTextCountInSlice;
return {
hits: hits,
start: start,
end: end,
searchTextCount: searchTextCountInSlice
};
}
var slicesOfTitle = [];
if (indexOfTitle.length != 0) {
slicesOfTitle.push(mergeIntoSlice(title, 0, title.length, indexOfTitle));
}
var slicesOfContent = [];
while (indexOfContent.length != 0) {
var item = indexOfContent[indexOfContent.length - 1];
var position = item.position;
var word = item.word;
// cut out 100 characters
var start = position - 20;
var end = position + 80;
if(start < 0){
start = 0;
}
if (end < position + word.length) {
end = position + word.length;
}
if(end > content.length){
end = content.length;
}
slicesOfContent.push(mergeIntoSlice(content, start, end, indexOfContent));
}
// sort slices in content by search text's count and hits' count
slicesOfContent.sort(function (sliceLeft, sliceRight) {
if (sliceLeft.searchTextCount !== sliceRight.searchTextCount) {
return sliceRight.searchTextCount - sliceLeft.searchTextCount;
} else if (sliceLeft.hits.length !== sliceRight.hits.length) {
return sliceRight.hits.length - sliceLeft.hits.length;
} else {
return sliceLeft.start - sliceRight.start;
}
});
// select top N slices in content
var upperBound = parseInt('1');
if (upperBound >= 0) {
slicesOfContent = slicesOfContent.slice(0, upperBound);
}
// highlight title and content
function highlightKeyword(text, slice) {
var result = '';
var prevEnd = slice.start;
slice.hits.forEach(function (hit) {
result += text.substring(prevEnd, hit.position);
var end = hit.position + hit.length;
result += '<b class="search-keyword">' + text.substring(hit.position, end) + '</b>';
prevEnd = end;
});
result += text.substring(prevEnd, slice.end);
return result;
}
var resultItem = '';
if (slicesOfTitle.length != 0) {
resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + highlightKeyword(title, slicesOfTitle[0]) + "</a>";
} else {
resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + title + "</a>";
}
slicesOfContent.forEach(function (slice) {
resultItem += "<a href='" + articleUrl + "'>" +
"<p class=\"search-result\">" + highlightKeyword(content, slice) +
"...</p>" + "</a>";
});
resultItem += "</li>";
resultItems.push({
item: resultItem,
searchTextCount: searchTextCount,
hitCount: hitCount,
id: resultItems.length
});
}
})
};
if (keywords.length === 1 && keywords[0] === "") {
resultContent.innerHTML = '<div id="no-result"><i class="fa fa-search fa-5x" /></div>'
} else if (resultItems.length === 0) {
resultContent.innerHTML = '<div id="no-result"><i class="fa fa-frown-o fa-5x" /></div>'
} else {
resultItems.sort(function (resultLeft, resultRight) {
if (resultLeft.searchTextCount !== resultRight.searchTextCount) {
return resultRight.searchTextCount - resultLeft.searchTextCount;
} else if (resultLeft.hitCount !== resultRight.hitCount) {
return resultRight.hitCount - resultLeft.hitCount;
} else {
return resultRight.id - resultLeft.id;
}
});
var searchResultList = '<ul class=\"search-result-list\">';
resultItems.forEach(function (result) {
searchResultList += result.item;
})
searchResultList += "</ul>";
resultContent.innerHTML = searchResultList;
}
}
if ('auto' === 'auto') {
input.addEventListener('input', inputEventFunction);
} else {
$('.search-icon').click(inputEventFunction);
input.addEventListener('keypress', function (event) {
if (event.keyCode === 13) {
inputEventFunction();
}
});
}
// remove loading animation
$(".local-search-pop-overlay").remove();
$('body').css('overflow', '');
proceedsearch();
}
});
}
// handle and trigger popup window;
$('.popup-trigger').click(function(e) {
e.stopPropagation();
if (isfetched === false) {
searchFunc(path, 'local-search-input', 'local-search-result');
} else {
proceedsearch();
};
});
$('.popup-btn-close').click(onPopupClose);
$('.popup').click(function(e){
e.stopPropagation();
});
$(document).on('keyup', function (event) {
var shouldDismissSearchPopup = event.which === 27 &&
$('.search-popup').is(':visible');
if (shouldDismissSearchPopup) {
onPopupClose();
}
});
</script><script>!function(){var t=document.createElement("script"),e=window.location.protocol.split(":")[0];t.src="https"===e?"https://zz.bdstatic.com/linksubmit/push.js":"http://push.zhanzhang.baidu.com/push.js";var s=document.getElementsByTagName("script")[0];s.parentNode.insertBefore(t,s)}()</script></body></html>