/
aks-cni-i.html
511 lines (389 loc) · 81.9 KB
/
aks-cni-i.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width">
<meta name="theme-color" content="#222" media="(prefers-color-scheme: light)">
<meta name="theme-color" content="#222" media="(prefers-color-scheme: dark)"><meta name="generator" content="Hexo 7.0.0-rc1">
<link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
<link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
<link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
<link rel="mask-icon" href="/images/logo.svg" color="#222">
<link rel="stylesheet" href="/css/main.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css" integrity="sha256-HtsXJanqjKTc8vVQjO4YMhiqFoXkfBsjBWcX91T1jr8=" crossorigin="anonymous">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/animate.css/3.1.1/animate.min.css" integrity="sha256-PR7ttpcvz8qrF57fur/yAx1qXMFJeJFiA6pSzWi0OIE=" crossorigin="anonymous">
<script class="next-config" data-name="main" type="application/json">{"hostname":"www.hwchiu.com","root":"/","images":"/images","scheme":"Gemini","darkmode":true,"version":"8.17.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12},"copycode":{"enable":false,"style":null},"bookmark":{"enable":false,"color":"#222","save":"auto"},"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"stickytabs":false,"motion":{"enable":true,"async":false,"transition":{"menu_item":"fadeInDown","post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"i18n":{"placeholder":"Searching...","empty":"We didn't find any results for the search: ${query}","hits_time":"${hits} results found in ${time} ms","hits":"${hits} results found"}}</script><script src="/js/config.js"></script>
<meta name="description" content="除了自行架設 Kubernetes 之外，採用公有雲廠商所提供的 Kubernetes Service 也是一個方便的選擇，然而這種情況下有許多的設定跟功能都會依賴該公有雲廠商自行實作，大部分的功能都會與公有雲本身的架構進行高度整合以提供更方便的使用與操作。本文針對 Container Network Interface (CNI) 於 Azure 中的實現與使用進行了討論，藉此了解公有雲的 CN">
<meta property="og:type" content="article">
<meta property="og:title" content="Azure Kubernetes Service (AKS) - CNI (I)">
<meta property="og:url" content="https://www.hwchiu.com/aks-cni-i.html">
<meta property="og:site_name" content="Hwchiu Learning Note">
<meta property="og:description" content="除了自行架設 Kubernetes 之外，採用公有雲廠商所提供的 Kubernetes Service 也是一個方便的選擇，然而這種情況下有許多的設定跟功能都會依賴該公有雲廠商自行實作，大部分的功能都會與公有雲本身的架構進行高度整合以提供更方便的使用與操作。本文針對 Container Network Interface (CNI) 於 Azure 中的實現與使用進行了討論，藉此了解公有雲的 CN">
<meta property="og:locale" content="en_US">
<meta property="og:image" content="https://i.imgur.com/dsrygTc.png">
<meta property="og:image" content="https://i.imgur.com/N1a8mEk.png">
<meta property="og:image" content="https://i.imgur.com/DSzvmqc.png">
<meta property="og:image" content="https://i.imgur.com/robQsi1.png">
<meta property="article:published_time" content="2019-03-25T16:32:42.000Z">
<meta property="article:modified_time" content="2023-06-23T05:16:12.598Z">
<meta property="article:author" content="Hwchiu">
<meta property="article:tag" content="Kubernetes">
<meta property="article:tag" content="Network">
<meta property="article:tag" content="Azure">
<meta property="article:tag" content="CNI">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://i.imgur.com/dsrygTc.png">
<link rel="canonical" href="https://www.hwchiu.com/aks-cni-i.html">
<script class="next-config" data-name="page" type="application/json">{"sidebar":"","isHome":false,"isPost":true,"lang":"en","comments":true,"permalink":"https://www.hwchiu.com/aks-cni-i.html","path":"aks-cni-i.html","title":"Azure Kubernetes Service (AKS) - CNI (I)"}</script>
<script class="next-config" data-name="calendar" type="application/json">""</script>
<title>Azure Kubernetes Service (AKS) - CNI (I) | Hwchiu Learning Note</title>
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-54006186-1"></script>
<script class="next-config" data-name="google_analytics" type="application/json">{"tracking_id":"UA-54006186-1","only_pageview":false}</script>
<script src="/js/third-party/analytics/google-analytics.js"></script>
<noscript>
<link rel="stylesheet" href="/css/noscript.css">
</noscript>
</head>
<body itemscope itemtype="http://schema.org/WebPage" class="use-motion">
<div class="headband"></div>
<main class="main">
<div class="column">
<header class="header" itemscope itemtype="http://schema.org/WPHeader"><div class="site-brand-container">
<div class="site-nav-toggle">
<div class="toggle" aria-label="Toggle navigation bar" role="button">
<span class="toggle-line"></span>
<span class="toggle-line"></span>
<span class="toggle-line"></span>
</div>
</div>
<div class="site-meta">
<a href="/" class="brand" rel="start">
<i class="logo-line"></i>
<p class="site-title">Hwchiu Learning Note</p>
<i class="logo-line"></i>
</a>
<p class="site-subtitle" itemprop="description">kubernetes, sdn, linux,devops</p>
<img class="custom-logo-image" src="/uploads/hwchiu.jpg" alt="Hwchiu Learning Note">
</div>
<div class="site-nav-right">
<div class="toggle popup-trigger" aria-label="Search" role="button">
</div>
</div>
</div>
<nav class="site-nav">
<ul class="main-menu menu"><li class="menu-item menu-item-home"><a href="/" rel="section"><i class="fa fa-home fa-fw"></i>Home</a></li><li class="menu-item menu-item-about"><a href="/about/" rel="section"><i class="fa fa-user fa-fw"></i>About</a></li><li class="menu-item menu-item-tags"><a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>Tags</a></li><li class="menu-item menu-item-archives"><a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>Archives</a></li><li class="menu-item menu-item-sitemap"><a href="/sitemap.xml" rel="section"><i class="fa fa-sitemap fa-fw"></i>Sitemap</a></li>
</ul>
</nav>
</header>
<aside class="sidebar">
<div class="sidebar-inner sidebar-nav-active sidebar-toc-active">
<ul class="sidebar-nav">
<li class="sidebar-nav-toc">
Table of Contents
</li>
<li class="sidebar-nav-overview">
Overview
</li>
</ul>
<div class="sidebar-panel-container">
<!--noindex-->
<div class="post-toc-wrap sidebar-panel">
<div class="post-toc animated"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#Preface"><span class="nav-number">1.</span> <span class="nav-text">Preface</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Introduction"><span class="nav-number">2.</span> <span class="nav-text">Introduction</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Azure-Container-Networking-CNI"><span class="nav-number">3.</span> <span class="nav-text">Azure-Container-Networking(CNI)</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#Azure-VNET"><span class="nav-number">3.1.</span> <span class="nav-text">Azure-VNET</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Azure-VNET-IPAM"><span class="nav-number">3.2.</span> <span class="nav-text">Azure-VNET-IPAM</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#API"><span class="nav-number">3.3.</span> <span class="nav-text">API</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Capacity"><span class="nav-number">3.4.</span> <span class="nav-text">Capacity</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Summary"><span class="nav-number">4.</span> <span class="nav-text">Summary</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Reference"><span class="nav-number">5.</span> <span class="nav-text">Reference</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#%E5%80%8B%E4%BA%BA%E8%B3%87%E8%A8%8A"><span class="nav-number">6.</span> <span class="nav-text">個人資訊</span></a></li></ol></div>
</div>
<!--/noindex-->
<div class="site-overview-wrap sidebar-panel">
<div class="site-author animated" itemprop="author" itemscope itemtype="http://schema.org/Person">
<img class="site-author-image" itemprop="image" alt="Hwchiu"
src="/uploads/avatar.jpg">
<p class="site-author-name" itemprop="name">Hwchiu</p>
<div class="site-description" itemprop="description">kubernetes/SDN/DevOps</div>
</div>
<div class="site-state-wrap animated">
<nav class="site-state">
<div class="site-state-item site-state-posts">
<a href="/archives/">
<span class="site-state-item-count">352</span>
<span class="site-state-item-name">posts</span>
</a>
</div>
<div class="site-state-item site-state-tags">
<a href="/tags/">
<span class="site-state-item-count">115</span>
<span class="site-state-item-name">tags</span></a>
</div>
</nav>
</div>
<div class="links-of-author animated">
<span class="links-of-author-item">
<a href="https://github.com/hwchiu" title="GitHub → https://github.com/hwchiu" rel="noopener me" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
</span>
<span class="links-of-author-item">
<a href="mailto:sppsorrg@gmail.com" title="E-Mail → mailto:sppsorrg@gmail.com" rel="noopener me" target="_blank"><i class="fa fa-envelope fa-fw"></i>E-Mail</a>
</span>
<span class="links-of-author-item">
<a href="https://twitter.com/hw_chiu" title="Twitter → https://twitter.com/hw_chiu" rel="noopener me" target="_blank"><i class="fab fa-twitter fa-fw"></i>Twitter</a>
</span>
<span class="links-of-author-item">
<a href="https://www.facebook.com/technologynoteniu" title="FB Page → https://www.facebook.com/technologynoteniu" rel="noopener me" target="_blank"><i class="fab fa-facebook fa-fw"></i>FB Page</a>
</span>
<span class="links-of-author-item">
<a href="https://www.youtube.com/channel/UCoYY8K9fbfDtTY7m68UCATA/videos" title="YouTube → https://www.youtube.com/channel/UCoYY8K9fbfDtTY7m68UCATA/videos" rel="noopener me" target="_blank"><i class="fab fa-youtube fa-fw"></i>YouTube</a>
</span>
<span class="links-of-author-item">
<a href="https://instagram.com/hwchiu" title="Instagram → https://instagram.com/hwchiu" rel="noopener me" target="_blank"><i class="fab fa-instagram fa-fw"></i>Instagram</a>
</span>
</div>
</div>
</div>
</div>
</aside>
</div>
<div class="main-inner post posts-expand">
<div class="post-block">
<article itemscope itemtype="http://schema.org/Article" class="post-content" lang="en">
<link itemprop="mainEntityOfPage" href="https://www.hwchiu.com/aks-cni-i.html">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/uploads/avatar.jpg">
<meta itemprop="name" content="Hwchiu">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="Hwchiu Learning Note">
<meta itemprop="description" content="kubernetes/SDN/DevOps">
</span>
<span hidden itemprop="post" itemscope itemtype="http://schema.org/CreativeWork">
<meta itemprop="name" content="Azure Kubernetes Service (AKS) - CNI (I) | Hwchiu Learning Note">
<meta itemprop="description" content="除了自行架設 Kubernetes 之外，採用公有雲廠商所提供的 Kubernetes Service 也是一個方便的選擇，然而這種情況下有許多的設定跟功能都會依賴該公有雲廠商自行實作，大部分的功能都會與公有雲本身的架構進行高度整合以提供更方便的使用與操作。本文針對 Container Network Interface (CNI) 於 Azure 中的實現與使用進行了討論，藉此了解公有雲的 CNI 有什麼特別的設計與使用方式">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
Azure Kubernetes Service (AKS) - CNI (I)
</h1>
<div class="post-meta-container">
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">Posted on</span>
<time title="Created: 2019-03-26 00:32:42" itemprop="dateCreated datePublished" datetime="2019-03-26T00:32:42+08:00">2019-03-26</time>
</span>
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar-check"></i>
</span>
<span class="post-meta-item-text">Edited on</span>
<time title="Modified: 2023-06-23 13:16:12" itemprop="dateModified" datetime="2023-06-23T13:16:12+08:00">2023-06-23</time>
</span>
<span class="post-meta-item" title="Views" id="busuanzi_container_page_pv">
<span class="post-meta-item-icon">
<i class="far fa-eye"></i>
</span>
<span class="post-meta-item-text">Views: </span>
<span id="busuanzi_value_page_pv"></span>
</span>
</div>
<div class="post-description">除了自行架設 Kubernetes 之外，採用公有雲廠商所提供的 Kubernetes Service 也是一個方便的選擇，然而這種情況下有許多的設定跟功能都會依賴該公有雲廠商自行實作，大部分的功能都會與公有雲本身的架構進行高度整合以提供更方便的使用與操作。本文針對 Container Network Interface (CNI) 於 Azure 中的實現與使用進行了討論，藉此了解公有雲的 CNI 有什麼特別的設計與使用方式</div>
</div>
</header>
<div class="post-body" itemprop="articleBody"><h1 id="Preface"><a href="#Preface" class="headerlink" title="Preface"></a>Preface</h1><p>除了自行架設 Kubernetes 之外，愈來愈多的公有雲提供商也都提供了 Kubernetes 的服務，從 Google 的 <code>GKE</code>(Google Kubernetes Engine), Amazon 的 <code>AKS</code>(Amazon Kubernetes Service) 到 Azure 的 <code>AKS</code>(Azure Kubernetes Service)，使用者都可以很簡易的透過其提供的操控介面創造一個全新的 <code>kubernetes</code> 集群來使用。</p>
<p>對我來說，這些公有雲集群與自建集群有非常多的差異，由於這些公有雲所提供的 <code>kubernetes</code> 都基於這些公有雲的基本架構上，所以有特別多的資源可以進行整合，譬如 <code>Load-Balancer/Ingress</code> 就可以搭配自家的 <code>LoadBalancer</code> 來使用，同時 <code>Ingress</code> 所使用的 <code>hostname</code> 也能夠跟相關的 <code>DNS</code> 整合。此外還有一個很明顯的差異就是 <code>IP</code> 的分配及使用。 不論是 <code>AKS</code> 或是 <code>EKS</code> 都提供了讓 <code>Node</code> 與 <code>Pod</code> 使用相同的 <code>IP/Subnet</code> 的機制，而這部分則是牽扯到 <code>Container Network Interface</code> 的設計，因此如 <code>AKS/EKS</code> 都有開源使用到的 <code>CNI</code>.</p>
<p><a target="_blank" rel="noopener" href="https://github.com/aws/amazon-vpc-cni-k8s">Amazon-VPC-CNI-K8S</a><br><a target="_blank" rel="noopener" href="https://github.com/Azure/azure-container-networking">Azure-Container-Networking</a></p>
<p>透過研究這些 <code>CNI</code> 的設計原理，可以學習到滿多的設計想法，同時若今天也有相同的需求，也可以作為一個參考對象去學習。</p>
<p>接下來會針對 <code>AKS</code> 的部分去探討其 <code>CNI</code> 的運作，並且分成兩篇文章來研究。第一篇文章主要會針對 <code>IP</code> 的管理，包含了分配與設定，以及相關的檔案資訊，第二篇文章則會針對 <code>Routing</code> 的設定，這種網路設定下，實際上封包再傳輸的時候有哪些不同於 <code>Flannel/Calico/Bridge</code> 的設定與用法。</p>
<h1 id="Introduction"><a href="#Introduction" class="headerlink" title="Introduction"></a>Introduction</h1><p>正式探討 <code>Azure-Container-Networking CNI</code> 的運作之前，有一些基本的概念要先瞭解，當你透過 <code>Azure Portal</code> 創立一個三個節點的 <code>AKS</code> 服務後，<code>Azure</code> 會幫你創建不少相關的資源, 其中跟本文關係較重要的幾個分別是</p>
<ol>
<li>Nodes (Virtual Machines)</li>
<li>VNet (Virtual Networks)</li>
<li>Kubernetes</li>
</ol>
<p>Nodes 的數量就是 <code>AKS</code> 的節點數量，而節點上能夠擁有的 <code>IP/Subnet</code> 則都是在 <code>VNet</code> 該資源中管理。 該 <code>Vnet</code> 所管理的 <code>IP/Subnet</code> 不但跟節點有關，也跟運行在該節點上的 <code>Pod</code> 有關，因為 <code>Pod</code> 最終使用的 <code>IP/Subnet</code> 都是在 <code>VNet</code> 這邊去設定與控管。<br>而這些 Nodes 之間的網路連線則是 <code>Azure</code> 本身幫忙處理的，實際上是透過 <code>Azure SDN Fabric</code> 的概念來幫忙處理這些節點間的網路傳輸，包含相關的 <code>Virtual Networks</code> 的處理。<br>最後則是基於這些架構下於節點上安裝 <code>kubernetes</code> 相關的元件並提供一個可操作的 <code>kubernetes</code> 叢集。</p>
<p>本文的所有環境都基於下列的 <code>AKS</code> 架構圖，如下圖。</p>
<p><img src="https://i.imgur.com/dsrygTc.png" alt="Imgur"><br><img src="https://i.imgur.com/N1a8mEk.png" alt="Imgur"></p>
<p>此外可以參考這篇<a target="_blank" rel="noopener" href="https://docs.microsoft.com/en-us/azure/aks/ssh">官方文章學習</a>如何透過 <code>SSH</code> 連線到 <code>AKS</code> 的節點中來進行本文的研究</p>
<h1 id="Azure-Container-Networking-CNI"><a href="#Azure-Container-Networking-CNI" class="headerlink" title="Azure-Container-Networking(CNI)"></a>Azure-Container-Networking(CNI)</h1><p><code>Container Network Interface CNI</code> 是一套規範，用來處理容器本身的網路設定，至於要提供什麼樣的網路能力本身並沒有定義，而是根據不同的 <code>CNI</code> 實現自行決定，根據不同的情境與設定來提供不同的功能與效果。</p>
<p><code>CNI</code> 本身的運作原理是以機器為單位，不以叢集為單位，因此每台機器都要安裝欲使用的 <code>CNI</code> 執行檔以及對應的設定檔案。<br>以本文的範例為例，三個節點就意味三個節點上都會安裝對應的 <code>CNI</code> 執行檔，也就是 <code>Azure-Container-Networking</code> 專案的產物，同時也會有對應的設定檔案。</p>
<p>我們可以於 <code>AKS</code> 的節點上面的 <code>/opt/cni/bin</code> 中觀察到目前系統中安裝的 <code>CNI</code> 執行檔</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">azureuser@aks-agentpool-15026905-1:~$ <span class="built_in">ls</span> /opt/cni/bin/</span><br><span class="line">azure-vnet bridge flannel host-local loopback portmap sample vlan</span><br><span class="line">azure-vnet-ipam dhcp host-device ipvlan macvlan ptp tuning</span><br></pre></td></tr></table></figure>
<p>其中 <code>azure-vnet</code>, <code>azure-vnet-ipam</code> 兩個則是 <code>Azure-Container-Networking</code> 這個專案所產生的，一個用來處理網路連接，一個用來處理 <code>IP/Subnet</code> 的取得與設定</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line">azureuser@aks-agentpool-15026905-1:~$ <span class="built_in">ls</span> /etc/cni/net.d/</span><br><span class="line">10-azure.conflist</span><br><span class="line">azureuser@aks-agentpool-15026905-1:~$ sudo <span class="built_in">cat</span> /etc/cni/net.d/10-azure.conflist</span><br><span class="line">{</span><br><span class="line"> <span class="string">"cniVersion"</span>:<span class="string">"0.3.0"</span>,</span><br><span class="line"> <span class="string">"name"</span>:<span class="string">"azure"</span>,</span><br><span class="line"> <span class="string">"plugins"</span>:[</span><br><span class="line"> {</span><br><span class="line"> <span class="string">"type"</span>:<span class="string">"azure-vnet"</span>,</span><br><span class="line"> <span class="string">"mode"</span>:<span class="string">"bridge"</span>,</span><br><span class="line"> <span class="string">"bridge"</span>:<span class="string">"azure0"</span>,</span><br><span class="line"> <span class="string">"ipam"</span>:{</span><br><span class="line"> <span class="string">"type"</span>:<span class="string">"azure-vnet-ipam"</span></span><br><span class="line"> }</span><br><span class="line"> },</span><br><span class="line"> {</span><br><span class="line"> <span class="string">"type"</span>:<span class="string">"portmap"</span>,</span><br><span class="line"> <span class="string">"capabilities"</span>:{</span><br><span class="line"> <span class="string">"portMappings"</span>:<span class="literal">true</span></span><br><span class="line"> },</span><br><span class="line"> <span class="string">"snat"</span>:<span class="literal">true</span></span><br><span class="line"> }</span><br><span class="line"> ]</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<p>這個設定檔案的內容我們先暫時不去解讀，只要知道每一台 <code>AKS</code> 節點上都會有一個獨立的檔案來負責該節點的 <code>CNI</code> 即可。</p>
<p><code>CNI</code> 的運作流程簡單來說可以分成多個步驟，分別是</p>
<ol>
<li>Kubelet 收到通知要創建對應的 Pod</li>
<li>Kubelet 透過 <code>CRI</code> 創建 <code>Pause Container</code></li>
<li>Kubelet 呼叫 <code>CNI</code> 並且將 <code>Pause Container</code> 的部份資訊當作參數傳給 <code>CNI</code></li>
<li><code>CNI</code> 根據自己的實現為 <code>Pause Container</code> 提供網路能力</li>
<li>Kubelet 相信 <code>CNI</code> 已經完成所需任務，接下來創建使用者需要的 <code>Containers</code>，這些 <code>Containers</code> 的網路空間都直接掛載到 <code>Pause Container</code> 上</li>
</ol>
<p>而本文主要探討的 <code>Azure-Vnet CNI</code> 主要負責的部分就是 <strong>4</strong> 所描述的工作，為 <code>Pause Container</code> 提供網路能力。</p>
<p>此外對於 CNI 這個議題有興趣的人可以參閱下列系列文章來學習更多 CNI 相關的資訊</p>
<ol>
<li><a href="https://www.hwchiu.com/cni-compare.html">常見 CNI (Container Network Interface) Plugin 介紹</a></li>
<li><a href="https://www.hwchiu.com/introduce-cni-ii.html">[Container Network Interface] CNI Introduction</a></li>
<li><a href="https://www.hwchiu.com/introduce-cni-i.html">[Container Network Interface] Bridge Network In Docker</a></li>
<li><a href="https://www.hwchiu.com/introduce-cni-iii.html">[Container Network Interface] Write a CNI Plugin By Golang</a></li>
<li><a href="(https://www.hwchiu.com/cni-questions.html">CNI 常見問題整理</a></li>
</ol>
<h2 id="Azure-VNET"><a href="#Azure-VNET" class="headerlink" title="Azure-VNET"></a>Azure-VNET</h2><p><code>Azure-VNET CNI</code> 的詳細原始碼都在 <a target="_blank" rel="noopener" href="https://github.com/Azure/azure-container-networking/tree/master/cni">Github-Azure-Container-Networking</a>, 有興趣的讀者可以自行閱讀來學習。</p>
<p>基本上 <code>Azure-VNET</code> 跟常見的 <code>L2 Linux Bridge</code> 非常相似，其運作流程如下</p>
<ol>
<li>創建一個 Linux Bridge <code>Azure0</code> (若存在就不創造)</li>
<li>創造一條 <code>Veth</code> 的 <code>Linux Logical Link</code></li>
<li>將該 <code>Veth</code> 的一端放到 <code>Pause Container</code> 內，並且命名為 <code>eth0</code>.</li>
<li>將該 <code>Veth</code> 的另一端綁到 <code>Azure0</code> 上，該 <code>Veth</code> 的名稱都是會 <code>azvxxxxxxx</code></li>
<li>呼叫 <code>Azure-VNET-IPAM</code> 去取得可用的 <code>IP/Subnet</code> 並且設定到 <code>Pause Container</code> 內的 <code>eth0</code> 介面<ul>
<li>這部分會在下個章節解釋其運作</li>
</ul>
</li>
</ol>
<p>有興趣的讀者也可以參閱 <a target="_blank" rel="noopener" href="https://docs.microsoft.com/en-us/azure/aks/ssh">AKS SSH to Node</a> 這篇文章的方式連接到 <code>AKS</code> 內部的節點來實際看看這些資訊</p>
<p>我們可以使用 <code>brctl</code> 這個工具來觀察 <code>Linux Bridge</code> 的關係</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">azureuser@aks-agentpool-15026905-1:~$ brctl show</span><br><span class="line">bridge name bridge <span class="built_in">id</span> STP enabled interfaces</span><br><span class="line">azure0 8000.000d3a51cdbb no azv1be2dcd6c83</span><br><span class="line"> azv2705efbd6d8</span><br><span class="line"> azv499967b4ec4</span><br><span class="line"> azv4d966079c93</span><br><span class="line"> azv5a6822c76a3</span><br><span class="line"> azv93928864e55</span><br><span class="line"> azva985f33c456</span><br><span class="line"> azvaa217c54e39</span><br><span class="line"> azvbc5198bad97</span><br><span class="line"> azvc1ab312d517</span><br><span class="line"> azvc92e6588502</span><br><span class="line"> eth0</span><br><span class="line">docker0 8000.024220b0b010 no</span><br></pre></td></tr></table></figure>
<p>基本上整個節點中的狀況如下圖，每個 <code>Pod</code> 都會透過 <code>Veth</code> 與 <code>Azure0 </code> 這個 <code>Linux Bridge</code>相連<br><img src="https://i.imgur.com/DSzvmqc.png" alt="Imgur"></p>
<p>基本上該節點上面有多少個沒有設定 <code>HostNetwork=true</code> 的 <code>Pod</code>, 就會有多少條對應的 <code>azvxxxx veth link</code>.</p>
<p>此外，我個人對於 <code>Azure-VNET CNI</code> 覺得很好也喜愛的地方就是留有大量地資訊在節點上，這部分不論是對於研究或是除錯都非常的好用。</p>
<p>該資訊被放置於 <code>/var/run/azure-vnet.json</code>, 當 <code>azure-vnet CNI</code> 每次被呼叫來執行對應工作的時候，都會詳細的紀錄這次的資訊，包含 <code>ContainerID</code>, <code>PodName</code>, <code>IP</code>, <code>Route</code>等各式各樣的設定資訊。</p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Network"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Version"</span><span class="punctuation">:</span> <span class="string">"v1.0.17"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"TimeStamp"</span><span class="punctuation">:</span> <span class="string">"2019-03-22T14:30:27.934886517Z"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"ExternalInterfaces"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"eth0"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Name"</span><span class="punctuation">:</span> <span class="string">"eth0"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Networks"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"azure"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Id"</span><span class="punctuation">:</span> <span class="string">"azure"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Mode"</span><span class="punctuation">:</span> <span class="string">"bridge"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"VlanId"</span><span class="punctuation">:</span> <span class="number">0</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Subnets"</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line"> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Family"</span><span class="punctuation">:</span> <span class="number">2</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Prefix"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"IP"</span><span class="punctuation">:</span> <span class="string">"10.240.0.0"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Mask"</span><span class="punctuation">:</span> <span class="string">"//8AAA=="</span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Gateway"</span><span class="punctuation">:</span> <span class="string">"10.240.0.1"</span></span><br><span class="line"> <span class="punctuation">}</span></span><br><span class="line"> <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Endpoints"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"1bd38ad8-eth0"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Id"</span><span class="punctuation">:</span> <span class="string">"1bd38ad8-eth0"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"SandboxKey"</span><span class="punctuation">:</span> <span class="string">""</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"IfName"</span><span class="punctuation">:</span> <span class="string">"eth0"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"HostIfName"</span><span class="punctuation">:</span> <span class="string">"azv5a6822c76a3"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"MacAddress"</span><span class="punctuation">:</span> <span class="string">"nkfKrRbG"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"InfraVnetIP"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"IP"</span><span class="punctuation">:</span> <span class="string">""</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Mask"</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">null</span></span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"IPAddresses"</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line"> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"IP"</span><span class="punctuation">:</span> <span class="string">"10.240.0.94"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Mask"</span><span class="punctuation">:</span> <span class="string">"//8AAA=="</span></span><br><span class="line"> <span class="punctuation">}</span></span><br><span class="line"> <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Gateways"</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line"> <span class="string">"10.240.0.1"</span></span><br><span class="line"> <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"DNS"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Suffix"</span><span class="punctuation">:</span> <span class="string">""</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Servers"</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line"> <span class="string">"168.63.129.16"</span></span><br><span class="line"> <span class="punctuation">]</span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Routes"</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line"> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Dst"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"IP"</span><span class="punctuation">:</span> <span class="string">"0.0.0.0"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Mask"</span><span class="punctuation">:</span> <span class="string">"AAAAAA=="</span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Src"</span><span class="punctuation">:</span> <span class="string">""</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Gw"</span><span class="punctuation">:</span> <span class="string">"10.240.0.1"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Protocol"</span><span class="punctuation">:</span> <span class="number">0</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"DevName"</span><span class="punctuation">:</span> <span class="string">""</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Scope"</span><span class="punctuation">:</span> <span class="string">"0"</span><span class="punctuation">}</span></span><br><span class="line"> <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"VlanID"</span><span class="punctuation">:</span> <span class="number">0</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"EnableSnatOnHost"</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"EnableInfraVnet"</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"EnableMultitenancy"</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"NetworkNameSpace"</span><span class="punctuation">:</span> <span class="string">"/proc/5336/ns/net"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"ContainerID"</span><span class="punctuation">:</span> <span class="string">"1bd38ad8d840dd1f84597d4343b3bd116188cd1e4a797cc31bdc1aa3dc654a5b"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"PODName"</span><span class="punctuation">:</span> <span class="string">"addon-http-application-routing-external-dns-74db4f974b-8w4wz"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"PODNameSpace"</span><span class="punctuation">:</span> <span class="string">"kube-system"</span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line">..................................</span><br></pre></td></tr></table></figure>
<p>上述 <code>Endpoints</code> 裡面的每個物件都會描述到一個 <code>Pause Container</code> 的網路環境與設定，不過對於 <code>CNI</code> 來說本身不在意是不是 <code>Pause Container</code>. 這部分是 <code>Kubelet</code> 自行實現的邏輯，所以基本上不會再 <code>CNI</code> 這邊看到 <code>Pause Container</code> 相關的文字。</p>
<h2 id="Azure-VNET-IPAM"><a href="#Azure-VNET-IPAM" class="headerlink" title="Azure-VNET-IPAM"></a>Azure-VNET-IPAM</h2><p>接下來要探討的是要如何分配 <code>IP/Subnet</code> 這件事情，基本上任意兩個 <code>Pod</code> 都不應該使用相同的 <code>IP/Subnet</code>。但是對於 <code>CNI</code> 這種非中央極權管理的執行程式來說，要做到不衝突就必須要有一些機制了。<br>各式各樣的 <code>CNI</code> 都有自己的機制來處理，不論是透過 <code>ectd</code> 或是自行實現集中式管理機制，只要能夠避免分配重複 <code>IP/Subnet</code> 即可</p>
<p>前面有提過，三大公有雲的 Kubernetes Service 相較於自建來說有更多的優勢與特色就是因為可以將 KUbernetes 與公有雲內的設施與狀態結合來提供更多的功能</p>
<p><code>Azure-VNET-IPAM</code> 這個 <code>IP/Subnet</code> 管理機制基本上就是與 <code>Azure</code> 的環境有高度整合，接下來我們來看一下其運作的原理。</p>
<p>每台機器上的 <code>Azure-VNET-IPAM</code> 這個 <code>CNI</code> 都會執行相同的運作原理，最後卻要取得不同的 <code>IP/Subnet</code>, 這整個運作原理如下</p>
<ol>
<li>每個 <code>Azure-VNET-IPAM CNI</code> 都會透過 <code>HTTP</code> 去詢問叢集 API, 來確認當前節點在 <code>VNET</code> 內可以擁有的 <code>IP</code> 數量</li>
<li>從可用的 <code>IP</code> 數量內隨機挑選一個 <code>IP</code> 並返回</li>
<li>最後 <code>Azure-VNET</code> 就會取得 <code>Azure-VNET-IPAM</code> 得到的 <code>IP/Subnet</code> 並且設定到對應的 <code>Pause Container</code> 裡面</li>
</ol>
<h2 id="API"><a href="#API" class="headerlink" title="API"></a>API</h2><p>從<a target="_blank" rel="noopener" href="https://github.com/Azure/azure-container-networking/blob/67debca9016218981b69ff954536418ea0903305/ipam/azure.go#L17-L23">原始碼</a>中可以觀察到下列 URL 的設定</p>
<figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">const</span> (</span><br><span class="line"> <span class="comment">// Host URL to query.</span></span><br><span class="line"> azureQueryUrl = <span class="string">"http://168.63.129.16/machine/plugins?comp=nmagent&type=getinterfaceinfov1"</span></span><br><span class="line"></span><br><span class="line"> <span class="comment">// Minimum time interval between consecutive queries.</span></span><br><span class="line"> azureQueryInterval = <span class="number">10</span> * time.Second</span><br><span class="line">)</span><br></pre></td></tr></table></figure>
<p>然而目前實際上使用沒有通，原因在於先前的 <a target="_blank" rel="noopener" href="https://github.com/Azure/azure-container-networking/commit/e5f6b0d03c96b3eab4ce76178ddb6d3d57fb92c9#diff-ce5c6e7b3ab2a2f573913afae54abd4b">Commit</a> 有修改過該 URL 的數值，而且該 Commit 距離這篇文章不到一個月前，所以我認為 Azure 上面還沒有採用新的版本，因此實際上操作時還是使用舊有的 URL。</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">Update host machine ip (#300)</span><br><span class="line"></span><br><span class="line">* Limiting the size of our buffered payload to ~2MB</span><br><span class="line"></span><br><span class="line">* Changing IPs for calls to host machines from 169.254.169.254 to 168.63.129.16.</span><br></pre></td></tr></table></figure>
<p>根據上述的原始碼，我們可以直接在 <code>AKS</code> 節點中直接透過 <code>curl</code> 的方式去詢問，結果如下</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line">azureuser@aks-agentpool-15026905-0:~$ curl "http://169.254.169.254/machine/plugins?comp=nmagent&type=getinterfaceinfov1"</span><br><span class="line"><span class="tag"><<span class="name">Interfaces</span>></span><span class="tag"><<span class="name">Interface</span> <span class="attr">MacAddress</span>=<span class="string">"000D3A51C490"</span> <span class="attr">IsPrimary</span>=<span class="string">"true"</span>></span><span class="tag"><<span class="name">IPSubnet</span> <span class="attr">Prefix</span>=<span class="string">"10.240.0.0/16"</span>></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.35"</span> <span class="attr">IsPrimary</span>=<span class="string">"true"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.36"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.37"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.38"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.39"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.40"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.41"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.42"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.43"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.44"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.45"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.46"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.47"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.48"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.49"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.50"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.51"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.52"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.53"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.54"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.55"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.56"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.57"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.58"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.59"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.60"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.61"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.62"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.63"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.64"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"><<span class="name">IPAddress</span> <span class="attr">Address</span>=<span class="string">"10.240.0.65"</span> <span class="attr">IsPrimary</span>=<span class="string">"false"</span>/></span></span><br><span class="line"><span class="tag"></<span class="name">IPSubnet</span>></span><span class="tag"></<span class="name">Interface</span>></span><span class="tag"></<span class="name">Interfaces</span>></span></span><br></pre></td></tr></table></figure>
<p>有趣的是，不同的 <code>AKS</code> 節點問到的結果會是不同的，所以每台節點上得 <code>AKS-VNET-IPAM</code> 都可以透過這個方式來取得該台節點上所擁有能夠使用的 <code>IP</code> 數量以及對應的網段。</p>
<p>此外 <code>AKS-VNET-IPAM</code> 也會在本機端記錄相對應的資訊, 檔案位置位於 <code>/var/run/azure-vnet-ipam.json</code></p>
<p>其內容主要會紀錄本機端可以使用的所有 <code>IP 位置</code></p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br></pre></td><td class="code"><pre><span class="line"><span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"IPAM"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Version"</span><span class="punctuation">:</span> <span class="string">"v1.0.17"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"TimeStamp"</span><span class="punctuation">:</span> <span class="string">"2019-03-18T21:48:15.199533623Z"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"AddressSpaces"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"local"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Id"</span><span class="punctuation">:</span> <span class="string">"local"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Scope"</span><span class="punctuation">:</span> <span class="number">0</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Pools"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"10.240.0.0/16"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"Id"</span><span class="punctuation">:</span> <span class="string">"10.240.0.0/16"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"IfName"</span><span class="punctuation">:</span> <span class="string">"eth0"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Subnet"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"IP"</span><span class="punctuation">:</span> <span class="string">"10.240.0.0"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Mask"</span><span class="punctuation">:</span> <span class="string">"//8AAA=="</span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Gateway"</span><span class="punctuation">:</span> <span class="string">"10.240.0.1"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Addresses"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"10.240.0.36"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"ID"</span><span class="punctuation">:</span> <span class="string">""</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Addr"</span><span class="punctuation">:</span> <span class="string">"10.240.0.36"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"InUse"</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">true</span></span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"10.240.0.37"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"ID"</span><span class="punctuation">:</span> <span class="string">""</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Addr"</span><span class="punctuation">:</span> <span class="string">"10.240.0.37"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"InUse"</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">true</span></span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"10.240.0.38"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"ID"</span><span class="punctuation">:</span> <span class="string">""</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Addr"</span><span class="punctuation">:</span> <span class="string">"10.240.0.38"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"InUse"</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">true</span></span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"10.240.0.39"</span><span class="punctuation">:</span> <span class="punctuation">{</span></span><br><span class="line"> <span class="attr">"ID"</span><span class="punctuation">:</span> <span class="string">""</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"Addr"</span><span class="punctuation">:</span> <span class="string">"10.240.0.39"</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">"InUse"</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span></span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">,</span></span><br><span class="line"> ..............</span><br><span class="line"> <span class="punctuation">}</span><span class="punctuation">}</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>
<p>基本上這些 <code>IP</code> 的資訊就跟 <code>Azure Portal</code> 裡面 <code>VNet (Virtual Networks)</code> 顯示的數量是一致的，因此我們可以透過 <code>Portal</code> 的方式就知道每台節點上運行的 <code>Pod</code> 的 <code>IP</code> 範圍。</p>
<h2 id="Capacity"><a href="#Capacity" class="headerlink" title="Capacity"></a>Capacity</h2><p>一個很有趣的問題這時候就會浮現了，因為每個節點上能夠使用的 <code>IP</code> 數量跟 <code>Virtual Networks</code> 裡面的每台節點有關，那如果我部署大量的 <code>Pod</code> 到該節點上是否會發生 <code>IP</code> 不足夠的問題?</p>
<p>針對這個問題，我嘗試部署了超過 <code>IP</code> 數量的 <code>Pod</code> 到節點上，結果最後看到的是滿滿的 <code>Pending</code></p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">hwchiu-utils-785f896cc5-cgjsq 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-4gmwv 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-brnl6 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-gwgcz 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-v5s45 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-knz2r 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-26pn2 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-xcmft 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-wlckh 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-dm49b 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-nkcgw 0/1 Pending 0 4m</span><br><span class="line">hwchiu-utils-785f896cc5-cwkbl 0/1 Pending 0 4m</span><br></pre></td></tr></table></figure>
<p>這意味者這些 <code>Pod</code> 在 <code>kubernetes scheduler/kubelet</code> 這階段就被阻止了，根本沒有機會讓 <code>CNI</code> 繼續往下執行，這時候我們可以研究一下該 <code>kubernetes node</code> 的設定, 可以發現到有趣的資訊</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">Capacity:</span><br><span class="line"> attachable-volumes-azure-disk: 8</span><br><span class="line"> cpu: 2</span><br><span class="line"> ephemeral-storage: 30428648Ki</span><br><span class="line"> hugepages-1Gi: 0</span><br><span class="line"> hugepages-2Mi: 0</span><br><span class="line"> memory: 7137108Ki</span><br><span class="line"> pods: 30</span><br><span class="line">Allocatable:</span><br><span class="line"> attachable-volumes-azure-disk: 8</span><br><span class="line"> cpu: 1931m</span><br><span class="line"> ephemeral-storage: 28043041951</span><br><span class="line"> hugepages-1Gi: 0</span><br><span class="line"> hugepages-2Mi: 0</span><br><span class="line"> memory: 5357396Ki</span><br><span class="line"> pods: 30</span><br></pre></td></tr></table></figure>
<p>這兩者的資訊可以參閱 <a target="_blank" rel="noopener" href="https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/">reserve-compute-resources</a></p>
<p>其中可以注意到 <code>Pod</code> 的數量被設定成 <code>30</code>，這個數字跟我們先前透過 <code>API</code> 去問到的 <code>IP</code> 數目幾乎是一致的<br>API 問到的 IP 還包括節點本身，所以是 30 + 1 = 31</p>
<p>所以這邊可以看到針對 <code>IP</code> 用光的問題， <code>AKS</code> 處理的方式除了 <code>CNI</code> 本身顯示錯誤訊息之外，最上層還先透過 <code>Kubernetes Node Capacity</code> 的方式進行了第一層的阻擋，理論上 <code>CNI</code> 本身不應該遇到 <code>IP</code> 用光的情形，因為根本不應該有超過數量的 <code>Pod</code> 被嘗試部署到該節點上。</p>
<h1 id="Summary"><a href="#Summary" class="headerlink" title="Summary"></a>Summary</h1><p>本文中我們研究了 <code>AKS</code> 中節點的 <code>CNI</code> 運作流程，包含了簡單的網路設定 (L2 Bridge)，同時也探討一下了 <code>IPAM</code> 的運作邏輯與流程，發現到該 <code>IPAM</code> 與 <code>Azure</code> 的架構本身有強烈的整合，透過 <code>Azure VNET</code> 的設定來控管每個節點上的 <code>Pod</code> 能夠使用的 <code>IP</code> 範圍與數量。</p>
<p>最後我們用一張流程圖來幫本文的 <code>CNI</code> 做一個總結<br><img src="https://i.imgur.com/robQsi1.png" alt="Imgur"></p>
<p>下一篇文章會著重在此基礎上，當上述的 <code>L2 Bridge</code> 與 <code>IPAM</code> 都處理完畢後，這些 <code>Pod</code> 彼此中間是怎麼溝通的，不論是同節點或是跨節點的傳輸，特別是這些節點實際上都是 <code>VM</code> 的情況下，是要如何做到跨節點傳輸的</p>
<h1 id="Reference"><a href="#Reference" class="headerlink" title="Reference"></a>Reference</h1><ul>
<li><a target="_blank" rel="noopener" href="https://docs.microsoft.com/en-us/azure/aks/ssh">Connect with SSH to Azure Kubernetes Service (AKS) cluster nodes</a></li>
<li><a target="_blank" rel="noopener" href="https://github.com/Azure/azure-container-networking/tree/master/cni">Github-Azure-Container-Networking</a></li>
<li><a target="_blank" rel="noopener" href="https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/">reserve-compute-resources</a></li>
</ul>
<h1 id="個人資訊"><a href="#個人資訊" class="headerlink" title="個人資訊"></a>個人資訊</h1><p>我目前於 Hiskio 平台上面有開設 Kubernetes 相關課程，歡迎有興趣的人參考並分享，裡面有我從底層到實戰中對於 Kubernetes 的各種想法</p>
<p>線上課程詳細資訊: <a target="_blank" rel="noopener" href="https://course.hwchiu.com/">https://course.hwchiu.com/</a><br>另外，歡迎按讚加入我個人的粉絲專頁，裡面會定期分享各式各樣的文章，有的是翻譯文章，也有部分是原創文章，主要會聚焦於 CNCF 領域<br><a target="_blank" rel="noopener" href="https://www.facebook.com/technologynoteniu">https://www.facebook.com/technologynoteniu</a></p>
<p>如果有使用 Telegram 的也可以訂閱下列頻道來，裡面我會定期推播通知各類文章<br><a target="_blank" rel="noopener" href="https://t.me/technologynote">https://t.me/technologynote</a></p>
<p>你的捐款將給予我文章成長的動力</p>
<script type="text/javascript" src="https://cdnjs.buymeacoffee.com/1.0.0/button.prod.min.js" data-name="bmc-button" data-slug="hwchiu" data-color="#000000" data-emoji="" data-font="Cookie" data-text="Buy me a coffee" data-outline-color="#fff" data-font-color="#fff" data-coffee-color="#fd0" ></script>
</div>
<footer class="post-footer">
<div class="followme">
<span>Welcome to my other publishing channels</span>
<div class="social-list">
<div class="social-item">
<a target="_blank" class="social-link" href="https://twitter.com/hw_chiu">
<span class="icon">
<i class="fab fa-twitter"></i>
</span>
<span class="label">Twitter</span>
</a>
</div>
<div class="social-item">
<a target="_blank" class="social-link" href="https://t.me/technologynote">
<span class="icon">
<i class="fab fa-telegram"></i>
</span>
<span class="label">Telegram</span>
</a>
</div>
<div class="social-item">
<a target="_blank" class="social-link" href="/atom.xml">
<span class="icon">
<i class="fa fa-rss"></i>
</span>
<span class="label">RSS</span>
</a>
</div>
</div>
</div>
<div class="post-tags">
<a href="/tags/Kubernetes/" rel="tag"># Kubernetes</a>
<a href="/tags/Network/" rel="tag"># Network</a>
<a href="/tags/Azure/" rel="tag"># Azure</a>
<a href="/tags/CNI/" rel="tag"># CNI</a>
</div>
<div class="post-nav">
<div class="post-nav-item">
<a href="/kubernetes-concept.html" rel="prev" title="你到底知不知道什麼是 Kubernetes?">
<i class="fa fa-chevron-left"></i> 你到底知不知道什麼是 Kubernetes?
</a>
</div>
<div class="post-nav-item">
<a href="/circleci.html" rel="next" title="CircleCI 使用經驗談">
CircleCI 使用經驗談 <i class="fa fa-chevron-right"></i>
</a>
</div>
</div>
</footer>
</article>
</div>
<div class="comments utterances-container"></div>
</div>
</main>
<footer class="footer">
<div class="footer-inner">
<div class="copyright">
©
<span itemprop="copyrightYear">2023</span>
<span class="with-love">
<i class="fa fa-heart"></i>
</span>
<span class="author" itemprop="copyrightHolder">Hwchiu</span>
</div>
<div class="busuanzi-count">
<span class="post-meta-item" id="busuanzi_container_site_uv">
<span class="post-meta-item-icon">
<i class="fa fa-user"></i>
</span>
<span class="site-uv" title="Total Visitors">
<span id="busuanzi_value_site_uv"></span>
</span>
</span>
<span class="post-meta-item" id="busuanzi_container_site_pv">
<span class="post-meta-item-icon">
<i class="fa fa-eye"></i>
</span>
<span class="site-pv" title="Total Views">
<span id="busuanzi_value_site_pv"></span>
</span>
</span>
</div>
<div class="powered-by">Powered by <a href="https://hexo.io/" rel="noopener" target="_blank">Hexo</a> & <a href="https://theme-next.js.org/" rel="noopener" target="_blank">NexT.Gemini</a>
</div>
</div>
</footer>
<div class="back-to-top" role="button" aria-label="Back to top">
<i class="fa fa-arrow-up fa-lg"></i>
<span>0%</span>
</div>
<noscript>
<div class="noscript-warning">Theme NexT works best with JavaScript enabled</div>
</noscript>
<script src="https://cdnjs.cloudflare.com/ajax/libs/animejs/3.2.1/anime.min.js" integrity="sha256-XL2inqUJaslATFnHdJOi9GfQ60on8Wx1C2H8DYiN1xY=" crossorigin="anonymous"></script>
<script src="/js/comments.js"></script><script src="/js/utils.js"></script><script src="/js/motion.js"></script><script src="/js/next-boot.js"></script>
<script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
<script class="next-config" data-name="utterances" type="application/json">{"enable":true,"repo":"hwchiu/blog-comment","issue_term":"pathname","theme":"github-light"}</script>
<script src="/js/third-party/comments/utterances.js"></script>
</body>
</html>