From 5fdd859aa389d3e9500e4ccbf14219526539f148 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Wed, 8 May 2024 20:38:57 +0000
Subject: [PATCH] Deployed 1b6a59e0 with MkDocs version: 1.6.0

---
 404.html                                      |   31 +-
 _partials/destroy/index.html                  |   31 +-
 faq/index.html                                |   31 +-
 getting-started/index.html                    |   31 +-
 index.html                                    |   31 +-
 internal/ci/index.html                        |   31 +-
 patterns/agones-game-controller/index.html    |   31 +-
 .../aws-vpc-cni-network-policy/index.html     |   31 +-
 patterns/blue-green-upgrade/index.html        |   31 +-
 patterns/external-secrets/index.html          |   31 +-
 patterns/fargate-serverless/index.html        |   31 +-
 patterns/fully-private-cluster/index.html     |   31 +-
 .../gitops-getting-started-argocd/index.html  |   31 +-
 .../index.html                                |   31 +-
 patterns/ipv6-eks-cluster/index.html          |   31 +-
 patterns/istio/index.html                     |   33 +-
 patterns/karpenter-mng/index.html             | 1712 +++++++++++++++++
 patterns/karpenter/index.html                 |   43 +-
 patterns/kubecost/index.html                  |   31 +-
 patterns/ml-capacity-block/index.html         |   31 +-
 patterns/multi-tenancy-with-teams/index.html  |   31 +-
 patterns/nvidia-gpu-efa/index.html            |   31 +-
 patterns/private-public-ingress/index.html    |   31 +-
 patterns/privatelink-access/index.html        |   31 +-
 patterns/sso-iam-identity-center/index.html   |   31 +-
 patterns/sso-okta/index.html                  |   31 +-
 patterns/stateful/index.html                  |   31 +-
 patterns/targeted-odcr/index.html             |   31 +-
 patterns/tls-with-aws-pca-issuer/index.html   |   31 +-
 .../client-server-communication/index.html    |   31 +-
 patterns/vpc-lattice/index.html               |   31 +-
 patterns/wireguard-with-cilium/index.html     |   31 +-
 search/search_index.json                      |    2 +-
 sitemap.xml                                   |    5 +
 sitemap.xml.gz                                |  Bin 652 -> 658 bytes
 snippets/ipv4-prefix-delegation/index.html    |   31 +-
 snippets/vpc-cni-custom-networking/index.html |   31 +-
 v4-to-v5/addons/index.html                    |   31 +-
 v4-to-v5/cluster/index.html                   |   31 +-
 v4-to-v5/example/index.html                   |   31 +-
 v4-to-v5/motivation/index.html                |   31 +-
 v4-to-v5/teams/index.html                     |   31 +-
 42 files changed, 2713 insertions(+), 198 deletions(-)
 create mode 100644 patterns/karpenter-mng/index.html

diff --git a/404.html b/404.html
index 183e77c5f5..d56d49280c 100644
--- a/404.html
+++ b/404.html
@@ -543,12 +543,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="/terraform-aws-eks-blueprints/patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="/terraform-aws-eks-blueprints/patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -845,10 +866,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -859,8 +880,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/_partials/destroy/index.html b/_partials/destroy/index.html
index 22b97442f2..ed220e1e3d 100644
--- a/_partials/destroy/index.html
+++ b/_partials/destroy/index.html
@@ -549,12 +549,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -851,10 +872,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -865,8 +886,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/faq/index.html b/faq/index.html
index c399aba608..118bc4c29a 100644
--- a/faq/index.html
+++ b/faq/index.html
@@ -552,12 +552,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -854,10 +875,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -868,8 +889,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/getting-started/index.html b/getting-started/index.html
index 587cc14611..1110b01df9 100644
--- a/getting-started/index.html
+++ b/getting-started/index.html
@@ -621,12 +621,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -923,10 +944,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -937,8 +958,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/index.html b/index.html
index ff3253fc91..2854d801ab 100644
--- a/index.html
+++ b/index.html
@@ -661,12 +661,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -963,10 +984,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -977,8 +998,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/internal/ci/index.html b/internal/ci/index.html
index f715bce205..ecda0ed530 100644
--- a/internal/ci/index.html
+++ b/internal/ci/index.html
@@ -554,12 +554,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -856,10 +877,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -870,8 +891,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/agones-game-controller/index.html b/patterns/agones-game-controller/index.html
index db6b64b224..7171bcd7d6 100644
--- a/patterns/agones-game-controller/index.html
+++ b/patterns/agones-game-controller/index.html
@@ -623,12 +623,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/aws-vpc-cni-network-policy/index.html b/patterns/aws-vpc-cni-network-policy/index.html
index 13654b29eb..11314601af 100644
--- a/patterns/aws-vpc-cni-network-policy/index.html
+++ b/patterns/aws-vpc-cni-network-policy/index.html
@@ -632,12 +632,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -934,10 +955,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -948,8 +969,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/blue-green-upgrade/index.html b/patterns/blue-green-upgrade/index.html
index f6a778623a..cdc90673b3 100644
--- a/patterns/blue-green-upgrade/index.html
+++ b/patterns/blue-green-upgrade/index.html
@@ -821,12 +821,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -1123,10 +1144,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1137,8 +1158,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/external-secrets/index.html b/patterns/external-secrets/index.html
index 584a406c8c..47c6e1640e 100644
--- a/patterns/external-secrets/index.html
+++ b/patterns/external-secrets/index.html
@@ -623,12 +623,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/fargate-serverless/index.html b/patterns/fargate-serverless/index.html
index 028e338f85..9758be3c09 100644
--- a/patterns/fargate-serverless/index.html
+++ b/patterns/fargate-serverless/index.html
@@ -638,12 +638,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -940,10 +961,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -954,8 +975,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/fully-private-cluster/index.html b/patterns/fully-private-cluster/index.html
index b518222f89..07d033cc34 100644
--- a/patterns/fully-private-cluster/index.html
+++ b/patterns/fully-private-cluster/index.html
@@ -623,12 +623,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/gitops-getting-started-argocd/index.html b/patterns/gitops-getting-started-argocd/index.html
index dccc0d0d3e..699b17bc6a 100644
--- a/patterns/gitops-getting-started-argocd/index.html
+++ b/patterns/gitops-getting-started-argocd/index.html
@@ -743,12 +743,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -1045,10 +1066,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1059,8 +1080,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/gitops-multi-cluster-hub-spoke-argocd/index.html b/patterns/gitops-multi-cluster-hub-spoke-argocd/index.html
index d7418a4028..8066e41080 100644
--- a/patterns/gitops-multi-cluster-hub-spoke-argocd/index.html
+++ b/patterns/gitops-multi-cluster-hub-spoke-argocd/index.html
@@ -761,12 +761,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -1063,10 +1084,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1077,8 +1098,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/ipv6-eks-cluster/index.html b/patterns/ipv6-eks-cluster/index.html
index a586af98e1..1c7a89f4ce 100644
--- a/patterns/ipv6-eks-cluster/index.html
+++ b/patterns/ipv6-eks-cluster/index.html
@@ -623,12 +623,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/istio/index.html b/patterns/istio/index.html
index 8d9474c4af..b01b68b560 100644
--- a/patterns/istio/index.html
+++ b/patterns/istio/index.html
@@ -16,7 +16,7 @@
         <link rel="prev" href="../ipv6-eks-cluster/">
       
       
-        <link rel="next" href="../karpenter/">
+        <link rel="next" href="../karpenter-mng/">
       
       
       <link rel="icon" href="../../images/colored-logo.png">
@@ -662,12 +662,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -964,10 +985,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -978,8 +999,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/karpenter-mng/index.html b/patterns/karpenter-mng/index.html
new file mode 100644
index 0000000000..ff24295dad
--- /dev/null
+++ b/patterns/karpenter-mng/index.html
@@ -0,0 +1,1712 @@
+
+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+    
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      
+      
+        <meta name="author" content="AWS">
+      
+      
+        <link rel="canonical" href="https://aws-ia.github.io/terraform-aws-eks-blueprints/patterns/karpenter-mng/">
+      
+      
+        <link rel="prev" href="../istio/">
+      
+      
+        <link rel="next" href="../karpenter/">
+      
+      
+      <link rel="icon" href="../../images/colored-logo.png">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.21">
+    
+    
+      
+        <title>Karpenter on EKS MNG - Amazon EKS Blueprints for Terraform</title>
+      
+    
+    
+      <link rel="stylesheet" href="../../assets/stylesheets/main.66ac8b77.min.css">
+      
+        
+        <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
+      
+      
+  
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+    
+    
+  
+  
+  <style>:root{--md-admonition-icon--note:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M1 7.775V2.75C1 1.784 1.784 1 2.75 1h5.025c.464 0 .91.184 1.238.513l6.25 6.25a1.75 1.75 0 0 1 0 2.474l-5.026 5.026a1.75 1.75 0 0 1-2.474 0l-6.25-6.25A1.752 1.752 0 0 1 1 7.775Zm1.5 0c0 .066.026.13.073.177l6.25 6.25a.25.25 0 0 0 .354 0l5.025-5.025a.25.25 0 0 0 0-.354l-6.25-6.25a.25.25 0 0 0-.177-.073H2.75a.25.25 0 0 0-.25.25ZM6 5a1 1 0 1 1 0 2 1 1 0 0 1 0-2Z"/></svg>');--md-admonition-icon--abstract:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M2.5 1.75v11.5c0 .138.112.25.25.25h3.17a.75.75 0 0 1 0 1.5H2.75A1.75 1.75 0 0 1 1 13.25V1.75C1 .784 1.784 0 2.75 0h8.5C12.216 0 13 .784 13 1.75v7.736a.75.75 0 0 1-1.5 0V1.75a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25Zm13.274 9.537v-.001l-4.557 4.45a.75.75 0 0 1-1.055-.008l-1.943-1.95a.75.75 0 0 1 1.062-1.058l1.419 1.425 4.026-3.932a.75.75 0 1 1 1.048 1.074ZM4.75 4h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1 0-1.5ZM4 7.75A.75.75 0 0 1 4.75 7h2a.75.75 0 0 1 0 1.5h-2A.75.75 0 0 1 4 7.75Z"/></svg>');--md-admonition-icon--info:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8Zm8-6.5a6.5 6.5 0 1 0 0 13 6.5 6.5 0 0 0 0-13ZM6.5 7.75A.75.75 0 0 1 7.25 7h1a.75.75 0 0 1 .75.75v2.75h.25a.75.75 0 0 1 0 1.5h-2a.75.75 0 0 1 0-1.5h.25v-2h-.25a.75.75 0 0 1-.75-.75ZM8 6a1 1 0 1 1 0-2 1 1 0 0 1 0 2Z"/></svg>');--md-admonition-icon--tip:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M3.499.75a.75.75 0 0 1 1.5 0v.996C5.9 2.903 6.793 3.65 7.662 4.376l.24.202c-.036-.694.055-1.422.426-2.163C9.1.873 10.794-.045 12.622.26 14.408.558 16 1.94 16 4.25c0 1.278-.954 2.575-2.44 2.734l.146.508.065.22c.203.701.412 1.455.476 2.226.142 1.707-.4 3.03-1.487 3.898C11.714 14.671 10.27 15 8.75 15h-6a.75.75 0 0 1 0-1.5h1.376a4.484 4.484 0 0 1-.563-1.191 3.835 3.835 0 0 1-.05-2.063 4.647 4.647 0 0 1-2.025-.293.75.75 0 0 1 .525-1.406c1.357.507 2.376-.006 2.698-.318l.009-.01a.747.747 0 0 1 1.06 0 .748.748 0 0 1-.012 1.074c-.912.92-.992 1.835-.768 2.586.221.74.745 1.337 1.196 1.621H8.75c1.343 0 2.398-.296 3.074-.836.635-.507 1.036-1.31.928-2.602-.05-.603-.216-1.224-.422-1.93l-.064-.221c-.12-.407-.246-.84-.353-1.29a2.425 2.425 0 0 1-.507-.441 3.075 3.075 0 0 1-.633-1.248.75.75 0 0 1 1.455-.364c.046.185.144.436.31.627.146.168.353.305.712.305.738 0 1.25-.615 1.25-1.25 0-1.47-.95-2.315-2.123-2.51-1.172-.196-2.227.387-2.706 1.345-.46.92-.27 1.774.019 3.062l.042.19a.884.884 0 0 1 .01.05c.348.443.666.949.94 1.553a.75.75 0 1 1-1.365.62c-.553-1.217-1.32-1.94-2.3-2.768L6.7 5.527c-.814-.68-1.75-1.462-2.692-2.619a3.737 3.737 0 0 0-1.023.88c-.406.495-.663 1.036-.722 1.508.116.122.306.21.591.239.388.038.797-.06 1.032-.19a.75.75 0 0 1 .728 1.31c-.515.287-1.23.439-1.906.373-.682-.067-1.473-.38-1.879-1.193L.75 5.677V5.5c0-.984.48-1.94 1.077-2.664.46-.559 1.05-1.055 1.673-1.353V.75Z"/></svg>');--md-admonition-icon--success:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"/></svg>');--md-admonition-icon--question:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8Zm8-6.5a6.5 6.5 0 1 0 0 13 6.5 6.5 0 0 0 0-13ZM6.92 6.085h.001a.749.749 0 1 1-1.342-.67c.169-.339.436-.701.849-.977C6.845 4.16 7.369 4 8 4a2.756 2.756 0 0 1 1.637.525c.503.377.863.965.863 1.725 0 .448-.115.83-.329 1.15-.205.307-.47.513-.692.662-.109.072-.22.138-.313.195l-.006.004a6.24 6.24 0 0 0-.26.16.952.952 0 0 0-.276.245.75.75 0 0 1-1.248-.832c.184-.264.42-.489.692-.661.103-.067.207-.132.313-.195l.007-.004c.1-.061.182-.11.258-.161a.969.969 0 0 0 .277-.245C8.96 6.514 9 6.427 9 6.25a.612.612 0 0 0-.262-.525A1.27 1.27 0 0 0 8 5.5c-.369 0-.595.09-.74.187a1.01 1.01 0 0 0-.34.398ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"/></svg>');--md-admonition-icon--warning:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"/></svg>');--md-admonition-icon--failure:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M2.344 2.343h-.001a8 8 0 0 1 11.314 11.314A8.002 8.002 0 0 1 .234 10.089a8 8 0 0 1 2.11-7.746Zm1.06 10.253a6.5 6.5 0 1 0 9.108-9.275 6.5 6.5 0 0 0-9.108 9.275ZM6.03 4.97 8 6.94l1.97-1.97a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l1.97 1.97a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-1.97 1.97a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L6.94 8 4.97 6.03a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018Z"/></svg>');--md-admonition-icon--danger:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M9.504.43a1.516 1.516 0 0 1 2.437 1.713L10.415 5.5h2.123c1.57 0 2.346 1.909 1.22 3.004l-7.34 7.142a1.249 1.249 0 0 1-.871.354h-.302a1.25 1.25 0 0 1-1.157-1.723L5.633 10.5H3.462c-1.57 0-2.346-1.909-1.22-3.004L9.503.429Zm1.047 1.074L3.286 8.571A.25.25 0 0 0 3.462 9H6.75a.75.75 0 0 1 .694 1.034l-1.713 4.188 6.982-6.793A.25.25 0 0 0 12.538 7H9.25a.75.75 0 0 1-.683-1.06l2.008-4.418.003-.006a.036.036 0 0 0-.004-.009l-.006-.006-.008-.001c-.003 0-.006.002-.009.004Z"/></svg>');--md-admonition-icon--bug:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M4.72.22a.75.75 0 0 1 1.06 0l1 .999a3.488 3.488 0 0 1 2.441 0l.999-1a.748.748 0 0 1 1.265.332.75.75 0 0 1-.205.729l-.775.776c.616.63.995 1.493.995 2.444v.327c0 .1-.009.197-.025.292.408.14.764.392 1.029.722l1.968-.787a.75.75 0 0 1 .556 1.392L13 7.258V9h2.25a.75.75 0 0 1 0 1.5H13v.5c0 .409-.049.806-.141 1.186l2.17.868a.75.75 0 0 1-.557 1.392l-2.184-.873A4.997 4.997 0 0 1 8 16a4.997 4.997 0 0 1-4.288-2.427l-2.183.873a.75.75 0 0 1-.558-1.392l2.17-.868A5.036 5.036 0 0 1 3 11v-.5H.75a.75.75 0 0 1 0-1.5H3V7.258L.971 6.446a.75.75 0 0 1 .558-1.392l1.967.787c.265-.33.62-.583 1.03-.722a1.677 1.677 0 0 1-.026-.292V4.5c0-.951.38-1.814.995-2.444L4.72 1.28a.75.75 0 0 1 0-1.06Zm.53 6.28a.75.75 0 0 0-.75.75V11a3.5 3.5 0 1 0 7 0V7.25a.75.75 0 0 0-.75-.75ZM6.173 5h3.654A.172.172 0 0 0 10 4.827V4.5a2 2 0 1 0-4 0v.327c0 .096.077.173.173.173Z"/></svg>');--md-admonition-icon--example:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M5 5.782V2.5h-.25a.75.75 0 0 1 0-1.5h6.5a.75.75 0 0 1 0 1.5H11v3.282l3.666 5.76C15.619 13.04 14.543 15 12.767 15H3.233c-1.776 0-2.852-1.96-1.899-3.458Zm-2.4 6.565a.75.75 0 0 0 .633 1.153h9.534a.75.75 0 0 0 .633-1.153L12.225 10.5h-8.45ZM9.5 2.5h-3V6c0 .143-.04.283-.117.403L4.73 9h6.54L9.617 6.403A.746.746 0 0 1 9.5 6Z"/></svg>');--md-admonition-icon--quote:url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M1.75 2.5h10.5a.75.75 0 0 1 0 1.5H1.75a.75.75 0 0 1 0-1.5Zm4 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5ZM2.5 7.75v6a.75.75 0 0 1-1.5 0v-6a.75.75 0 0 1 1.5 0Z"/></svg>');}</style>
+
+
+
+    
+    
+      
+    
+    
+      
+        
+        
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=ember:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"ember";--md-code-font:"Roboto Mono"}</style>
+      
+    
+    
+    <script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+    
+      
+
+    
+    
+    
+  </head>
+  
+  
+    
+    
+    
+    
+    
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="orange" data-md-color-accent="orange">
+  
+    
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+      
+        
+        <a href="#karpenter-on-eks-mng" class="md-skip">
+          Skip to content
+        </a>
+      
+    </div>
+    <div data-md-component="announce">
+      
+    </div>
+    
+      <div data-md-color-scheme="default" data-md-component="outdated" hidden>
+        
+      </div>
+    
+    
+      
+
+  
+
+<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="../.." title="Amazon EKS Blueprints for Terraform" class="md-header__button md-logo" aria-label="Amazon EKS Blueprints for Terraform" data-md-component="logo">
+      
+  <img src="../../images/white-logo.png" alt="logo">
+
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            Amazon EKS Blueprints for Terraform
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            
+              Karpenter on EKS MNG
+            
+          </span>
+        </div>
+      </div>
+    </div>
+    
+      
+    
+    
+    
+    
+      <label class="md-header__button md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+      </label>
+      <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
+        </button>
+      </nav>
+      
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+    
+    
+      <div class="md-header__source">
+        <a href="https://github.com/aws-ia/terraform-aws-eks-blueprints" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    terraform-aws-eks-blueprints
+  </div>
+</a>
+      </div>
+    
+  </nav>
+  
+    
+  
+</header>
+    
+    <div class="md-container" data-md-component="container">
+      
+      
+        
+      
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+          
+            
+              
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+
+
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="../.." title="Amazon EKS Blueprints for Terraform" class="md-nav__button md-logo" aria-label="Amazon EKS Blueprints for Terraform" data-md-component="logo">
+      
+  <img src="../../images/white-logo.png" alt="logo">
+
+    </a>
+    Amazon EKS Blueprints for Terraform
+  </label>
+  
+    <div class="md-nav__source">
+      <a href="https://github.com/aws-ia/terraform-aws-eks-blueprints" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    terraform-aws-eks-blueprints
+  </div>
+</a>
+    </div>
+  
+  <ul class="md-nav__list" data-md-scrollfix>
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../.." class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Overview
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../getting-started/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Getting Started
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+    
+  
+  
+  
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" checked>
+        
+          
+          <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Patterns
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_3">
+            <span class="md-nav__icon md-icon"></span>
+            Patterns
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../agones-game-controller/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Agones Game Controller
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../aws-vpc-cni-network-policy/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    AWS VPC CNI Network Policy
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../blue-green-upgrade/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Blue/Green Upgrade
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../external-secrets/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    External Secrets
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../fargate-serverless/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Fargate Serverless
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../fully-private-cluster/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Fully Private Cluster
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../gitops-getting-started-argocd/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    GitOps Getting Started (ArgoCD)
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../gitops-multi-cluster-hub-spoke-argocd/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    GitOps Multi-Cluster Hub-Spoke Topology (ArgoCD)
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ipv6-eks-cluster/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    IPv6 Networking
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../istio/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Istio
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+    
+  
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+      
+      
+        
+      
+      
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+      
+        
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#code" class="md-nav__link">
+    <span class="md-ellipsis">
+      Code
+    </span>
+  </a>
+  
+    <nav class="md-nav" aria-label="Code">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#vpc" class="md-nav__link">
+    <span class="md-ellipsis">
+      VPC
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#ec2nodeclass-and-nodepool" class="md-nav__link">
+    <span class="md-ellipsis">
+      EC2NodeClass and NodePool
+    </span>
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#deploy" class="md-nav__link">
+    <span class="md-ellipsis">
+      Deploy
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#validate" class="md-nav__link">
+    <span class="md-ellipsis">
+      Validate
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#destroy" class="md-nav__link">
+    <span class="md-ellipsis">
+      Destroy
+    </span>
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../karpenter/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS Fargate
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../kubecost/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Kubecost
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ml-capacity-block/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    ML Capacity Block Reservation (CBR)
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../multi-tenancy-with-teams/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Multi-Tenancy w/ Teams
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../nvidia-gpu-efa/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    NVIDIA GPUs with EFA
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../private-public-ingress/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Private and Public Ingress
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../privatelink-access/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    PrivateLink Access
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sso-iam-identity-center/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    SSO - IAM Identity Center
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../sso-okta/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    SSO - Okta
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../stateful/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Stateful
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../targeted-odcr/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Targeted On-Demand Capacity Reservation (ODCR)
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../tls-with-aws-pca-issuer/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TLS w/ AWS PCA Issuer
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../vpc-lattice/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Amazon VPC Lattice
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../wireguard-with-cilium/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Wireguard /w Cilium
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
+        
+          
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Vpc lattice
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
+            <span class="md-nav__icon md-icon"></span>
+            Vpc lattice
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../vpc-lattice/client-server-communication/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Amazon VPC Lattice Client-server Communication
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
+        
+          
+          <label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Snippets
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_4">
+            <span class="md-nav__icon md-icon"></span>
+            Snippets
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../snippets/ipv4-prefix-delegation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    IPv4 Prefix Delegation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../snippets/vpc-cni-custom-networking/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    VPC CNI Custom Networking
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
+        
+          
+          <label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    v4 to v5 Migration
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_5">
+            <span class="md-nav__icon md-icon"></span>
+            v4 to v5 Migration
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../v4-to-v5/motivation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Motivation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../v4-to-v5/cluster/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Cluster
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../v4-to-v5/addons/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Addons
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../v4-to-v5/teams/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Teams
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../faq/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    FAQ
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+            
+              
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#code" class="md-nav__link">
+    <span class="md-ellipsis">
+      Code
+    </span>
+  </a>
+  
+    <nav class="md-nav" aria-label="Code">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#vpc" class="md-nav__link">
+    <span class="md-ellipsis">
+      VPC
+    </span>
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#ec2nodeclass-and-nodepool" class="md-nav__link">
+    <span class="md-ellipsis">
+      EC2NodeClass and NodePool
+    </span>
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#deploy" class="md-nav__link">
+    <span class="md-ellipsis">
+      Deploy
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#validate" class="md-nav__link">
+    <span class="md-ellipsis">
+      Validate
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#destroy" class="md-nav__link">
+    <span class="md-ellipsis">
+      Destroy
+    </span>
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+          
+          
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+                
+                  
+
+  
+  
+
+
+<!-- BEGIN INCLUDE ../../patterns/karpenter-mng/README.md -->
+<h1 id="karpenter-on-eks-mng">Karpenter on EKS MNG<a class="headerlink" href="#karpenter-on-eks-mng" title="Permanent link">&para;</a></h1>
+<p>This pattern demonstrates how to provision Karpenter on an EKS managed node group. Deploying onto standard EC2 instances created by an EKS managed node group will allow for daemonsets to run on the nodes created for the Karpenter controller, and therefore better unification of tooling across your data plane. This solution is comprised of the following components:</p>
+<ol>
+<li>An EKS managed node group that applies both a taint as well as a label for the Karpenter controller. We want the Karpenter controller to target these nodes via a <code>nodeSelector</code> in order to avoid the controller pods from running on nodes that Karpenter itself creates and manages. In addition, we are applying a taint to keep other pods off of these nodes as they are primarily intended for the controller pods. We apply a toleration to the CoreDNS addon, to allow those pods to run on the controller nodes as well. This is needed so that when a cluster is created, the CoreDNS pods have a place to run in order for the Karpenter controller to be provisioned and start managing the additional compute requirements for the cluster. Without letting CoreDNS run on these nodes, the controllers would fail to deploy and the data plane would be in a "deadlock" waiting for resources to deploy but unable to do so.</li>
+<li>The <code>eks-pod-identity-agent</code> addon has been provisioned to allow the Karpenter controller to utilize EKS Pod Identity for AWS permissions via an IAM role.</li>
+<li>The VPC subnets and node security group have been tagged with <code>"karpenter.sh/discovery" = local.name</code> for discoverability by the controller. The controller will discover these resources and use them to provision EC2 resources for the cluster.</li>
+<li>An IAM role for the Karpenter controller has been created with a trust policy that trusts the EKS Pod Identity service principal. This allows the EKS Pod Identity service to provide AWS credentials to the Karpenter controller pods in order to call AWS APIs.</li>
+<li>An IAM role for the nodes that Karpenter will create has been created along with a cluster access entry which allows the nodes to acquire permissions to join the cluster. Karpenter will create and manage the instance profile that utilizes this IAM role.</li>
+<li>An SQS queue has been created that is subscribed to certain EC2 CloudWatch events. This queue is used by Karpenter, allowing it to respond to certain EC2 lifecycle events and gracefully migrate pods off the instance before it is terminated.</li>
+</ol>
+<h2 id="code">Code<a class="headerlink" href="#code" title="Permanent link">&para;</a></h2>
+<p>The areas of significance related to this pattern are highlighted in the code provided below:</p>
+<div class="language-terraform highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="c1">################################################################################</span>
+</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="c1"># Cluster</span>
+</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="c1">################################################################################</span>
+</span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a>
+</span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a><span class="kr">module</span><span class="w"> </span><span class="nv">&quot;eks&quot;</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-6"><a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="w">  </span><span class="na">source</span><span class="w">  </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;terraform-aws-modules/eks/aws&quot;</span>
+</span><span id="__span-0-7"><a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="w">  </span><span class="na">version</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;~&gt; 20.9&quot;</span>
+</span><span id="__span-0-8"><a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a>
+</span><span id="__span-0-9"><a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a><span class="w">  </span><span class="na">cluster_name</span><span class="w">    </span><span class="o">=</span><span class="w"> </span><span class="nv">local.name</span>
+</span><span id="__span-0-10"><a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a><span class="w">  </span><span class="na">cluster_version</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;1.29&quot;</span>
+</span><span id="__span-0-11"><a id="__codelineno-0-11" name="__codelineno-0-11" href="#__codelineno-0-11"></a>
+</span><span id="__span-0-12"><a id="__codelineno-0-12" name="__codelineno-0-12" href="#__codelineno-0-12"></a><span class="c1">  # Give the Terraform identity admin access to the cluster</span>
+</span><span id="__span-0-13"><a id="__codelineno-0-13" name="__codelineno-0-13" href="#__codelineno-0-13"></a><span class="c1">  # which will allow it to deploy resources into the cluster</span>
+</span><span id="__span-0-14"><a id="__codelineno-0-14" name="__codelineno-0-14" href="#__codelineno-0-14"></a><span class="w">  </span><span class="na">enable_cluster_creator_admin_permissions</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="no">true</span>
+</span><span id="__span-0-15"><a id="__codelineno-0-15" name="__codelineno-0-15" href="#__codelineno-0-15"></a><span class="w">  </span><span class="na">cluster_endpoint_public_access</span><span class="w">           </span><span class="o">=</span><span class="w"> </span><span class="no">true</span>
+</span><span id="__span-0-16"><a id="__codelineno-0-16" name="__codelineno-0-16" href="#__codelineno-0-16"></a>
+</span><span id="__span-0-17"><a id="__codelineno-0-17" name="__codelineno-0-17" href="#__codelineno-0-17"></a><span class="w">  </span><span class="nb">cluster_addons</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-18"><a id="__codelineno-0-18" name="__codelineno-0-18" href="#__codelineno-0-18"></a><span class="w">    </span><span class="nb">coredns</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-19"><a id="__codelineno-0-19" name="__codelineno-0-19" href="#__codelineno-0-19"></a><span class="w">      </span><span class="na">configuration_values</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nf">jsonencode</span><span class="p">({</span>
+</span><span id="__span-0-20"><a id="__codelineno-0-20" name="__codelineno-0-20" href="#__codelineno-0-20"></a><span class="hll"><span class="w">        </span><span class="na">tolerations</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">[</span>
+</span></span><span id="__span-0-21"><a id="__codelineno-0-21" name="__codelineno-0-21" href="#__codelineno-0-21"></a><span class="hll"><span class="c1">          # Allow CoreDNS to run on the same nodes as the Karpenter controller</span>
+</span></span><span id="__span-0-22"><a id="__codelineno-0-22" name="__codelineno-0-22" href="#__codelineno-0-22"></a><span class="hll"><span class="c1">          # for use during cluster creation when Karpenter nodes do not yet exist</span>
+</span></span><span id="__span-0-23"><a id="__codelineno-0-23" name="__codelineno-0-23" href="#__codelineno-0-23"></a><span class="hll"><span class="w">          </span><span class="p">{</span>
+</span></span><span id="__span-0-24"><a id="__codelineno-0-24" name="__codelineno-0-24" href="#__codelineno-0-24"></a><span class="hll"><span class="w">            </span><span class="na">key</span><span class="w">    </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;karpenter.sh/controller&quot;</span>
+</span></span><span id="__span-0-25"><a id="__codelineno-0-25" name="__codelineno-0-25" href="#__codelineno-0-25"></a><span class="hll"><span class="w">            </span><span class="na">value</span><span class="w">  </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;true&quot;</span>
+</span></span><span id="__span-0-26"><a id="__codelineno-0-26" name="__codelineno-0-26" href="#__codelineno-0-26"></a><span class="hll"><span class="w">            </span><span class="na">effect</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;NoSchedule&quot;</span>
+</span></span><span id="__span-0-27"><a id="__codelineno-0-27" name="__codelineno-0-27" href="#__codelineno-0-27"></a><span class="hll"><span class="w">          </span><span class="p">}</span>
+</span></span><span id="__span-0-28"><a id="__codelineno-0-28" name="__codelineno-0-28" href="#__codelineno-0-28"></a><span class="hll"><span class="w">        </span><span class="p">]</span>
+</span></span><span id="__span-0-29"><a id="__codelineno-0-29" name="__codelineno-0-29" href="#__codelineno-0-29"></a><span class="w">      </span><span class="p">})</span>
+</span><span id="__span-0-30"><a id="__codelineno-0-30" name="__codelineno-0-30" href="#__codelineno-0-30"></a><span class="w">    </span><span class="p">}</span>
+</span><span id="__span-0-31"><a id="__codelineno-0-31" name="__codelineno-0-31" href="#__codelineno-0-31"></a><span class="hll"><span class="w">    </span><span class="nb">eks-pod-identity-agent</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{}</span>
+</span></span><span id="__span-0-32"><a id="__codelineno-0-32" name="__codelineno-0-32" href="#__codelineno-0-32"></a><span class="w">    </span><span class="nb">kube-proxy</span><span class="w">             </span><span class="o">=</span><span class="w"> </span><span class="p">{}</span>
+</span><span id="__span-0-33"><a id="__codelineno-0-33" name="__codelineno-0-33" href="#__codelineno-0-33"></a><span class="w">    </span><span class="nb">vpc-cni</span><span class="w">                </span><span class="o">=</span><span class="w"> </span><span class="p">{}</span>
+</span><span id="__span-0-34"><a id="__codelineno-0-34" name="__codelineno-0-34" href="#__codelineno-0-34"></a><span class="w">  </span><span class="p">}</span>
+</span><span id="__span-0-35"><a id="__codelineno-0-35" name="__codelineno-0-35" href="#__codelineno-0-35"></a>
+</span><span id="__span-0-36"><a id="__codelineno-0-36" name="__codelineno-0-36" href="#__codelineno-0-36"></a><span class="w">  </span><span class="na">enable_efa_support</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="no">true</span>
+</span><span id="__span-0-37"><a id="__codelineno-0-37" name="__codelineno-0-37" href="#__codelineno-0-37"></a>
+</span><span id="__span-0-38"><a id="__codelineno-0-38" name="__codelineno-0-38" href="#__codelineno-0-38"></a><span class="w">  </span><span class="na">vpc_id</span><span class="w">     </span><span class="o">=</span><span class="w"> </span><span class="nv">module.vpc.vpc_id</span>
+</span><span id="__span-0-39"><a id="__codelineno-0-39" name="__codelineno-0-39" href="#__codelineno-0-39"></a><span class="w">  </span><span class="na">subnet_ids</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">module.vpc.private_subnets</span>
+</span><span id="__span-0-40"><a id="__codelineno-0-40" name="__codelineno-0-40" href="#__codelineno-0-40"></a>
+</span><span id="__span-0-41"><a id="__codelineno-0-41" name="__codelineno-0-41" href="#__codelineno-0-41"></a><span class="w">  </span><span class="nb">eks_managed_node_groups</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-42"><a id="__codelineno-0-42" name="__codelineno-0-42" href="#__codelineno-0-42"></a><span class="w">    </span><span class="nb">karpenter</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-43"><a id="__codelineno-0-43" name="__codelineno-0-43" href="#__codelineno-0-43"></a><span class="w">      </span><span class="na">instance_types</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">[</span><span class="s2">&quot;m5.large&quot;</span><span class="p">]</span>
+</span><span id="__span-0-44"><a id="__codelineno-0-44" name="__codelineno-0-44" href="#__codelineno-0-44"></a>
+</span><span id="__span-0-45"><a id="__codelineno-0-45" name="__codelineno-0-45" href="#__codelineno-0-45"></a><span class="w">      </span><span class="na">min_size</span><span class="w">     </span><span class="o">=</span><span class="w"> </span><span class="m">2</span>
+</span><span id="__span-0-46"><a id="__codelineno-0-46" name="__codelineno-0-46" href="#__codelineno-0-46"></a><span class="w">      </span><span class="na">max_size</span><span class="w">     </span><span class="o">=</span><span class="w"> </span><span class="m">3</span>
+</span><span id="__span-0-47"><a id="__codelineno-0-47" name="__codelineno-0-47" href="#__codelineno-0-47"></a><span class="w">      </span><span class="na">desired_size</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="m">2</span>
+</span><span id="__span-0-48"><a id="__codelineno-0-48" name="__codelineno-0-48" href="#__codelineno-0-48"></a>
+</span><span id="__span-0-49"><a id="__codelineno-0-49" name="__codelineno-0-49" href="#__codelineno-0-49"></a><span class="hll"><span class="w">      </span><span class="nb">labels</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+</span></span><span id="__span-0-50"><a id="__codelineno-0-50" name="__codelineno-0-50" href="#__codelineno-0-50"></a><span class="hll"><span class="c1">        # Used to ensure Karpenter runs on nodes that it does not manage</span>
+</span></span><span id="__span-0-51"><a id="__codelineno-0-51" name="__codelineno-0-51" href="#__codelineno-0-51"></a><span class="hll"><span class="w">        </span><span class="s2">&quot;karpenter.sh/controller&quot;</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;true&quot;</span>
+</span></span><span id="__span-0-52"><a id="__codelineno-0-52" name="__codelineno-0-52" href="#__codelineno-0-52"></a><span class="hll"><span class="w">      </span><span class="p">}</span>
+</span></span><span id="__span-0-53"><a id="__codelineno-0-53" name="__codelineno-0-53" href="#__codelineno-0-53"></a><span class="hll">
+</span></span><span id="__span-0-54"><a id="__codelineno-0-54" name="__codelineno-0-54" href="#__codelineno-0-54"></a><span class="hll"><span class="w">      </span><span class="nb">taints</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+</span></span><span id="__span-0-55"><a id="__codelineno-0-55" name="__codelineno-0-55" href="#__codelineno-0-55"></a><span class="hll"><span class="c1">        # The pods that do not tolerate this taint should run on nodes</span>
+</span></span><span id="__span-0-56"><a id="__codelineno-0-56" name="__codelineno-0-56" href="#__codelineno-0-56"></a><span class="hll"><span class="c1">        # created by Karpenter</span>
+</span></span><span id="__span-0-57"><a id="__codelineno-0-57" name="__codelineno-0-57" href="#__codelineno-0-57"></a><span class="hll"><span class="w">        </span><span class="nb">karpenter</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+</span></span><span id="__span-0-58"><a id="__codelineno-0-58" name="__codelineno-0-58" href="#__codelineno-0-58"></a><span class="hll"><span class="w">          </span><span class="na">key</span><span class="w">    </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;karpenter.sh/controller&quot;</span>
+</span></span><span id="__span-0-59"><a id="__codelineno-0-59" name="__codelineno-0-59" href="#__codelineno-0-59"></a><span class="hll"><span class="w">          </span><span class="na">value</span><span class="w">  </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;true&quot;</span>
+</span></span><span id="__span-0-60"><a id="__codelineno-0-60" name="__codelineno-0-60" href="#__codelineno-0-60"></a><span class="hll"><span class="w">          </span><span class="na">effect</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;NO_SCHEDULE&quot;</span>
+</span></span><span id="__span-0-61"><a id="__codelineno-0-61" name="__codelineno-0-61" href="#__codelineno-0-61"></a><span class="hll"><span class="w">        </span><span class="p">}</span>
+</span></span><span id="__span-0-62"><a id="__codelineno-0-62" name="__codelineno-0-62" href="#__codelineno-0-62"></a><span class="hll"><span class="w">      </span><span class="p">}</span>
+</span></span><span id="__span-0-63"><a id="__codelineno-0-63" name="__codelineno-0-63" href="#__codelineno-0-63"></a><span class="w">    </span><span class="p">}</span>
+</span><span id="__span-0-64"><a id="__codelineno-0-64" name="__codelineno-0-64" href="#__codelineno-0-64"></a><span class="w">  </span><span class="p">}</span>
+</span><span id="__span-0-65"><a id="__codelineno-0-65" name="__codelineno-0-65" href="#__codelineno-0-65"></a>
+</span><span id="__span-0-66"><a id="__codelineno-0-66" name="__codelineno-0-66" href="#__codelineno-0-66"></a><span class="w">  </span><span class="na">tags</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nf">merge</span><span class="p">(</span><span class="nv">local.tags</span><span class="p">,</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-67"><a id="__codelineno-0-67" name="__codelineno-0-67" href="#__codelineno-0-67"></a><span class="hll"><span class="c1">    # NOTE - if creating multiple security groups with this module, only tag the</span>
+</span></span><span id="__span-0-68"><a id="__codelineno-0-68" name="__codelineno-0-68" href="#__codelineno-0-68"></a><span class="hll"><span class="c1">    # security group that Karpenter should utilize with the following tag</span>
+</span></span><span id="__span-0-69"><a id="__codelineno-0-69" name="__codelineno-0-69" href="#__codelineno-0-69"></a><span class="hll"><span class="c1">    # (i.e. - at most, only one security group should have this tag in your account)</span>
+</span></span><span id="__span-0-70"><a id="__codelineno-0-70" name="__codelineno-0-70" href="#__codelineno-0-70"></a><span class="hll"><span class="w">    </span><span class="s2">&quot;karpenter.sh/discovery&quot;</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">local.name</span>
+</span></span><span id="__span-0-71"><a id="__codelineno-0-71" name="__codelineno-0-71" href="#__codelineno-0-71"></a><span class="w">  </span><span class="p">})</span>
+</span><span id="__span-0-72"><a id="__codelineno-0-72" name="__codelineno-0-72" href="#__codelineno-0-72"></a><span class="p">}</span>
+</span><span id="__span-0-73"><a id="__codelineno-0-73" name="__codelineno-0-73" href="#__codelineno-0-73"></a>
+</span><span id="__span-0-74"><a id="__codelineno-0-74" name="__codelineno-0-74" href="#__codelineno-0-74"></a><span class="kr">output</span><span class="w"> </span><span class="nv">&quot;configure_kubectl&quot;</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-75"><a id="__codelineno-0-75" name="__codelineno-0-75" href="#__codelineno-0-75"></a><span class="w">  </span><span class="na">description</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;Configure kubectl: make sure you&#39;re logged in with the correct AWS profile and run the following command to update your kubeconfig&quot;</span>
+</span><span id="__span-0-76"><a id="__codelineno-0-76" name="__codelineno-0-76" href="#__codelineno-0-76"></a><span class="w">  </span><span class="na">value</span><span class="w">       </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}&quot;</span>
+</span><span id="__span-0-77"><a id="__codelineno-0-77" name="__codelineno-0-77" href="#__codelineno-0-77"></a><span class="p">}</span>
+</span><span id="__span-0-78"><a id="__codelineno-0-78" name="__codelineno-0-78" href="#__codelineno-0-78"></a>
+</span><span id="__span-0-79"><a id="__codelineno-0-79" name="__codelineno-0-79" href="#__codelineno-0-79"></a><span class="c1">################################################################################</span>
+</span><span id="__span-0-80"><a id="__codelineno-0-80" name="__codelineno-0-80" href="#__codelineno-0-80"></a><span class="c1"># Controller &amp; Node IAM roles, SQS Queue, Eventbridge Rules</span>
+</span><span id="__span-0-81"><a id="__codelineno-0-81" name="__codelineno-0-81" href="#__codelineno-0-81"></a><span class="c1">################################################################################</span>
+</span><span id="__span-0-82"><a id="__codelineno-0-82" name="__codelineno-0-82" href="#__codelineno-0-82"></a>
+</span><span id="__span-0-83"><a id="__codelineno-0-83" name="__codelineno-0-83" href="#__codelineno-0-83"></a><span class="kr">module</span><span class="w"> </span><span class="nv">&quot;karpenter&quot;</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-84"><a id="__codelineno-0-84" name="__codelineno-0-84" href="#__codelineno-0-84"></a><span class="w">  </span><span class="na">source</span><span class="w">  </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;terraform-aws-modules/eks/aws//modules/karpenter&quot;</span>
+</span><span id="__span-0-85"><a id="__codelineno-0-85" name="__codelineno-0-85" href="#__codelineno-0-85"></a><span class="w">  </span><span class="na">version</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;~&gt; 20.9&quot;</span>
+</span><span id="__span-0-86"><a id="__codelineno-0-86" name="__codelineno-0-86" href="#__codelineno-0-86"></a>
+</span><span id="__span-0-87"><a id="__codelineno-0-87" name="__codelineno-0-87" href="#__codelineno-0-87"></a><span class="w">  </span><span class="na">cluster_name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">module.eks.cluster_name</span>
+</span><span id="__span-0-88"><a id="__codelineno-0-88" name="__codelineno-0-88" href="#__codelineno-0-88"></a>
+</span><span id="__span-0-89"><a id="__codelineno-0-89" name="__codelineno-0-89" href="#__codelineno-0-89"></a><span class="hll"><span class="c1">  # Name needs to match role name passed to the EC2NodeClass</span>
+</span></span><span id="__span-0-90"><a id="__codelineno-0-90" name="__codelineno-0-90" href="#__codelineno-0-90"></a><span class="hll"><span class="w">  </span><span class="na">node_iam_role_use_name_prefix</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="no">false</span>
+</span></span><span id="__span-0-91"><a id="__codelineno-0-91" name="__codelineno-0-91" href="#__codelineno-0-91"></a><span class="hll"><span class="w">  </span><span class="na">node_iam_role_name</span><span class="w">            </span><span class="o">=</span><span class="w"> </span><span class="nv">local.name</span>
+</span></span><span id="__span-0-92"><a id="__codelineno-0-92" name="__codelineno-0-92" href="#__codelineno-0-92"></a>
+</span><span id="__span-0-93"><a id="__codelineno-0-93" name="__codelineno-0-93" href="#__codelineno-0-93"></a><span class="w">  </span><span class="na">tags</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">local.tags</span>
+</span><span id="__span-0-94"><a id="__codelineno-0-94" name="__codelineno-0-94" href="#__codelineno-0-94"></a><span class="p">}</span>
+</span><span id="__span-0-95"><a id="__codelineno-0-95" name="__codelineno-0-95" href="#__codelineno-0-95"></a>
+</span><span id="__span-0-96"><a id="__codelineno-0-96" name="__codelineno-0-96" href="#__codelineno-0-96"></a><span class="kr">resource</span><span class="w"> </span><span class="nc">&quot;aws_eks_pod_identity_association&quot;</span><span class="w"> </span><span class="nv">&quot;karpenter&quot;</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-97"><a id="__codelineno-0-97" name="__codelineno-0-97" href="#__codelineno-0-97"></a><span class="hll"><span class="w">  </span><span class="na">cluster_name</span><span class="w">    </span><span class="o">=</span><span class="w"> </span><span class="nv">module.eks.cluster_name</span>
+</span></span><span id="__span-0-98"><a id="__codelineno-0-98" name="__codelineno-0-98" href="#__codelineno-0-98"></a><span class="hll"><span class="w">  </span><span class="na">namespace</span><span class="w">       </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;kube-system&quot;</span>
+</span></span><span id="__span-0-99"><a id="__codelineno-0-99" name="__codelineno-0-99" href="#__codelineno-0-99"></a><span class="hll"><span class="w">  </span><span class="na">service_account</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;karpenter&quot;</span>
+</span></span><span id="__span-0-100"><a id="__codelineno-0-100" name="__codelineno-0-100" href="#__codelineno-0-100"></a><span class="hll"><span class="w">  </span><span class="na">role_arn</span><span class="w">        </span><span class="o">=</span><span class="w"> </span><span class="nv">module.karpenter.iam_role_arn</span>
+</span></span><span id="__span-0-101"><a id="__codelineno-0-101" name="__codelineno-0-101" href="#__codelineno-0-101"></a><span class="p">}</span>
+</span><span id="__span-0-102"><a id="__codelineno-0-102" name="__codelineno-0-102" href="#__codelineno-0-102"></a>
+</span><span id="__span-0-103"><a id="__codelineno-0-103" name="__codelineno-0-103" href="#__codelineno-0-103"></a><span class="c1">################################################################################</span>
+</span><span id="__span-0-104"><a id="__codelineno-0-104" name="__codelineno-0-104" href="#__codelineno-0-104"></a><span class="c1"># Helm charts</span>
+</span><span id="__span-0-105"><a id="__codelineno-0-105" name="__codelineno-0-105" href="#__codelineno-0-105"></a><span class="c1">################################################################################</span>
+</span><span id="__span-0-106"><a id="__codelineno-0-106" name="__codelineno-0-106" href="#__codelineno-0-106"></a>
+</span><span id="__span-0-107"><a id="__codelineno-0-107" name="__codelineno-0-107" href="#__codelineno-0-107"></a><span class="kr">resource</span><span class="w"> </span><span class="nc">&quot;helm_release&quot;</span><span class="w"> </span><span class="nv">&quot;karpenter&quot;</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-108"><a id="__codelineno-0-108" name="__codelineno-0-108" href="#__codelineno-0-108"></a><span class="hll"><span class="w">  </span><span class="na">namespace</span><span class="w">           </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;kube-system&quot;</span>
+</span></span><span id="__span-0-109"><a id="__codelineno-0-109" name="__codelineno-0-109" href="#__codelineno-0-109"></a><span class="hll"><span class="w">  </span><span class="na">name</span><span class="w">                </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;karpenter&quot;</span>
+</span></span><span id="__span-0-110"><a id="__codelineno-0-110" name="__codelineno-0-110" href="#__codelineno-0-110"></a><span class="hll"><span class="w">  </span><span class="na">repository</span><span class="w">          </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;oci://public.ecr.aws/karpenter&quot;</span>
+</span></span><span id="__span-0-111"><a id="__codelineno-0-111" name="__codelineno-0-111" href="#__codelineno-0-111"></a><span class="hll"><span class="w">  </span><span class="na">repository_username</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">data.aws_ecrpublic_authorization_token.token.user_name</span>
+</span></span><span id="__span-0-112"><a id="__codelineno-0-112" name="__codelineno-0-112" href="#__codelineno-0-112"></a><span class="hll"><span class="w">  </span><span class="na">repository_password</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">data.aws_ecrpublic_authorization_token.token.password</span>
+</span></span><span id="__span-0-113"><a id="__codelineno-0-113" name="__codelineno-0-113" href="#__codelineno-0-113"></a><span class="hll"><span class="w">  </span><span class="na">chart</span><span class="w">               </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;karpenter&quot;</span>
+</span></span><span id="__span-0-114"><a id="__codelineno-0-114" name="__codelineno-0-114" href="#__codelineno-0-114"></a><span class="hll"><span class="w">  </span><span class="na">version</span><span class="w">             </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;0.36.1&quot;</span>
+</span></span><span id="__span-0-115"><a id="__codelineno-0-115" name="__codelineno-0-115" href="#__codelineno-0-115"></a><span class="hll"><span class="w">  </span><span class="na">wait</span><span class="w">                </span><span class="o">=</span><span class="w"> </span><span class="no">false</span>
+</span></span><span id="__span-0-116"><a id="__codelineno-0-116" name="__codelineno-0-116" href="#__codelineno-0-116"></a><span class="hll">
+</span></span><span id="__span-0-117"><a id="__codelineno-0-117" name="__codelineno-0-117" href="#__codelineno-0-117"></a><span class="hll"><span class="w">  </span><span class="na">values</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">[</span>
+</span></span><span id="__span-0-118"><a id="__codelineno-0-118" name="__codelineno-0-118" href="#__codelineno-0-118"></a><span class="hll"><span class="w">    </span><span class="o">&lt;&lt;-</span><span class="dl">EOT</span>
+</span></span><span id="__span-0-119"><a id="__codelineno-0-119" name="__codelineno-0-119" href="#__codelineno-0-119"></a><span class="hll"><span class="sh">    nodeSelector:</span>
+</span></span><span id="__span-0-120"><a id="__codelineno-0-120" name="__codelineno-0-120" href="#__codelineno-0-120"></a><span class="hll"><span class="sh">      karpenter.sh/controller: &#39;true&#39;</span>
+</span></span><span id="__span-0-121"><a id="__codelineno-0-121" name="__codelineno-0-121" href="#__codelineno-0-121"></a><span class="hll"><span class="sh">    tolerations:</span>
+</span></span><span id="__span-0-122"><a id="__codelineno-0-122" name="__codelineno-0-122" href="#__codelineno-0-122"></a><span class="hll"><span class="sh">      - key: CriticalAddonsOnly</span>
+</span></span><span id="__span-0-123"><a id="__codelineno-0-123" name="__codelineno-0-123" href="#__codelineno-0-123"></a><span class="hll"><span class="sh">        operator: Exists</span>
+</span></span><span id="__span-0-124"><a id="__codelineno-0-124" name="__codelineno-0-124" href="#__codelineno-0-124"></a><span class="hll"><span class="sh">      - key: karpenter.sh/controller</span>
+</span></span><span id="__span-0-125"><a id="__codelineno-0-125" name="__codelineno-0-125" href="#__codelineno-0-125"></a><span class="hll"><span class="sh">        operator: Exists</span>
+</span></span><span id="__span-0-126"><a id="__codelineno-0-126" name="__codelineno-0-126" href="#__codelineno-0-126"></a><span class="hll"><span class="sh">        effect: NoSchedule</span>
+</span></span><span id="__span-0-127"><a id="__codelineno-0-127" name="__codelineno-0-127" href="#__codelineno-0-127"></a><span class="hll"><span class="sh">    settings:</span>
+</span></span><span id="__span-0-128"><a id="__codelineno-0-128" name="__codelineno-0-128" href="#__codelineno-0-128"></a><span class="hll"><span class="sh">      clusterName: ${module.eks.cluster_name}</span>
+</span></span><span id="__span-0-129"><a id="__codelineno-0-129" name="__codelineno-0-129" href="#__codelineno-0-129"></a><span class="hll"><span class="sh">      clusterEndpoint: ${module.eks.cluster_endpoint}</span>
+</span></span><span id="__span-0-130"><a id="__codelineno-0-130" name="__codelineno-0-130" href="#__codelineno-0-130"></a><span class="hll"><span class="sh">      interruptionQueue: ${module.karpenter.queue_name}</span>
+</span></span><span id="__span-0-131"><a id="__codelineno-0-131" name="__codelineno-0-131" href="#__codelineno-0-131"></a><span class="hll"><span class="dl">    EOT</span>
+</span></span><span id="__span-0-132"><a id="__codelineno-0-132" name="__codelineno-0-132" href="#__codelineno-0-132"></a><span class="hll"><span class="w">  </span><span class="p">]</span>
+</span></span><span id="__span-0-133"><a id="__codelineno-0-133" name="__codelineno-0-133" href="#__codelineno-0-133"></a>
+</span><span id="__span-0-134"><a id="__codelineno-0-134" name="__codelineno-0-134" href="#__codelineno-0-134"></a><span class="w">  </span><span class="nb">lifecycle</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-0-135"><a id="__codelineno-0-135" name="__codelineno-0-135" href="#__codelineno-0-135"></a><span class="w">    </span><span class="na">ignore_changes</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">[</span>
+</span><span id="__span-0-136"><a id="__codelineno-0-136" name="__codelineno-0-136" href="#__codelineno-0-136"></a><span class="nb">      repository_password</span>
+</span><span id="__span-0-137"><a id="__codelineno-0-137" name="__codelineno-0-137" href="#__codelineno-0-137"></a><span class="nb">    </span><span class="p">]</span>
+</span><span id="__span-0-138"><a id="__codelineno-0-138" name="__codelineno-0-138" href="#__codelineno-0-138"></a><span class="w">  </span><span class="p">}</span>
+</span><span id="__span-0-139"><a id="__codelineno-0-139" name="__codelineno-0-139" href="#__codelineno-0-139"></a><span class="p">}</span>
+</span></code></pre></div>
+<h3 id="vpc">VPC<a class="headerlink" href="#vpc" title="Permanent link">&para;</a></h3>
+<div class="language-terraform highlight"><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="kr">module</span><span class="w"> </span><span class="nv">&quot;vpc&quot;</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="w">  </span><span class="na">source</span><span class="w">  </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;terraform-aws-modules/vpc/aws&quot;</span>
+</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="w">  </span><span class="na">version</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;~&gt; 5.0&quot;</span>
+</span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a>
+</span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="w">  </span><span class="na">name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">local.name</span>
+</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a><span class="w">  </span><span class="na">cidr</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">local.vpc_cidr</span>
+</span><span id="__span-1-7"><a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a>
+</span><span id="__span-1-8"><a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a><span class="w">  </span><span class="na">azs</span><span class="w">             </span><span class="o">=</span><span class="w"> </span><span class="nv">local.azs</span>
+</span><span id="__span-1-9"><a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a><span class="w">  </span><span class="na">private_subnets</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">[</span><span class="err">for</span><span class="w"> </span><span class="err">k</span><span class="p">,</span><span class="w"> </span><span class="err">v</span><span class="w"> </span><span class="err">in</span><span class="w"> </span><span class="nv">local.azs</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="nf">cidrsubnet</span><span class="p">(</span><span class="nv">local.vpc_cidr</span><span class="p">,</span><span class="w"> </span><span class="m">4</span><span class="p">,</span><span class="w"> </span><span class="err">k</span><span class="p">)]</span>
+</span><span id="__span-1-10"><a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a><span class="w">  </span><span class="na">public_subnets</span><span class="w">  </span><span class="o">=</span><span class="w"> </span><span class="p">[</span><span class="err">for</span><span class="w"> </span><span class="err">k</span><span class="p">,</span><span class="w"> </span><span class="err">v</span><span class="w"> </span><span class="err">in</span><span class="w"> </span><span class="nv">local.azs</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="nf">cidrsubnet</span><span class="p">(</span><span class="nv">local.vpc_cidr</span><span class="p">,</span><span class="w"> </span><span class="m">8</span><span class="p">,</span><span class="w"> </span><span class="err">k</span><span class="w"> </span><span class="err">+</span><span class="w"> </span><span class="m">48</span><span class="p">)]</span>
+</span><span id="__span-1-11"><a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a>
+</span><span id="__span-1-12"><a id="__codelineno-1-12" name="__codelineno-1-12" href="#__codelineno-1-12"></a><span class="w">  </span><span class="na">enable_nat_gateway</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="no">true</span>
+</span><span id="__span-1-13"><a id="__codelineno-1-13" name="__codelineno-1-13" href="#__codelineno-1-13"></a><span class="w">  </span><span class="na">single_nat_gateway</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="no">true</span>
+</span><span id="__span-1-14"><a id="__codelineno-1-14" name="__codelineno-1-14" href="#__codelineno-1-14"></a>
+</span><span id="__span-1-15"><a id="__codelineno-1-15" name="__codelineno-1-15" href="#__codelineno-1-15"></a><span class="w">  </span><span class="nb">public_subnet_tags</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-1-16"><a id="__codelineno-1-16" name="__codelineno-1-16" href="#__codelineno-1-16"></a><span class="w">    </span><span class="s2">&quot;kubernetes.io/role/elb&quot;</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="m">1</span>
+</span><span id="__span-1-17"><a id="__codelineno-1-17" name="__codelineno-1-17" href="#__codelineno-1-17"></a><span class="w">  </span><span class="p">}</span>
+</span><span id="__span-1-18"><a id="__codelineno-1-18" name="__codelineno-1-18" href="#__codelineno-1-18"></a>
+</span><span id="__span-1-19"><a id="__codelineno-1-19" name="__codelineno-1-19" href="#__codelineno-1-19"></a><span class="w">  </span><span class="nb">private_subnet_tags</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
+</span><span id="__span-1-20"><a id="__codelineno-1-20" name="__codelineno-1-20" href="#__codelineno-1-20"></a><span class="w">    </span><span class="s2">&quot;kubernetes.io/role/internal-elb&quot;</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="m">1</span>
+</span><span id="__span-1-21"><a id="__codelineno-1-21" name="__codelineno-1-21" href="#__codelineno-1-21"></a><span class="hll"><span class="c1">    # Tags subnets for Karpenter auto-discovery</span>
+</span></span><span id="__span-1-22"><a id="__codelineno-1-22" name="__codelineno-1-22" href="#__codelineno-1-22"></a><span class="hll"><span class="w">    </span><span class="s2">&quot;karpenter.sh/discovery&quot;</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">local.name</span>
+</span></span><span id="__span-1-23"><a id="__codelineno-1-23" name="__codelineno-1-23" href="#__codelineno-1-23"></a><span class="w">  </span><span class="p">}</span>
+</span><span id="__span-1-24"><a id="__codelineno-1-24" name="__codelineno-1-24" href="#__codelineno-1-24"></a>
+</span><span id="__span-1-25"><a id="__codelineno-1-25" name="__codelineno-1-25" href="#__codelineno-1-25"></a><span class="w">  </span><span class="na">tags</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">local.tags</span>
+</span><span id="__span-1-26"><a id="__codelineno-1-26" name="__codelineno-1-26" href="#__codelineno-1-26"></a><span class="p">}</span>
+</span></code></pre></div>
+<h3 id="ec2nodeclass-and-nodepool">EC2NodeClass and NodePool<a class="headerlink" href="#ec2nodeclass-and-nodepool" title="Permanent link">&para;</a></h3>
+<div class="language-yaml highlight"><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="nn">---</span>
+</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="nt">apiVersion</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">karpenter.k8s.aws/v1beta1</span>
+</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">EC2NodeClass</span>
+</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="nt">metadata</span><span class="p">:</span>
+</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="w">  </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">default</span>
+</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a><span class="nt">spec</span><span class="p">:</span>
+</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a><span class="w">  </span><span class="nt">amiFamily</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">AL2</span>
+</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a><span class="hll"><span class="w">  </span><span class="nt">role</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ex-karpenter-mng</span>
+</span></span><span id="__span-2-9"><a id="__codelineno-2-9" name="__codelineno-2-9" href="#__codelineno-2-9"></a><span class="hll"><span class="w">  </span><span class="nt">subnetSelectorTerms</span><span class="p">:</span>
+</span></span><span id="__span-2-10"><a id="__codelineno-2-10" name="__codelineno-2-10" href="#__codelineno-2-10"></a><span class="hll"><span class="w">    </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">tags</span><span class="p">:</span>
+</span></span><span id="__span-2-11"><a id="__codelineno-2-11" name="__codelineno-2-11" href="#__codelineno-2-11"></a><span class="hll"><span class="w">        </span><span class="nt">karpenter.sh/discovery</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ex-karpenter-mng</span>
+</span></span><span id="__span-2-12"><a id="__codelineno-2-12" name="__codelineno-2-12" href="#__codelineno-2-12"></a><span class="hll"><span class="w">  </span><span class="nt">securityGroupSelectorTerms</span><span class="p">:</span>
+</span></span><span id="__span-2-13"><a id="__codelineno-2-13" name="__codelineno-2-13" href="#__codelineno-2-13"></a><span class="hll"><span class="w">    </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">tags</span><span class="p">:</span>
+</span></span><span id="__span-2-14"><a id="__codelineno-2-14" name="__codelineno-2-14" href="#__codelineno-2-14"></a><span class="hll"><span class="w">        </span><span class="nt">karpenter.sh/discovery</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ex-karpenter-mng</span>
+</span></span><span id="__span-2-15"><a id="__codelineno-2-15" name="__codelineno-2-15" href="#__codelineno-2-15"></a><span class="hll"><span class="w">  </span><span class="nt">tags</span><span class="p">:</span>
+</span></span><span id="__span-2-16"><a id="__codelineno-2-16" name="__codelineno-2-16" href="#__codelineno-2-16"></a><span class="hll"><span class="w">    </span><span class="nt">karpenter.sh/discovery</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ex-karpenter-mng</span>
+</span></span><span id="__span-2-17"><a id="__codelineno-2-17" name="__codelineno-2-17" href="#__codelineno-2-17"></a><span class="nn">---</span>
+</span><span id="__span-2-18"><a id="__codelineno-2-18" name="__codelineno-2-18" href="#__codelineno-2-18"></a><span class="nt">apiVersion</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">karpenter.sh/v1beta1</span>
+</span><span id="__span-2-19"><a id="__codelineno-2-19" name="__codelineno-2-19" href="#__codelineno-2-19"></a><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">NodePool</span>
+</span><span id="__span-2-20"><a id="__codelineno-2-20" name="__codelineno-2-20" href="#__codelineno-2-20"></a><span class="nt">metadata</span><span class="p">:</span>
+</span><span id="__span-2-21"><a id="__codelineno-2-21" name="__codelineno-2-21" href="#__codelineno-2-21"></a><span class="w">  </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">default</span>
+</span><span id="__span-2-22"><a id="__codelineno-2-22" name="__codelineno-2-22" href="#__codelineno-2-22"></a><span class="nt">spec</span><span class="p">:</span>
+</span><span id="__span-2-23"><a id="__codelineno-2-23" name="__codelineno-2-23" href="#__codelineno-2-23"></a><span class="w">  </span><span class="nt">template</span><span class="p">:</span>
+</span><span id="__span-2-24"><a id="__codelineno-2-24" name="__codelineno-2-24" href="#__codelineno-2-24"></a><span class="w">    </span><span class="nt">spec</span><span class="p">:</span>
+</span><span id="__span-2-25"><a id="__codelineno-2-25" name="__codelineno-2-25" href="#__codelineno-2-25"></a><span class="w">      </span><span class="nt">nodeClassRef</span><span class="p">:</span>
+</span><span id="__span-2-26"><a id="__codelineno-2-26" name="__codelineno-2-26" href="#__codelineno-2-26"></a><span class="w">        </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">default</span>
+</span><span id="__span-2-27"><a id="__codelineno-2-27" name="__codelineno-2-27" href="#__codelineno-2-27"></a><span class="w">      </span><span class="nt">requirements</span><span class="p">:</span>
+</span><span id="__span-2-28"><a id="__codelineno-2-28" name="__codelineno-2-28" href="#__codelineno-2-28"></a><span class="w">        </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">key</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;karpenter.k8s.aws/instance-category&quot;</span>
+</span><span id="__span-2-29"><a id="__codelineno-2-29" name="__codelineno-2-29" href="#__codelineno-2-29"></a><span class="w">          </span><span class="nt">operator</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">In</span>
+</span><span id="__span-2-30"><a id="__codelineno-2-30" name="__codelineno-2-30" href="#__codelineno-2-30"></a><span class="w">          </span><span class="nt">values</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;c&quot;</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="s">&quot;m&quot;</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="s">&quot;r&quot;</span><span class="p p-Indicator">]</span>
+</span><span id="__span-2-31"><a id="__codelineno-2-31" name="__codelineno-2-31" href="#__codelineno-2-31"></a><span class="w">        </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">key</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;karpenter.k8s.aws/instance-cpu&quot;</span>
+</span><span id="__span-2-32"><a id="__codelineno-2-32" name="__codelineno-2-32" href="#__codelineno-2-32"></a><span class="w">          </span><span class="nt">operator</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">In</span>
+</span><span id="__span-2-33"><a id="__codelineno-2-33" name="__codelineno-2-33" href="#__codelineno-2-33"></a><span class="w">          </span><span class="nt">values</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;4&quot;</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="s">&quot;8&quot;</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="s">&quot;16&quot;</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="s">&quot;32&quot;</span><span class="p p-Indicator">]</span>
+</span><span id="__span-2-34"><a id="__codelineno-2-34" name="__codelineno-2-34" href="#__codelineno-2-34"></a><span class="w">        </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">key</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;karpenter.k8s.aws/instance-hypervisor&quot;</span>
+</span><span id="__span-2-35"><a id="__codelineno-2-35" name="__codelineno-2-35" href="#__codelineno-2-35"></a><span class="w">          </span><span class="nt">operator</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">In</span>
+</span><span id="__span-2-36"><a id="__codelineno-2-36" name="__codelineno-2-36" href="#__codelineno-2-36"></a><span class="w">          </span><span class="nt">values</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;nitro&quot;</span><span class="p p-Indicator">]</span>
+</span><span id="__span-2-37"><a id="__codelineno-2-37" name="__codelineno-2-37" href="#__codelineno-2-37"></a><span class="w">        </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">key</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;karpenter.k8s.aws/instance-generation&quot;</span>
+</span><span id="__span-2-38"><a id="__codelineno-2-38" name="__codelineno-2-38" href="#__codelineno-2-38"></a><span class="w">          </span><span class="nt">operator</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Gt</span>
+</span><span id="__span-2-39"><a id="__codelineno-2-39" name="__codelineno-2-39" href="#__codelineno-2-39"></a><span class="w">          </span><span class="nt">values</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="s">&quot;2&quot;</span><span class="p p-Indicator">]</span>
+</span><span id="__span-2-40"><a id="__codelineno-2-40" name="__codelineno-2-40" href="#__codelineno-2-40"></a><span class="w">  </span><span class="nt">limits</span><span class="p">:</span>
+</span><span id="__span-2-41"><a id="__codelineno-2-41" name="__codelineno-2-41" href="#__codelineno-2-41"></a><span class="w">    </span><span class="nt">cpu</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1000</span>
+</span><span id="__span-2-42"><a id="__codelineno-2-42" name="__codelineno-2-42" href="#__codelineno-2-42"></a><span class="w">  </span><span class="nt">disruption</span><span class="p">:</span>
+</span><span id="__span-2-43"><a id="__codelineno-2-43" name="__codelineno-2-43" href="#__codelineno-2-43"></a><span class="w">    </span><span class="nt">consolidationPolicy</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WhenEmpty</span>
+</span><span id="__span-2-44"><a id="__codelineno-2-44" name="__codelineno-2-44" href="#__codelineno-2-44"></a><span class="w">    </span><span class="nt">consolidateAfter</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
+</span></code></pre></div>
+<h2 id="deploy">Deploy<a class="headerlink" href="#deploy" title="Permanent link">&para;</a></h2>
+<p>See <a href="https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites">here</a> for the prerequisites and steps to deploy this pattern.</p>
+<h2 id="validate">Validate<a class="headerlink" href="#validate" title="Permanent link">&para;</a></h2>
+<ol>
+<li>
+<p>Test by listing the nodes in the cluster. You should see four Fargate nodes in the cluster:</p>
+<div class="language-sh highlight"><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a>kubectl<span class="w"> </span>get<span class="w"> </span>nodes
+</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>
+</span><span id="__span-3-3"><a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a>NAME<span class="w">                                      </span>STATUS<span class="w">   </span>ROLES<span class="w">    </span>AGE<span class="w">     </span>VERSION
+</span><span id="__span-3-4"><a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a>ip-10-0-38-5.us-west-2.compute.internal<span class="w">   </span>Ready<span class="w">    </span>&lt;none&gt;<span class="w">   </span>2m40s<span class="w">   </span>v1.29.3-eks-ae9a62a
+</span><span id="__span-3-5"><a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a>ip-10-0-9-38.us-west-2.compute.internal<span class="w">   </span>Ready<span class="w">    </span>&lt;none&gt;<span class="w">   </span>2m35s<span class="w">   </span>v1.29.3-eks-ae9a62a
+</span></code></pre></div>
+</li>
+<li>
+<p>Provision the Karpenter <code>EC2NodeClass</code> and <code>NodePool</code> resources which provide Karpenter the necessary configurations to provision EC2 resources:</p>
+<div class="language-sh highlight"><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a>kubectl<span class="w"> </span>apply<span class="w"> </span>-f<span class="w"> </span>karpenter.yaml
+</span></code></pre></div>
+</li>
+<li>
+<p>Once the Karpenter resources are in place, Karpenter will provision the necessary EC2 resources to satisfy any pending pods in the scheduler's queue. You can demonstrate this with the example deployment provided. First deploy the example deployment which has the initial number replicas set to 0:</p>
+<div class="language-sh highlight"><pre><span></span><code><span id="__span-5-1"><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a>kubectl<span class="w"> </span>apply<span class="w"> </span>-f<span class="w"> </span>example.yaml
+</span></code></pre></div>
+</li>
+<li>
+<p>When you scale the example deployment, you should see Karpenter respond by quickly provisioning EC2 resources to satisfy those pending pod requests:</p>
+<div class="language-sh highlight"><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a>kubectl<span class="w"> </span>scale<span class="w"> </span>deployment<span class="w"> </span>inflate<span class="w"> </span>--replicas<span class="o">=</span><span class="m">3</span>
+</span></code></pre></div>
+</li>
+<li>
+<p>Listing the nodes should now show some EC2 compute that Karpenter has created for the example deployment:</p>
+<div class="language-sh highlight"><pre><span></span><code><span id="__span-7-1"><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a>kubectl<span class="w"> </span>get<span class="w"> </span>nodes
+</span><span id="__span-7-2"><a id="__codelineno-7-2" name="__codelineno-7-2" href="#__codelineno-7-2"></a>
+</span><span id="__span-7-3"><a id="__codelineno-7-3" name="__codelineno-7-3" href="#__codelineno-7-3"></a>NAME<span class="w">                                        </span>STATUS<span class="w">   </span>ROLES<span class="w">    </span>AGE<span class="w">     </span>VERSION
+</span><span id="__span-7-4"><a id="__codelineno-7-4" name="__codelineno-7-4" href="#__codelineno-7-4"></a>ip-10-0-38-109.us-west-2.compute.internal<span class="w">   </span>Ready<span class="w">    </span>&lt;none&gt;<span class="w">   </span>11s<span class="w">     </span>v1.29.3-eks-ae9a62a<span class="w"> </span><span class="c1"># &lt;== EC2 created by Karpenter</span>
+</span><span id="__span-7-5"><a id="__codelineno-7-5" name="__codelineno-7-5" href="#__codelineno-7-5"></a>ip-10-0-38-5.us-west-2.compute.internal<span class="w">     </span>Ready<span class="w">    </span>&lt;none&gt;<span class="w">   </span>3m54s<span class="w">   </span>v1.29.3-eks-ae9a62a
+</span><span id="__span-7-6"><a id="__codelineno-7-6" name="__codelineno-7-6" href="#__codelineno-7-6"></a>ip-10-0-9-38.us-west-2.compute.internal<span class="w">     </span>Ready<span class="w">    </span>&lt;none&gt;<span class="w">   </span>3m49s<span class="w">   </span>v1.29.3-eks-ae9a62a
+</span></code></pre></div>
+</li>
+</ol>
+<h2 id="destroy">Destroy<a class="headerlink" href="#destroy" title="Permanent link">&para;</a></h2>
+<p>Scale down the deployment to de-provision Karpenter created resources first:</p>
+<div class="language-sh highlight"><pre><span></span><code><span id="__span-8-1"><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>-f<span class="w"> </span>example.yaml
+</span></code></pre></div>
+<!-- BEGIN INCLUDE ../../docs/_partials/destroy.md -->
+<div class="language-sh highlight"><pre><span></span><code><span id="__span-9-1"><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a>terraform<span class="w"> </span>destroy<span class="w"> </span>-target<span class="o">=</span><span class="s2">&quot;module.eks_blueprints_addons&quot;</span><span class="w"> </span>-auto-approve
+</span><span id="__span-9-2"><a id="__codelineno-9-2" name="__codelineno-9-2" href="#__codelineno-9-2"></a>terraform<span class="w"> </span>destroy<span class="w"> </span>-target<span class="o">=</span><span class="s2">&quot;module.eks&quot;</span><span class="w"> </span>-auto-approve
+</span><span id="__span-9-3"><a id="__codelineno-9-3" name="__codelineno-9-3" href="#__codelineno-9-3"></a>terraform<span class="w"> </span>destroy<span class="w"> </span>-auto-approve
+</span></code></pre></div>
+<p>See <a href="https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#destroy">here</a> for more details on cleaning up the resources created.</p>
+<!-- END INCLUDE -->
+
+<!-- END INCLUDE -->
+
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+
+                
+              </article>
+            </div>
+          
+          
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+        
+      </main>
+      
+        <footer class="md-footer">
+  
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+  
+    <div class="md-copyright__highlight">
+      Copyright &copy; Amazon 2024
+    </div>
+  
+  
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+  
+</div>
+      
+    </div>
+  </div>
+</footer>
+      
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+    
+    
+    <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs.sticky"], "search": "../../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": {"provider": "mike"}}</script>
+    
+    
+      <script src="../../assets/javascripts/bundle.a7c05c9e.min.js"></script>
+      
+    
+  </body>
+</html>
\ No newline at end of file
diff --git a/patterns/karpenter/index.html b/patterns/karpenter/index.html
index 517667c668..e8704b468b 100644
--- a/patterns/karpenter/index.html
+++ b/patterns/karpenter/index.html
@@ -13,7 +13,7 @@
         <link rel="canonical" href="https://aws-ia.github.io/terraform-aws-eks-blueprints/patterns/karpenter/">
       
       
-        <link rel="prev" href="../istio/">
+        <link rel="prev" href="../karpenter-mng/">
       
       
         <link rel="next" href="../kubecost/">
@@ -24,7 +24,7 @@
     
     
       
-        <title>Karpenter - Amazon EKS Blueprints for Terraform</title>
+        <title>Karpenter on EKS Fargate - Amazon EKS Blueprints for Terraform</title>
       
     
     
@@ -115,7 +115,7 @@
     <div data-md-component="skip">
       
         
-        <a href="#karpenter" class="md-skip">
+        <a href="#karpenter-on-eks-fargate" class="md-skip">
           Skip to content
         </a>
       
@@ -154,7 +154,7 @@
         <div class="md-header__topic" data-md-component="header-topic">
           <span class="md-ellipsis">
             
-              Karpenter
+              Karpenter on EKS Fargate
             
           </span>
         </div>
@@ -554,6 +554,27 @@
                 
   
   
+  
+  
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
     
   
   
@@ -570,7 +591,7 @@
           
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -581,7 +602,7 @@
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
@@ -1284,7 +1305,7 @@
 
 
 <!-- BEGIN INCLUDE ../../patterns/karpenter/README.md -->
-<h1 id="karpenter">Karpenter<a class="headerlink" href="#karpenter" title="Permanent link">&para;</a></h1>
+<h1 id="karpenter-on-eks-fargate">Karpenter on EKS Fargate<a class="headerlink" href="#karpenter-on-eks-fargate" title="Permanent link">&para;</a></h1>
 <p>This pattern demonstrates how to provision Karpenter on a serverless cluster (serverless data plane) using Fargate Profiles.</p>
 <h2 id="deploy">Deploy<a class="headerlink" href="#deploy" title="Permanent link">&para;</a></h2>
 <p>See <a href="https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites">here</a> for the prerequisites and steps to deploy this pattern.</p>
diff --git a/patterns/kubecost/index.html b/patterns/kubecost/index.html
index ef3f3a1942..86687e2510 100644
--- a/patterns/kubecost/index.html
+++ b/patterns/kubecost/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -934,10 +955,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -948,8 +969,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/ml-capacity-block/index.html b/patterns/ml-capacity-block/index.html
index c7b52ae339..1b45d40533 100644
--- a/patterns/ml-capacity-block/index.html
+++ b/patterns/ml-capacity-block/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/multi-tenancy-with-teams/index.html b/patterns/multi-tenancy-with-teams/index.html
index 5de09bb2c6..3c19075ba2 100644
--- a/patterns/multi-tenancy-with-teams/index.html
+++ b/patterns/multi-tenancy-with-teams/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/nvidia-gpu-efa/index.html b/patterns/nvidia-gpu-efa/index.html
index 2d4979d3c4..0853bcf35c 100644
--- a/patterns/nvidia-gpu-efa/index.html
+++ b/patterns/nvidia-gpu-efa/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -934,10 +955,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -948,8 +969,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/private-public-ingress/index.html b/patterns/private-public-ingress/index.html
index d1a4e6dbdc..e869983877 100644
--- a/patterns/private-public-ingress/index.html
+++ b/patterns/private-public-ingress/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/privatelink-access/index.html b/patterns/privatelink-access/index.html
index 53f93a7226..d136b5266d 100644
--- a/patterns/privatelink-access/index.html
+++ b/patterns/privatelink-access/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -967,10 +988,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -981,8 +1002,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/sso-iam-identity-center/index.html b/patterns/sso-iam-identity-center/index.html
index 70c313d569..ca28ae0cd8 100644
--- a/patterns/sso-iam-identity-center/index.html
+++ b/patterns/sso-iam-identity-center/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -949,10 +970,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -963,8 +984,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/sso-okta/index.html b/patterns/sso-okta/index.html
index 34445bce5d..08081ea1a4 100644
--- a/patterns/sso-okta/index.html
+++ b/patterns/sso-okta/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/stateful/index.html b/patterns/stateful/index.html
index 73befa207a..ea651b244a 100644
--- a/patterns/stateful/index.html
+++ b/patterns/stateful/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -976,10 +997,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -990,8 +1011,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/targeted-odcr/index.html b/patterns/targeted-odcr/index.html
index 776051010b..133c8feeb1 100644
--- a/patterns/targeted-odcr/index.html
+++ b/patterns/targeted-odcr/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -934,10 +955,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -948,8 +969,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/tls-with-aws-pca-issuer/index.html b/patterns/tls-with-aws-pca-issuer/index.html
index c0f524fe53..af486ee8b3 100644
--- a/patterns/tls-with-aws-pca-issuer/index.html
+++ b/patterns/tls-with-aws-pca-issuer/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -925,10 +946,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -939,8 +960,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/vpc-lattice/client-server-communication/index.html b/patterns/vpc-lattice/client-server-communication/index.html
index 354c0f73c6..77bcd2ecf0 100644
--- a/patterns/vpc-lattice/client-server-communication/index.html
+++ b/patterns/vpc-lattice/client-server-communication/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -860,10 +881,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" checked>
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -874,8 +895,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/vpc-lattice/index.html b/patterns/vpc-lattice/index.html
index 93e8bccd00..64997887a0 100644
--- a/patterns/vpc-lattice/index.html
+++ b/patterns/vpc-lattice/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -916,10 +937,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -930,8 +951,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/patterns/wireguard-with-cilium/index.html b/patterns/wireguard-with-cilium/index.html
index 7668b11772..c1808ff3a4 100644
--- a/patterns/wireguard-with-cilium/index.html
+++ b/patterns/wireguard-with-cilium/index.html
@@ -556,12 +556,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -934,10 +955,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -948,8 +969,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/search/search_index.json b/search/search_index.json
index c5ab3a4bbf..efd25a6cb2 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Overview","text":""},{"location":"#amazon-eks-blueprints-for-terraform","title":"Amazon EKS Blueprints for Terraform","text":"<p>Welcome to Amazon EKS Blueprints for Terraform!</p> <p>This project contains a collection of Amazon EKS cluster patterns implemented in Terraform that demonstrate how fast and easy it is for customers to adopt Amazon EKS. The patterns can be used by AWS customers, partners, and internal AWS teams to configure and manage complete EKS clusters that are fully bootstrapped with the operational software that is needed to deploy and operate workloads.</p>"},{"location":"#motivation","title":"Motivation","text":"<p>Kubernetes is a powerful and extensible container orchestration technology that allows you to deploy and manage containerized applications at scale. The extensible nature of Kubernetes also allows you to use a wide range of popular open-source tools in Kubernetes clusters. However, With the wide array of tooling and design choices available, configuring an EKS cluster that meets your organization\u2019s specific needs can take a significant amount of time. It involves integrating a wide range of open-source tools and AWS services as well as expertise in AWS and Kubernetes.</p> <p>AWS customers have asked for patterns that demonstrate how to integrate the landscape of Kubernetes tools and make it easy for them to provision complete, opinionated EKS clusters that meet specific application requirements. Customers can utilize EKS Blueprints to configure and deploy purpose built EKS clusters, and start onboarding workloads in days, rather than months.</p>"},{"location":"#consumption","title":"Consumption","text":"<p>EKS Blueprints for Terraform has been designed to be consumed in the following manners:</p> <ol> <li>Reference: Users can refer to the patterns and snippets provided to help guide them to their desired solution. Users will typically view how the pattern or snippet is configured to achieve the desired end result and then replicate that in their environment.</li> <li>Copy &amp; Paste: Users can copy and paste the patterns and snippets into their own environment, using EKS Blueprints as the starting point for their implementation. Users can then adapt the initial pattern to customize it to their specific needs.</li> </ol> <p>EKS Blueprints for Terraform are not intended to be consumed as-is directly from this project. In \"Terraform speak\" - the patterns and snippets provided in this repository are not designed to be consumed as a Terraform module. Therefore, the patterns provided only contain <code>variables</code> when certain information is required to deploy the pattern (i.e. - a Route53 hosted zone ID, or ACM certificate ARN) and generally use local variables. If you wish to deploy the patterns into a different region or with other changes, it is recommended that you make those modifications locally before applying the pattern. EKS Blueprints for Terraform will not expose variables and outputs in the same manner that Terraform modules follow in order to avoid confusion around the consumption model.</p> <p>However, we do have a number of Terraform modules that were created to support EKS Blueprints in addition to the community hosted modules. Please see the respective projects for more details on the modules constructed to support EKS Blueprints for Terraform; those projects are listed below.</p> <ul> <li><code>terraform-aws-eks-blueprint-addon</code> - (Note the singular form) Terraform module which can provision an addon using the Terraform <code>helm_release</code> resource in addition to an IAM role for service account (IRSA).</li> <li><code>terraform-aws-eks-blueprint-addons</code> - (Note the plural form) Terraform module which can provision multiple addons; both EKS addons using the <code>aws_eks_addon</code> resource as well as Helm chart based addons using the <code>terraform-aws-eks-blueprint-addon</code> module.</li> <li><code>terraform-aws-eks-blueprints-teams</code> - Terraform module that creates Kubernetes multi-tenancy resources and configurations, allowing both administrators and application developers to access only the resources which they are responsible for.</li> </ul>"},{"location":"#related-projects","title":"Related Projects","text":"<p>In addition to the supporting EKS Blueprints Terraform modules listed above, there are a number of related projects that users should be aware of:</p> <ol> <li> <p>GitOps</p> <ul> <li><code>terraform-aws-eks-ack-addons</code> -   Terraform module to deploy ACK controllers onto EKS clusters</li> <li><code>crossplane-on-eks</code> - Crossplane Blueprints is an open source repo to bootstrap Amazon EKS clusters and provision AWS resources using a library of Crossplane Compositions (XRs) with Composite Resource Definitions (XRDs).</li> </ul> </li> <li> <p>Data on EKS</p> <ul> <li><code>data-on-eks</code> - A collection of blueprints intended for data workloads on Amazon EKS.</li> <li><code>terraform-aws-eks-data-addons</code> - Terraform module to deploy multiple addons that are specific to data workloads on EKS clusters.</li> </ul> </li> <li> <p>Observability Accelerator</p> <ul> <li><code>terraform-aws-observability-accelerator</code> - A set of opinionated modules to help you set up observability for your AWS environments with AWS-managed observability services such as Amazon Managed Service for Prometheus, Amazon Managed Grafana, AWS Distro for OpenTelemetry (ADOT) and Amazon CloudWatch</li> </ul> </li> <li> <p>Karpenter Blueprints</p> </li> <li><code>karpenter-blueprints</code> - includes a list of common workload scenarios,    some of them go in depth with the explanation of why configuring Karpenter and Kubernetes objects in such a way is important.</li> </ol>"},{"location":"#terraform-caveats","title":"Terraform Caveats","text":"<p>EKS Blueprints for Terraform does not intend to teach users the recommended practices for Terraform nor does it offer guidance on how users should structure their Terraform projects. The patterns provided are intended to show users how they can achieve a defined architecture or configuration in a way that they can quickly and easily get up and running to start interacting with that pattern. Therefore, there are a few caveats users should be aware of when using EKS Blueprints for Terraform:</p> <ol> <li> <p>We recognize that most users will already have an existing VPC in a separate Terraform workspace. However, the patterns provided come complete with a VPC to ensure a stable, deployable example that has been tested and validated.</p> </li> <li> <p>Hashicorp does not recommend providing computed values in provider blocks , which means that the cluster configuration should be defined in a workspace separate from the resources deployed onto the cluster (i.e. - addons). However, to simplify the pattern experience, we have defined everything in one workspace and provided instructions to provision the patterns using a targeted apply approach. Users are encouraged to investigate a Terraform project structure that suites their needs; EKS Blueprints for Terraform does not have an opinion in this matter and will defer to Hashicorp's guidance.</p> </li> <li> <p>Patterns are not intended to be consumed in-place in the same manner that one would consume a module. Therefore, we do not provide variables and outputs to expose various levels of configuration for the examples. Users can modify the pattern locally after cloning to suite their requirements.</p> </li> <li> <p>Please see the FAQ section on authenticating Kubernetes based providers (<code>kubernetes</code>, <code>helm</code>, <code>kubectl</code>) to Amazon EKS clusters regarding the use of static tokens versus dynamic tokens using the <code>awscli</code>.</p> </li> </ol>"},{"location":"#support-feedback","title":"Support &amp; Feedback","text":"<p>EKS Blueprints for Terraform is maintained by AWS Solution Architects. It is not part of an AWS service and support is provided as a best-effort by the EKS Blueprints community. To provide feedback, please use the issues templates provided. If you are interested in contributing to EKS Blueprints, see the Contribution guide.</p>"},{"location":"#security","title":"Security","text":"<p>See CONTRIBUTING for more information.</p>"},{"location":"#license","title":"License","text":"<p>Apache-2.0 Licensed. See LICENSE.</p>"},{"location":"faq/","title":"Frequently Asked Questions","text":""},{"location":"faq/#timeouts-on-destroy","title":"Timeouts on destroy","text":"<p>Customers who are deleting their environments using <code>terraform destroy</code> may see timeout errors when VPCs are being deleted. This is due to a known issue in the vpc-cni</p> <p>Customers may face a situation where ENIs that were attached to EKS managed nodes (same may apply to self-managed nodes) are not being deleted by the VPC CNI as expected which leads to IaC tool failures, such as:</p> <ul> <li>ENIs are left on subnets</li> <li>EKS managed security group which is attached to the ENI can\u2019t be deleted by EKS</li> </ul> <p>The current recommendation is to execute cleanup in the following order:</p> <ol> <li>delete all pods that have been created in the cluster.</li> <li>add delay/ wait</li> <li>delete VPC CNI</li> <li>delete nodes</li> <li>delete cluster</li> </ol>"},{"location":"faq/#leaked-cloudwatch-logs-group","title":"Leaked CloudWatch Logs Group","text":"<p>Sometimes, customers may see the CloudWatch Log Group for EKS cluster being created is left behind after their blueprint has been destroyed using <code>terraform destroy</code>. This happens because even after terraform deletes the CW log group, there\u2019s still logs being processed behind the scene by AWS EKS and service continues to write logs after recreating the log group using the EKS service IAM role which users don't have control over. This results in a terraform failure when the same blueprint is being recreated due to the existing log group left behind.</p> <p>There are two options here:</p> <ol> <li> <p>During cluster creation set <code>var.create_cloudwatch_log_group</code> to <code>false</code>. This will tell the EKS module to not create the log group, but instead let the EKS service create the log group. This means that upon cluster deletion the log group will be left behind but there will not be Terraform failures if you re-create the same cluster as Terraform does not manage the log group creation/deletion anymore.</p> </li> <li> <p>During cluster creation set <code>var.create_cloudwatch_log_group</code> to <code>true</code>. This will tell the EKS module to create the log group via Terraform. The EKS service will detect the log group and will start forwarding the logs for the log types enabled. Upon deletion terraform will delete the log group but depending upon any un-forwarded logs, the EKS service may recreate log group using the service role. This will result in terraform errors if the same blueprint is recreated. To proceed, manually delete the log group using the console or cli rerun the <code>terraform destroy</code>.</p> </li> </ol>"},{"location":"faq/#provider-authentication","title":"Provider Authentication","text":"<p>The chain of events when provisioning an example is typically in the stages of VPC -&gt; EKS cluster -&gt; addons and manifests. Per Terraform's recommendation, it is not recommended to pass an unknown value into provider configurations. However, for the sake of simplicity and ease of use, Blueprints does specify the AWS provider along with the Kubernetes, Helm, and Kubectl providers in order to show the full configuration required for provisioning example. Note - this is the configuration required to provision the example, not necessarily the shape of how the configuration should be structured; users are encouraged to split up EKS cluster creation from addon and manifest provisioning to align with Terraform's recommendations.</p> <p>With that said, the examples here are combining the providers and users can sometimes encounter various issues with the provider authentication methods. There are primarily two methods for authenticating the Kubernetes, Helm, and Kubectl providers to the EKS cluster created:</p> <ol> <li>Using a static token which has a lifetime of 15 minutes per the EKS service documentation.</li> <li>Using the <code>exec()</code> method which will fetch a token at the time of Terraform invocation.</li> </ol> <p>The Kubernetes and Helm providers recommend the <code>exec()</code> method, however this has the caveat that it requires the awscli to be installed on the machine running Terraform AND of at least a minimum version to support the API spec used by the provider (i.e. - <code>\"client.authentication.k8s.io/v1alpha1\"</code>, <code>\"client.authentication.k8s.io/v1beta1\"</code>, etc.). Selecting the appropriate provider authentication method is left up to users, and the examples used in this project will default to using the static token method for ease of use.</p> <p>Users of the static token method should be aware that if they receive a <code>401 Unauthorized</code> message, they might have a token that has expired and will need to run <code>terraform refresh</code> to get a new token. Users of the <code>exec()</code> method should be aware that the <code>exec()</code> method is reliant on the awscli and the associated authentication API version; the awscli version may need to be updated to support a later API version required by the Kubernetes version in use.</p> <p>The following examples demonstrate either method that users can utilize - please refer to the associated provider's documentation for further details on configuration.</p>"},{"location":"faq/#static-token-example","title":"Static Token Example","text":"<pre><code>provider \"kubernetes\" {\n  host                   = module.eks.cluster_endpoint\n  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n  token                  = data.aws_eks_cluster_auth.this.token\n}\n\nprovider \"helm\" {\n  kubernetes {\n    host                   = module.eks.cluster_endpoint\n    cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n    token                  = data.aws_eks_cluster_auth.this.token\n  }\n}\n\nprovider \"kubectl\" {\n  apply_retry_count      = 10\n  host                   = module.eks.cluster_endpoint\n  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n  load_config_file       = false\n  token                  = data.aws_eks_cluster_auth.this.token\n}\n\ndata \"aws_eks_cluster_auth\" \"this\" {\n  name = module.eks.cluster_name\n}\n</code></pre>"},{"location":"faq/#exec-example","title":"<code>exec()</code> Example","text":"<p>Usage of exec plugin for AWS credentials</p> <p>Links to References related to this issue</p> <ul> <li>https://github.com/hashicorp/terraform/issues/29182</li> <li>https://github.com/aws/aws-cli/pull/6476</li> </ul> <pre><code>provider \"kubernetes\" {\n  host                   = module.eks.cluster_endpoint\n  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n\n  exec {\n    api_version = \"client.authentication.k8s.io/v1beta1\"\n    command     = \"aws\"\n    # This requires the awscli to be installed locally where Terraform is executed\n    args = [\"eks\", \"get-token\", \"--cluster-name\", module.eks.cluster_name]\n  }\n}\n\nprovider \"helm\" {\n  kubernetes {\n    host                   = module.eks.cluster_endpoint\n    cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n\n    exec {\n      api_version = \"client.authentication.k8s.io/v1beta1\"\n      command     = \"aws\"\n      # This requires the awscli to be installed locally where Terraform is executed\n      args = [\"eks\", \"get-token\", \"--cluster-name\", module.eks.cluster_name]\n    }\n  }\n}\n\nprovider \"kubectl\" {\n  apply_retry_count      = 5\n  host                   = module.eks.cluster_endpoint\n  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n  load_config_file       = false\n\n  exec {\n    api_version = \"client.authentication.k8s.io/v1beta1\"\n    command     = \"aws\"\n    # This requires the awscli to be installed locally where Terraform is executed\n    args = [\"eks\", \"get-token\", \"--cluster-name\", module.eks.cluster_name]\n  }\n}\n</code></pre>"},{"location":"faq/#unable-to-destroy-namespace-created-by-terraform","title":"Unable to destroy namespace created by Terraform","text":"<p>In some cases, when you try to run terraform destroy on kubernetes resources created by Terraform such as namespace, you may end up seeing failures such as timeout and context deadline exceeded failures. Namespace one of those resources we've seen before, the main reason this happens is because orphaned resources created through CRDs of addons (such as ArgoCD, AWS LBC and more) are left behind after the addons are being deleted, this is case by case scenario. For example, with namespaces:</p> <ol> <li> <p>Confirm the namespace is hanging in status <code>Terminating</code></p> <pre><code>kubectl get namespaces\n</code></pre> </li> <li> <p>Check for any orphaned resources in the namespace, make sure to replace  with your namespace: <pre><code>kubectl api-resources --verbs=list --namespaced -o name   | xargs -n 1 kubectl get  \\\n--show-kind --ignore-not-found -n &lt;namespace_name&gt;\n</code></pre> <li> <p>For any of the above output, patch the resource finalize:</p> <pre><code>kubectl patch RESOURCE NAME -p '{\"metadata\":{\"finalizers\":[]}}' --type=merge\n</code></pre> </li> <li> <p>Check the status of the namespace, if needed you may need to patch the namespace finalizers as-well</p> <pre><code>kubectl patch ns &lt;ns-name&gt; -p '{\"spec\":{\"finalizers\":null}}'\n</code></pre> </li>"},{"location":"getting-started/","title":"Getting Started","text":"<p>This getting started guide will help you deploy your first pattern using EKS Blueprints.</p>"},{"location":"getting-started/#prerequisites","title":"Prerequisites","text":"<p>Ensure that you have installed the following tools locally:</p> <ul> <li>awscli</li> <li>kubectl</li> <li>terraform</li> </ul>"},{"location":"getting-started/#deploy","title":"Deploy","text":"<ol> <li> <p>For consuming EKS Blueprints, please see the Consumption section. For exploring and trying out the patterns provided, please clone the project locally to quickly get up and running with a pattern. After cloning the project locally, <code>cd</code> into the pattern directory of your choice.</p> </li> <li> <p>To provision the pattern, the typical steps of execution are as follows:</p> <pre><code>terraform init\nterraform apply -target=\"module.vpc\" -auto-approve\nterraform apply -target=\"module.eks\" -auto-approve\nterraform apply -auto-approve\n</code></pre> <p>For patterns that deviate from this general flow, see the pattern's respective <code>README.md</code> for more details.</p> <p>Terraform targeted apply</p> <p>Please see the Terraform Caveats section for details on the use of targeted Terraform apply's</p> </li> <li> <p>Once all of the resources have successfully been provisioned, the following command can be used to update the <code>kubeconfig</code> on your local machine and allow you to interact with your EKS Cluster using <code>kubectl</code>.</p> <pre><code>aws eks --region &lt;REGION&gt; update-kubeconfig --name &lt;CLUSTER_NAME&gt;\n</code></pre> <p>Pattern Terraform outputs</p> <p>Most examples will output the <code>aws eks update-kubeconfig ...</code> command as part of the Terraform apply output to simplify this process for users</p> <p>Private clusters</p> <p>Clusters that do not enable the clusters public endpoint will require users to access the cluster from within the VPC. For these patterns, a sample EC2 or other means are provided to demonstrate how to access those clusters privately</p> <p>and without exposing the public endpoint. Please see the respective pattern's <code>README.md</code> for more details.</p> </li> <li> <p>Once you have updated your <code>kubeconfig</code>, you can verify that you are able to interact with your cluster by running the following command:</p> <pre><code>kubectl get nodes\n</code></pre> <p>This should return a list of the node(s) running in the cluster created. If any errors are encountered, please re-trace the steps above and consult the pattern's <code>README.md</code> for more details on any additional/specific steps that may be required.</p> </li> </ol>"},{"location":"getting-started/#destroy","title":"Destroy","text":"<p>To teardown and remove the resources created in the pattern, the typical steps of execution are as follows:</p> <pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>Resources created outside of Terraform</p> <p>Depending on the pattern, some resources may have been created that Terraform is not aware of that will cause issues when attempting to clean up the pattern. For example, Karpenter is responsible for creating additional EC2 instances to satisfy the pod scheduling requirements. These instances will not be cleaned up by Terraform and will need to be de-provisioned BEFORE attempting to <code>terraform destroy</code>. This is why it is important that the addons, or any resources provisioned onto the cluster are cleaned up first. Please see the respective pattern's <code>README.md</code> for more details.</p>"},{"location":"_partials/destroy/","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"internal/ci/","title":"E2E tests","text":"<p>We use GitHub Actions to run an end-to-end tests to verify all PRs. The GitHub Actions used are a combination of <code>aws-actions/configure-aws-credentials</code> and <code>hashicorp/setup-terraform@v1</code>.</p>"},{"location":"internal/ci/#setup","title":"Setup","text":"<ol> <li>Use the following CloudFormation template to setup a new IAM role.</li> </ol> <pre><code>Parameters:\n  GitHubOrg:\n    Type: String\n  RepositoryName:\n    Type: String\n  OIDCProviderArn:\n    Description: Arn for the GitHub OIDC Provider.\n    Default: \"\"\n    Type: String\n\nConditions:\n  CreateOIDCProvider: !Equals\n    - !Ref OIDCProviderArn\n    - \"\"\n\nResources:\n  Role:\n    Type: AWS::IAM::Role\n    Properties:\n      AssumeRolePolicyDocument:\n        Statement:\n          - Effect: Allow\n            Action: sts:AssumeRoleWithWebIdentity\n            Principal:\n              Federated: !If\n                - CreateOIDCProvider\n                - !Ref GithubOidc\n                - !Ref OIDCProviderArn\n            Condition:\n              StringLike:\n                token.actions.githubusercontent.com:sub: !Sub repo:${GitHubOrg}/${RepositoryName}:*\n\n  GithubOidc:\n    Type: AWS::IAM::OIDCProvider\n    Condition: CreateOIDCProvider\n    Properties:\n      Url: https://token.actions.githubusercontent.com\n      ClientIdList:\n        - sts.amazonaws.com\n      ThumbprintList:\n        - a031c46782e6e6c662c2c87c76da9aa62ccabd8e\n\nOutputs:\n  Role:\n    Value: !GetAtt Role.Arn\n</code></pre> <ol> <li> <p>Add a permissible IAM Policy to the above create role. For our purpose <code>AdministratorAccess</code> works the best.</p> </li> <li> <p>Setup a GitHub repo secret called <code>ROLE_TO_ASSUME</code> and set it to ARN of the role created in 1.</p> </li> <li> <p>We use an S3 backend for the e2e tests. This allows us to recover from any failures during the <code>apply</code> stage. If you are setting up your own CI pipeline change the s3 bucket name in backend configuration of the example.</p> </li> </ol>"},{"location":"patterns/agones-game-controller/","title":"Agones Game Controller","text":""},{"location":"patterns/agones-game-controller/#amazon-eks-deployment-with-agones-gaming-kubernetes-controller","title":"Amazon EKS Deployment with Agones Gaming Kubernetes Controller","text":"<p>This pattern shows how to deploy and run gaming applications on Amazon EKS using the Agones Kubernetes Controller</p> <p>Agones is an open source Kubernetes controller that provisions and manages dedicated game server processes within Kubernetes clusters using standard Kubernetes tooling and APIs. This model also allows any matchmaker to interact directly with Agones via the Kubernetes API to provision a dedicated game server</p> <p>Amazon GameLift enables developers to deploy, operate, and scale dedicated, low-cost servers in the cloud for session-based, multiplayer games. Built on AWS global computing infrastructure, GameLift helps deliver high-performance, high-reliability, low-cost game servers while dynamically scaling your resource usage to meet worldwide player demand. See below for more information on how GameLift FleetIQ can be integrated with Agones deployed on Amazon EKS.</p> <p>Amazon GameLift FleetIQ optimizes the use of low-cost Spot Instances for cloud-based game hosting with Amazon EC2. With GameLift FleetIQ, you can work directly with your hosting resources in Amazon EC2 and Auto Scaling while taking advantage of GameLift optimizations to deliver inexpensive, resilient game hosting for your players and makes the use of low-cost Spot Instances viable for game hosting</p> <p>This blog walks through the details of deploying EKS Cluster using eksctl and deploy Agones with GameLift FleetIQ.</p>"},{"location":"patterns/agones-game-controller/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/agones-game-controller/#validate","title":"Validate","text":"<ol> <li> <p>Deploy the sample game server</p> <pre><code>kubectl create -f https://raw.githubusercontent.com/googleforgames/agones/release-1.32.0/examples/simple-game-server/gameserver.yaml\nkubectl get gs\n</code></pre> <pre><code>NAME                       STATE   ADDRESS         PORT   NODE                                        AGE\nsimple-game-server-7r6jr   Ready   34.243.345.22   7902   ip-10-1-23-233.eu-west-1.compute.internal   11h\n</code></pre> </li> <li> <p>Test the sample game server using <code>netcat</code></p> <pre><code>echo -n \"UDP test - Hello EKS Blueprints!\" | nc -u 34.243.345.22 7902\n</code></pre> <pre><code>Hello EKS Blueprints!\nACK: Hello EKS Blueprints!\nEXIT\nACK: EXIT\n</code></pre> </li> </ol>"},{"location":"patterns/agones-game-controller/#destroy","title":"Destroy","text":"<p>Delete the resources created by the sample game server first:</p> <pre><code>kubectl -n default delete gs --all || true\n</code></pre> <pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/aws-vpc-cni-network-policy/","title":"AWS VPC CNI Network Policy","text":""},{"location":"patterns/aws-vpc-cni-network-policy/#amazon-eks-cluster-w-network-policies","title":"Amazon EKS Cluster w/ Network Policies","text":"<p>This pattern demonstrates an EKS cluster that uses the native Network Policy support provided by the Amazon VPC CNI (1.14.0 or higher).</p> <ul> <li>Documentation</li> <li>Launch Blog</li> </ul>"},{"location":"patterns/aws-vpc-cni-network-policy/#scenario","title":"Scenario","text":"<p>This pattern deploys an Amazon EKS Cluster with Network Policies support implemented by the Amazon VPC CNI. Further it deploys a simple demo application (distributed as a Helm Chart) and some sample Network Policies to restrict the traffic between different components of the application.</p> <p>For a detailed description of the demo application and the Network Policies, please refer to the Stars demo of network policy section in the official Documentation.</p>"},{"location":"patterns/aws-vpc-cni-network-policy/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/aws-vpc-cni-network-policy/#validate","title":"Validate","text":"<ol> <li> <p>List out the pods running currently:</p> <pre><code>kubectl get pods -A\n</code></pre> <p><code>text    NAMESPACE         NAME                                       READY   STATUS    RESTARTS   AGE [...] client            client-xlf2c                               1/1     Running   0          5m19s [...] management-ui     management-ui-qrb2g                        1/1     Running   0          5m24s stars             backend-sz87q                              1/1     Running   0          5m23s stars             frontend-c1cnf                             1/1     Running   0          5m21s [...]</code></p> <p>In your output, you should see pods in the namespaces shown in the following output. The NAMES of your pods and the number of pods in the READY column are different than those in the following output. Don't continue until you see pods with similar names and they all have Running in the STATUS column.</p> </li> <li> <p>Connect to the management user interface using the EXTERNAL IP of the running service and observe the traffic flow and restrictions based on the Network Policies deployed:</p> <pre><code>kubectl get service/management-ui -n management-ui\n</code></pre> <p>Open the browser based on the URL obtained from the previous step to see the connection map and restrictions put in place by the Network Policies deployed.</p> </li> </ol>"},{"location":"patterns/aws-vpc-cni-network-policy/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/blue-green-upgrade/","title":"Blue/Green Upgrade","text":""},{"location":"patterns/blue-green-upgrade/#bluegreen-migration","title":"Blue/Green Migration","text":"<p>This directory provides a solution based on EKS Blueprint for Terraform that shows how to leverage blue/green or canary application workload migration between EKS clusters, using Amazon Route 53 weighted routing feature. The workloads will be dynamically exposed using AWS LoadBalancer Controller and External DNS add-on.</p> <p>We are leveraging the existing EKS Blueprints Workloads GitHub repository sample to deploy our GitOps ArgoCD applications, which are defined as helm charts. We are leveraging ArgoCD Apps of apps pattern where an ArgoCD Application can also reference other Helm charts to deploy.</p> <p>You can also find more informations in the associated blog post</p>"},{"location":"patterns/blue-green-upgrade/#table-of-content","title":"Table of content","text":"<ul> <li>Blue/Green Migration</li> <li>Table of content</li> <li>Project structure</li> <li>Prerequisites</li> <li>Quick Start<ul> <li>Configure the Stacks</li> <li>Create the environment stack</li> <li>Create the Blue cluster</li> <li>Create the Green cluster</li> </ul> </li> <li>How this work<ul> <li>Watch our Workload: we focus on team-burnham namespace.</li> <li>Using AWS Route53 and External DNS</li> <li>Configure Ingress resources with weighted records</li> </ul> </li> <li>Automate the migration from Terraform</li> <li>Delete the Stack<ul> <li>Delete the EKS Cluster(s)</li> <li>TL;DR</li> </ul> </li> <li>Troubleshoot<ul> <li>External DNS Ownership</li> <li>Check Route 53 Record status</li> <li>Check current resolution and TTL value</li> <li>Get ArgoCD UI Password</li> </ul> </li> </ul>"},{"location":"patterns/blue-green-upgrade/#project-structure","title":"Project structure","text":"<p>See the Architecture of what we are building</p> <p> </p> <p>Our sample is composed of four main directory:</p> <ul> <li>environment \u2192 this stack will create the common VPC and its dependencies used by our EKS clusters: create a Route53 sub domain hosted zone for our sample, a wildcard certificate on Certificate Manager for our applications TLS endpoints, and a SecretManager password for the ArgoCD UIs.</li> <li>modules/eks_cluster \u2192 local module defining the EKS blueprint cluster with ArgoCD add-on which will automatically deploy additional add-ons and our demo workloads</li> <li>eks-blue \u2192 an instance of the eks_cluster module to create blue cluster</li> <li>eks-green \u2192 an instance of the eks_cluster module to create green cluster</li> </ul> <p>So we are going to create 2 EKS clusters, sharing the same VPC, and each one of them will install locally our workloads from the central GitOps repository leveraging ArgoCD add-on. In the GitOps workload repository, we have configured our applications deployments to leverage AWS Load Balancers Controllers annotations, so that applications will be exposed on AWS Load Balancers, created from our Kubernetes manifests. We will have 1 load balancer per cluster for each of our applications.</p> <p>We have configured ExternalDNS add-ons in our two clusters to share the same Route53 Hosted Zone. The workloads in both clusters also share the same Route 53 DNS records, we rely on AWS Route53 weighted records to allow us to configure canary workload migration between our two EKS clusters.</p> <p>We are leveraging the gitops-bridge-argocd-bootstrap terraform module that allow us to dynamically provide metadatas from Terraform to ArgoCD deployed in the cluster. For doing this, the module will extract all metadatas from the terraform-aws-eks-blueprints-addons module, configured to create all resources except installing the addon's Helm chart. The addon Installation will be delegate to ArgoCD Itself using the eks-blueprints-add-ons git repository containing ArgoCD ApplicaitonSets for each supported Addons.</p> <p>The gitops-bridge will create a secret in the EKS cluster containing all metadatas that will be dynamically used by ArgoCD ApplicationSets at deployment time, so that we can adapt their configuration to our EKS cluster context.</p> <p></p> <p>Our objective here is to show you how Application teams and Platform teams can configure their infrastructure and workloads so that application teams are able to deploy autonomously their workloads to the EKS clusters thanks to ArgoCD, and platform team can keep the control of migrating production workloads from one cluster to another without having to synchronized operations with applications teams, or asking them to build a complicated CD pipeline.</p> <p>In this example we show how you can seamlessly migrate your stateless workloads between the 2 clusters for a blue/green or Canary migration, but you can also leverage the same architecture to have your workloads for example separated in different accounts or regions, for either High Availability or Lower latency Access from your customers.</p>"},{"location":"patterns/blue-green-upgrade/#prerequisites","title":"Prerequisites","text":"<ul> <li>Terraform (tested version v1.3.5 on linux)</li> <li>Git</li> <li>AWS CLI</li> <li>AWS test account with administrator role access</li> <li>For working with this repository, you will need an existing Amazon Route 53 Hosted Zone that will be used to create our project hosted zone. It will be provided via the Terraform variable <code>hosted_zone_name</code> defined in terraform.tfvars.example.</li> <li>Before moving to the next step, you will need to register a parent domain with AWS Route 53 (https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html) in case you don\u2019t have one created yet.</li> <li>Accessing GitOps git repositories with SSH access requiring an SSH key for authentication. In this example our workloads repositories are stored in GitHub, you can see in GitHub documentation on how to connect with SSH.</li> <li>Your GitHub private ssh key value is supposed to be stored in plain text in AWS Secret Manager in a secret named <code>github-blueprint-ssh-key</code>, but you can change it using the terraform variable <code>workload_repo_secret</code> in terraform.tfvars.example.</li> <li></li> </ul>"},{"location":"patterns/blue-green-upgrade/#quick-start","title":"Quick Start","text":""},{"location":"patterns/blue-green-upgrade/#configure-the-stacks","title":"Configure the Stacks","text":"<ol> <li>Clone the repository</li> </ol> <pre><code>git clone https://github.com/aws-ia/terraform-aws-eks-blueprints.git\ncd patterns/blue-green-upgrade/\n</code></pre> <ol> <li>Copy the <code>terraform.tfvars.example</code> to <code>terraform.tfvars</code> and symlink it on each <code>environment</code>, <code>eks-blue</code> and <code>eks-green</code> folders, and change region, hosted_zone_name, eks_admin_role_name according to your needs.</li> </ol> <pre><code>cp terraform.tfvars.example terraform.tfvars\nln -s ../terraform.tfvars environment/terraform.tfvars\nln -s ../terraform.tfvars eks-blue/terraform.tfvars\nln -s ../terraform.tfvars eks-green/terraform.tfvars\n</code></pre> <ul> <li>You will need to provide the <code>hosted_zone_name</code> for example <code>my-example.com</code>. Terraform will create a new hosted zone for the project with name: <code>${environment}.${hosted_zone_name}</code> so in our example <code>eks-blueprint.my-example.com</code>.</li> <li>You need to provide a valid IAM role in <code>eks_admin_role_name</code> to have EKS cluster admin rights, generally the one uses in the EKS console.</li> </ul>"},{"location":"patterns/blue-green-upgrade/#create-the-environment-stack","title":"Create the environment stack","text":"<p>More info in the environment Readme</p> <pre><code>cd environment\nterraform init\nterraform apply\n</code></pre> <p>There can be some Warnings due to not declare variables. This is normal and you can ignore them as we share the same <code>terraform.tfvars</code> for the 3 projects by using symlinks for a unique file, and we declare some variables used for the eks-blue and eks-green directory</p>"},{"location":"patterns/blue-green-upgrade/#create-the-blue-cluster","title":"Create the Blue cluster","text":"<p>More info in the eks-blue Readme, you can also see the detailed step in the local module Readme</p> <pre><code>cd eks-blue\nterraform init\nterraform apply\n</code></pre> <p>This can take 8mn for EKS cluster, 15mn</p>"},{"location":"patterns/blue-green-upgrade/#create-the-green-cluster","title":"Create the Green cluster","text":"<pre><code>cd eks-green\nterraform init\nterraform apply\n</code></pre> <p>By default the only differences in the 2 clusters are the values defined in main.tf. We will change those values to upgrade Kubernetes version of new cluster, and to migrate our stateless workloads between clusters.</p>"},{"location":"patterns/blue-green-upgrade/#how-this-work","title":"How this work","text":""},{"location":"patterns/blue-green-upgrade/#watch-our-workload-we-focus-on-team-burnham-namespace","title":"Watch our Workload: we focus on team-burnham namespace.","text":"<p>Our clusters are configured with existing ArgoCD Github repository that is synchronized into each of the clusters:</p> <ul> <li>EKS Blueprints Add-ons repository</li> <li>Workloads repository</li> </ul> <p> </p> <p>We are going to look after one of the application deployed from the workload repository as example to demonstrate our migration automation: the <code>Burnham</code> workload in the team-burnham namespace. We have set up a simple go application than can respond in it's body the name of the cluster it is running on. With this it will be easy to see the current migration on our workload.</p> <pre><code>&lt;head&gt;\n  &lt;title&gt;Hello EKS Blueprint&lt;/title&gt;\n&lt;/head&gt;\n&lt;div class=\"info\"&gt;\n  &lt;h&gt;Hello EKS Blueprint Version 1.4&lt;/h&gt;\n  &lt;p&gt;&lt;span&gt;Server&amp;nbsp;address:&lt;/span&gt; &lt;span&gt;10.0.2.201:34120&lt;/span&gt;&lt;/p&gt;\n  &lt;p&gt;&lt;span&gt;Server&amp;nbsp;name:&lt;/span&gt; &lt;span&gt;burnham-9d686dc7b-dw45m&lt;/span&gt;&lt;/p&gt;\n  &lt;p class=\"smaller\"&gt;&lt;span&gt;Date:&lt;/span&gt; &lt;span&gt;2022.10.13 07:27:28&lt;/span&gt;&lt;/p&gt;\n  &lt;p class=\"smaller\"&gt;&lt;span&gt;URI:&lt;/span&gt; &lt;span&gt;/&lt;/span&gt;&lt;/p&gt;\n  &lt;p class=\"smaller\"&gt;&lt;span&gt;HOST:&lt;/span&gt; &lt;span&gt;burnham.eks-blueprint.mon-domain.com&lt;/span&gt;&lt;/p&gt;\n  &lt;p class=\"smaller\"&gt;&lt;span&gt;CLUSTER_NAME:&lt;/span&gt; &lt;span&gt;eks-blueprint-blue&lt;/span&gt;&lt;/p&gt;\n&lt;/div&gt;\n</code></pre> <p>The application is deployed from our  workload repository manifest <p>Connect to the cluster: Execute one of the EKS cluster login commands from the <code>terraform output</code> command, depending on the IAM role you can assume to access to the cluster. If you want EKS Admin cluster, you can execute the command associated to the eks_blueprints_admin_team_configure_kubectl output. It should be something similar to:</p> <pre><code>aws eks --region eu-west-3 update-kubeconfig --name eks-blueprint-blue  --role-arn arn:aws:iam::0123456789:role/admin-team-20230505075455219300000002\n</code></pre> <p>Note it will allow the role associated to the parameter eks_admin_role_name to assume the role.</p> <p>You can also connect with the user who created the EKS cluster without specifying the <code>--role-arn</code> parameter</p> <p>Next, you can interact with the cluster and see the deployment</p> <pre><code>$ kubectl get deployment -n team-burnham -l app=burnham\nNAME      READY   UP-TO-DATE   AVAILABLE   AGE\nburnham   3/3     3            3           3d18h\n</code></pre> <p>See the pods</p> <pre><code>$ kubectl get pods -n team-burnham -l app=burnham\nNAME                       READY   STATUS    RESTARTS   AGE\nburnham-7db4c6fdbb-82hxn   1/1     Running   0          3d18h\nburnham-7db4c6fdbb-dl59v   1/1     Running   0          3d18h\nburnham-7db4c6fdbb-hpq6h   1/1     Running   0          3d18h\n</code></pre> <p>See the logs:</p> <pre><code>$ kubectl logs -n team-burnham -l app=burnham\n2022/10/10 12:35:40 {url: / }, cluster: eks-blueprint-blue }\n2022/10/10 12:35:49 {url: / }, cluster: eks-blueprint-blue }\n</code></pre> <p>You can make a request to the service, and filter the output to know on which cluster it runs:</p> <pre><code>$ URL=$(echo -n \"https://\" ; kubectl get ing -n team-burnham burnham-ingress -o json | jq \".spec.rules[0].host\" -r)\n$ curl -s $URL | grep CLUSTER_NAME | awk -F \"&lt;span&gt;|&lt;/span&gt;\" '{print $4}'\neks-blueprint-blue\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#using-aws-route53-and-external-dns","title":"Using AWS Route53 and External DNS","text":"<p>We have configured both our clusters to configure the same Amazon Route 53 Hosted Zones. This is done by having the same configuration of ExternalDNS add-on in <code>main.tf</code>:</p> <p>This is the Terraform configuration to configure the ExternalDNS Add-on which is deployed by the Blueprint using ArgoCD. we specify the Route53 zone that external-dns needs to monitor.</p> <pre><code>  enable_external_dns = true\n  external_dns_route53_zone_arns      = [data.aws_route53_zone.sub.arn]\n</code></pre> <p>we also configure the addons_metadata to provide more configurations to external-dns:</p> <pre><code>addons_metadata = merge(\n  ...\n  external_dns_policy        = \"sync\"\n</code></pre> <ul> <li>We use ExternalDNS in <code>sync</code> mode so that the controller can create but also remove DNS records accordingly to service or ingress objects creation.</li> <li>We also configured the <code>txtOwnerId</code> with the name of the cluster, so that each controller will be able to create/update/delete records but only for records which are associated to the proper OwnerId.</li> <li>Each Route53 record will be also associated with a <code>txt</code> record. This record is used to specify the owner of the associated record and is in the form of:</li> </ul> <pre><code>\"heritage=external-dns,external-dns/owner=eks-blueprint-blue,external-dns/resource=ingress/team-burnham/burnham-ingress\"\n</code></pre> <p>So in this example the Owner of the record is the external-dns controller, from the eks-blueprint-blue EKS cluster, and correspond to the Kubernetes ingress resource names burnham-ingress in the team-burnham namespace.</p> <p>Using this feature, and relying on weighted records, we will be able to do blue/green or canary migration by changing the weight of ingress resources defined in each cluster.</p>"},{"location":"patterns/blue-green-upgrade/#configure-ingress-resources-with-weighted-records","title":"Configure Ingress resources with weighted records","text":"<p>Since we have configured ExternalDNS add-on, we can now defined specific annotation in our <code>ingress</code> object. You may already know that our workload are synchronized using ArgoCD from our workload repository sample.</p> <p>We are focusing on the burnham deployment, which is defined here where we configure the <code>burnham-ingress</code> ingress object with:</p> <pre><code>    external-dns.alpha.kubernetes.io/set-identifier: {{ .Values.spec.clusterName }}\n    external-dns.alpha.kubernetes.io/aws-weight: '{{ .Values.spec.ingress.route53_weight }}'\n</code></pre> <p>We rely on two external-dns annotation to configure how the record will be created. the <code>set-identifier</code> annotation will contain the name of the cluster we want to create, which must match the one define in the external-dns <code>txtOwnerId</code> configuration.</p> <p>The <code>aws-weight</code> will be used to configure the value of the weighted record, and it will be deployed from Helm values, that will be injected by Terraform in our example, so that our platform team will be able to control autonomously how and when they want to migrate workloads between the EKS clusters.</p> <p>Amazon Route 53 weighted records works like this:</p> <ul> <li>If we specify a value of 100 in eks-blue cluster and 0 in eks-green cluster, then Route 53 will route all requests to eks-blue cluster.</li> <li>If we specify a value of 0 in eks-blue cluster and 0 in eks-green cluster, then Route 53 will route all requests to eks-green cluster.</li> <li>we can also define any intermediate values like 100 in eks-blue cluster and 100 in eks-green cluster, so we will have 50% on eks-blue and 50% on eks-green.</li> </ul>"},{"location":"patterns/blue-green-upgrade/#automate-the-migration-from-terraform","title":"Automate the migration from Terraform","text":"<p>Now that we have setup our 2 clusters, deployed with ArgoCD and that the weighed records from <code>values.yaml</code> are injected from Terraform, let's see how our Platform team can trigger the workload migration.</p> <ol> <li>At first, 100% of burnham traffic is set to the eks-blue cluster, this is controlled from the <code>eks-blue/main.tf</code> &amp; <code>eks-green/main.tf</code> files with the parameter <code>route53_weight = \"100\"</code>. The same parameter is set to 0 in cluster eks-green.</li> </ol> <p> </p> <p>Which correspond to :</p> <p> </p> <p>All requests to our endpoint should response with <code>eks-blueprint-blue</code> we can test it with the following command:</p> <pre><code>URL=$(echo -n \"https://\" ; kubectl get ing -n team-burnham burnham-ingress -o json | jq \".spec.rules[0].host\" -r)\ncurl -s $URL | grep CLUSTER_NAME | awk -F \"&lt;span&gt;|&lt;/span&gt;\" '{print $4}'\n</code></pre> <p>you should see:</p> <pre><code>eks-blueprint-blue\n</code></pre> <ol> <li>Let's change traffic to 50% eks-blue and 50% eks-green by activating also value 100 in eks-green locals.tf (<code>route53_weight = \"100\"</code>) and let's <code>terraform apply</code> to let terraform update the configuration</li> </ol> <p> </p> <p>Which correspond to :</p> <p> </p> <p>All records have weight of 100, so we will have 50% requests on each clusters.</p> <p>We can check the ratio of requests resolution between both clusters</p> <pre><code>URL=$(echo -n \"https://\" ; kubectl get ing -n team-burnham burnham-ingress -o json | jq \".spec.rules[0].host\" -r)\nrepeat 10 curl -s $URL | grep CLUSTER_NAME | awk -F \"&lt;span&gt;|&lt;/span&gt;\" '{print $4}' &amp;&amp; sleep 60\n</code></pre> <p>Result should be similar to:</p> <pre><code>eks-blueprint-blue\neks-blueprint-blue\neks-blueprint-blue\neks-blueprint-blue\neks-blueprint-green\neks-blueprint-green\neks-blueprint-blue\neks-blueprint-green\neks-blueprint-blue\neks-blueprint-green\n</code></pre> <p>The default TTL is for 60 seconds, and you have 50% chance to have blue or green cluster, then you may need to replay the previous command several times to have an idea of the repartition, which theoretically is 50%</p> <ol> <li>Now that we see that our green cluster is taking requests correctly, we can update the eks-blue cluster configuration to have the weight to 0 and apply again. after a few moment, your route53 records should look like the below screenshot, and all requests should now reach eks-green cluster.</li> </ol> <p> </p> <p>Which correspond to :</p> <p> </p> <p>At this step, once all DNS TTL will be up to date, all the traffic will be coming on the eks-green cluster. You can either, delete the eks-blue cluster, or decide to make upgrades on the blue cluster and send back traffic on eks-blue afterward, or simply keep it as a possibility for rollback if needed.</p> <p>In this sample, we uses a simple terraform variable to control the weight for all applications, we can also choose to have several parameters, let's say one per application, so you can finer control your migration strategy application by application.</p>"},{"location":"patterns/blue-green-upgrade/#delete-the-stack","title":"Delete the Stack","text":""},{"location":"patterns/blue-green-upgrade/#delete-the-eks-clusters","title":"Delete the EKS Cluster(s)","text":"<p>This section, can be executed in either eks-blue or eks-green folders, or in both if you want to delete both clusters.</p> <p>In order to properly destroy the Cluster, we need first to remove the ArgoCD workloads, while keeping the ArgoCD addons. We will also need to remove our Karpenter provisioners, and any other objects you created outside of Terraform that needs to be cleaned before destroying the terraform stack.</p> <p>Why doing this? When we remove an ingress object, we want the associated Kubernetes add-ons like aws load balancer controller and External DNS to correctly free the associated AWS resources. If we directly ask terraform to destroy everything, it can remove first theses controllers without allowing them the time to remove associated aws resources that will still existing in AWS, preventing us to completely delete our cluster.</p>"},{"location":"patterns/blue-green-upgrade/#tldr","title":"TL;DR","text":"<pre><code>../tear-down.sh\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#troubleshoot","title":"Troubleshoot","text":""},{"location":"patterns/blue-green-upgrade/#external-dns-ownership","title":"External DNS Ownership","text":"<p>The Amazon Route 53 records association are controlled by ExternalDNS controller. You can see the logs from the controller to understand what is happening by executing the following command in each cluster:</p> <pre><code>kubectl logs  -n external-dns -l app.kubernetes.io/name=external-dns -f\n</code></pre> <p>In eks-blue cluster, you can see logs like the following, which showcase that the eks-blueprint-blue controller won't make any change in records owned by eks-blueprint-green cluster, the reverse is also true.</p> <pre><code>time=\"2022-10-10T15:46:54Z\" level=debug msg=\"Skipping endpoint skiapp.eks-blueprint.sallaman.people.aws.dev 300 IN CNAME eks-blueprint-green k8s-riker-68438cd99f-893407990.eu-west-1.elb.amazonaws.com [{aws/evaluate-target-health true} {alias true} {aws/weight 100}] because owner id does not match, found: \\\"eks-blueprint-green\\\", required: \\\"eks-blueprint-blue\\\"\"\ntime=\"2022-10-10T15:46:54Z\" level=debug msg=\"Refreshing zones list cache\"\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#check-route-53-record-status","title":"Check Route 53 Record status","text":"<p>We can also use the CLI to see our current Route 53 configuration:</p> <pre><code>export ROOT_DOMAIN=&lt;your-domain-name&gt; # the value you put in hosted_zone_name\nZONE_ID=$(aws route53 list-hosted-zones-by-name --output json --dns-name \"eks-blueprint.${ROOT_DOMAIN}.\" --query \"HostedZones[0].Id\" --out text)\necho $ZONE_ID\naws route53 list-resource-record-sets \\\n  --output json \\\n  --hosted-zone-id $ZONE_ID \\\n  --query \"ResourceRecordSets[?Name == 'burnham.eks-blueprint.$ROOT_DOMAIN.']|[?Type == 'A']\"\n\naws route53 list-resource-record-sets \\\n  --output json \\\n  --hosted-zone-id $ZONE_ID \\\n  --query \"ResourceRecordSets[?Name == 'burnham.eks-blueprint.$ROOT_DOMAIN.']|[?Type == 'TXT']\"\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#check-current-resolution-and-ttl-value","title":"Check current resolution and TTL value","text":"<p>As DNS migration is dependent of DNS caching, normally relying on the TTL, you can use dig to see the current value of the TTL used locally</p> <pre><code>export ROOT_DOMAIN=&lt;your-domain-name&gt; # the value you put for hosted_zone_name\ndig +noauthority +noquestion +noadditional +nostats +ttlunits +ttlid A burnham.eks-blueprint.$ROOT_DOMAIN\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#get-argocd-ui-password","title":"Get ArgoCD UI Password","text":"<p>You can connect to the ArgoCD UI using the service :</p> <pre><code>kubectl get svc -n argocd argo-cd-argocd-server -o json | jq '.status.loadBalancer.ingress[0].hostname' -r\n</code></pre> <p>Then login with admin and get the password from AWS Secret Manager:</p> <pre><code>aws secretsmanager get-secret-value \\\n  --secret-id argocd-admin-secret.eks-blueprint \\\n  --query SecretString \\\n  --output text --region $AWS_REGION\n</code></pre>"},{"location":"patterns/external-secrets/","title":"External Secrets","text":""},{"location":"patterns/external-secrets/#amazon-eks-cluster-w-external-secrets-operator","title":"Amazon EKS Cluster w/ External Secrets Operator","text":"<p>This pattern deploys an EKS Cluster with the External Secrets Operator. The cluster is populated with a ClusterSecretStore and SecretStore example using SecretManager and Parameter Store respectively. A secret for each store is also created. Both stores use IRSA to retrieve the secret values from AWS.</p>"},{"location":"patterns/external-secrets/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/external-secrets/#validate","title":"Validate","text":"<ol> <li> <p>List the secret resources in the <code>external-secrets</code> namespace</p> <pre><code>kubectl get externalsecrets -n external-secrets\nkubectl get secrets -n external-secrets\n</code></pre> </li> </ol>"},{"location":"patterns/external-secrets/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/fargate-serverless/","title":"Fargate Serverless","text":""},{"location":"patterns/fargate-serverless/#serverless-amazon-eks-cluster","title":"Serverless Amazon EKS Cluster","text":"<p>This pattern demonstrates an Amazon EKS Cluster that utilizes Fargate profiles for a serverless data plane.</p>"},{"location":"patterns/fargate-serverless/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/fargate-serverless/#validate","title":"Validate","text":"<ol> <li> <p>List the nodes in in the cluster; you should see Fargate instances:</p> <pre><code>kubectl get nodes\n</code></pre> <pre><code>NAME                                                STATUS   ROLES    AGE   VERSION\nfargate-ip-10-0-17-17.us-west-2.compute.internal    Ready    &lt;none&gt;   25m   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-20-244.us-west-2.compute.internal   Ready    &lt;none&gt;   71s   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-41-143.us-west-2.compute.internal   Ready    &lt;none&gt;   25m   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-44-95.us-west-2.compute.internal    Ready    &lt;none&gt;   25m   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-45-153.us-west-2.compute.internal   Ready    &lt;none&gt;   77s   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-47-31.us-west-2.compute.internal    Ready    &lt;none&gt;   75s   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-6-175.us-west-2.compute.internal    Ready    &lt;none&gt;   25m   v1.26.3-eks-f4dc2c0\n</code></pre> </li> <li> <p>List the pods. All the pods should reach a status of <code>Running</code> after approximately 60 seconds:</p> <pre><code>kubectl get pods -A\n</code></pre> <pre><code>NAMESPACE       NAME                                            READY   STATUS    RESTARTS   AGE\napp-2048        app-2048-65bd744dfb-7g9rx                       1/1     Running   0          2m34s\napp-2048        app-2048-65bd744dfb-n1cbm                       1/1     Running   0          2m34s\napp-2048        app-2048-65bd744dfb-z4b6z                       1/1     Running   0          2m34s\nkube-system     aws-load-balancer-controller-6cbdb58654-f1skt   1/1     Running   0          26m\nkube-system     aws-load-balancer-controller-6cbdb58654-sc7dk   1/1     Running   0          26m\nkube-system     coredns-7b7bddbc85-j1bv6                        1/1     Running   0          26m\nkube-system     coredns-7b7bddbc85-rg2zq                        1/1     Running   0          26m\n</code></pre> </li> <li> <p>Validate the <code>aws-logging</code> configMap for Fargate Fluentbit was created:</p> <pre><code>kubectl -n aws-observability get configmap aws-logging -o yaml\n</code></pre> <pre><code>apiVersion: v1\ndata:\n  filters.conf: |\n    [FILTER]\n      Name parser\n      Match *\n      Key_Name log\n      Parser regex\n      Preserve_Key True\n      Reserve_Data True\n  flb_log_cw: \"true\"\n  output.conf: |\n    [OUTPUT]\n      Name cloudwatch_logs\n      Match *\n      region us-west-2\n      log_group_name /fargate-serverless/fargate-fluentbit-logs20230509014113352200000006\n      log_stream_prefix fargate-logs-\n      auto_create_group true\n  parsers.conf: |\n    [PARSER]\n      Name regex\n      Format regex\n      Regex ^(?&lt;time&gt;[^ ]+) (?&lt;stream&gt;[^ ]+) (?&lt;logtag&gt;[^ ]+) (?&lt;message&gt;.+)$\n      Time_Key time\n      Time_Format %Y-%m-%dT%H:%M:%S.%L%z\n      Time_Keep On\n      Decode_Field_As json message\nimmutable: false\nkind: ConfigMap\nmetadata:\n  creationTimestamp: \"2023-05-08T21:14:52Z\"\n  name: aws-logging\n  namespace: aws-observability\n  resourceVersion: \"1795\"\n  uid: d822bcf5-a441-4996-857e-7fb1357bc07e\n</code></pre> <p>You can also validate if the CloudWatch LogGroup was created accordingly, and LogStreams were populated:</p> <pre><code>aws logs describe-log-groups \\\n    --log-group-name-prefix \"/fargate-serverless/fargate-fluentbit\"\n</code></pre> <pre><code>{\n    \"logGroups\": [\n        {\n            \"logGroupName\": \"/fargate-serverless/fargate-fluentbit-logs20230509014113352200000006\",\n            \"creationTime\": 1683580491652,\n            \"retentionInDays\": 90,\n            \"metricFilterCount\": 0,\n            \"arn\": \"arn:aws:logs:us-west-2:111222333444:log-group:/fargate-serverless/fargate-fluentbit-logs20230509014113352200000006:*\",\n            \"storedBytes\": 0\n        }\n    ]\n}\n</code></pre> <pre><code>aws logs describe-log-streams \\\n    --log-group-name \"/fargate-serverless/fargate-fluentbit-logs20230509014113352200000006\" \\\n    --log-stream-name-prefix fargate-logs --query 'logStreams[].logStreamName'\n</code></pre> <pre><code>[\n    \"fargate-logs-flblogs.var.log.fluent-bit.log\",\n    \"fargate-logs-kube.var.log.containers.aws-load-balancer-controller-7f989fc6c-gr1sq_kube-system_aws-load-balancer-controller-feaa22b4cdaa71ecfc8355feb81d4b61ea85598a7bb57aef07667c767c6b98e4.log\",\n    \"fargate-logs-kube.var.log.containers.aws-load-balancer-controller-7f989fc6c-wzr46_kube-system_aws-load-balancer-controller-69075ea9ab3c7474eac2a1696d3a84a848a151420cd783d79aeef960b181567f.log\",\n    \"fargate-logs-kube.var.log.containers.coredns-7b7bddbc85-8cx1q_kube-system_coredns-9e4f3ab435269a566bcbaa606c02c146ad58508e67cef09fa87d5c09e4ac0088.log\",\n    \"fargate-logs-kube.var.log.containers.coredns-7b7bddbc85-g1jwp_kube-system_coredns-11016818361cd68c32bf8f0b1328f3d92a6d7b8cf5879bfe8b301f393cb011cc.log\"\n]\n</code></pre> </li> </ol>"},{"location":"patterns/fargate-serverless/#example","title":"Example","text":"<ol> <li> <p>Create an ingress resource using the AWS load balancer controller deployed, pointing to our application service:</p> <pre><code>kubectl get svc -n app-2048\n</code></pre> <pre><code>NAME       TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE\napp-2048   NodePort   172.20.33.217   &lt;none&gt;        80:32568/TCP   2m48s\n</code></pre> <pre><code>kubectl -n app-2048 create ingress app-2048 --class alb --rule=\"/*=app-2048:80\" \\\n--annotation alb.ingress.kubernetes.io/scheme=internet-facing \\\n--annotation alb.ingress.kubernetes.io/target-type=ip\n</code></pre> <pre><code>kubectl -n app-2048 get ingress\n</code></pre> <pre><code>NAME       CLASS   HOSTS   ADDRESS                                                                 PORTS   AGE\napp-2048   alb     *       k8s-app2048-app2048-6d9c5e92d6-1234567890.us-west-2.elb.amazonaws.com   80      4m9s\n</code></pre> </li> <li> <p>Open the browser to access the application via the URL address shown in the last output in the ADDRESS column.</p> <p>In our example: <code>k8s-app2048-app2048-6d9c5e92d6-1234567890.us-west-2.elb.amazonaws.com</code></p> <p>Info</p> <p>You might need to wait a few minutes, and then refresh your browser. If your Ingress isn't created after several minutes, then run this command to view the AWS Load Balancer Controller logs:</p> <pre><code>kubectl logs -n kube-system deployment.apps/aws-load-balancer-controller\n</code></pre> </li> </ol>"},{"location":"patterns/fargate-serverless/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/fully-private-cluster/","title":"Fully Private Cluster","text":""},{"location":"patterns/fully-private-cluster/#fully-private-amazon-eks-cluster","title":"Fully Private Amazon EKS Cluster","text":"<p>This pattern demonstrates an Amazon EKS cluster that does not have internet access. The private cluster must pull images from a container registry that is within in your VPC, and also must have endpoint private access enabled. This is required for nodes to register with the cluster endpoint.</p> <p>Please see this document for more details on configuring fully private EKS Clusters.</p> <p>For fully Private EKS clusters requires the following VPC endpoints to be created to communicate with AWS services. This example solution will provide these endpoints if you choose to create VPC. If you are using an existing VPC then you may need to ensure these endpoints are created.</p> <pre><code>com.amazonaws.region.aps-workspaces       - If using AWS Managed Prometheus Workspace\ncom.amazonaws.region.ssm                  - Secrets Management\ncom.amazonaws.region.ec2\ncom.amazonaws.region.ecr.api\ncom.amazonaws.region.ecr.dkr\ncom.amazonaws.region.logs                 \u2013 For CloudWatch Logs\ncom.amazonaws.region.sts                  \u2013 If using AWS Fargate or IAM roles for service accounts\ncom.amazonaws.region.elasticloadbalancing \u2013 If using Application Load Balancers\ncom.amazonaws.region.autoscaling          \u2013 If using Cluster Autoscaler\ncom.amazonaws.region.s3\n</code></pre>"},{"location":"patterns/fully-private-cluster/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/fully-private-cluster/#validate","title":"Validate","text":"<ol> <li> <p>Test by listing Nodes in in the cluster:</p> <pre><code>kubectl get nodes\n</code></pre> <pre><code>NAME                                        STATUS   ROLES    AGE     VERSION\nip-10-0-19-90.us-west-2.compute.internal    Ready    &lt;none&gt;   8m34s   v1.26.2-eks-a59e1f0\nip-10-0-44-110.us-west-2.compute.internal   Ready    &lt;none&gt;   8m36s   v1.26.2-eks-a59e1f0\nip-10-0-9-147.us-west-2.compute.internal    Ready    &lt;none&gt;   8m35s   v1.26.2-eks-a59e1f0\n</code></pre> </li> <li> <p>Test by listing all the Pods running currently. All the Pods should reach a status of <code>Running</code> after approximately 60 seconds:</p> <pre><code>kubectl get pods -A\n</code></pre> <pre><code>NAMESPACE     NAME                       READY   STATUS    RESTARTS   AGE\nkube-system   aws-node-j2n9x             1/1     Running   0          7m42s\nkube-system   aws-node-m1jlf             1/1     Running   0          7m45s\nkube-system   aws-node-q458h             1/1     Running   0          7m49s\nkube-system   coredns-6c45d94f67-495rr   1/1     Running   0          14m\nkube-system   coredns-6c45d94f67-5c8tc   1/1     Running   0          14m\nkube-system   kube-proxy-47wfh           1/1     Running   0          8m32s\nkube-system   kube-proxy-f6chz           1/1     Running   0          8m30s\nkube-system   kube-proxy-x2fkc           1/1     Running   0          8m31s\n</code></pre> </li> </ol>"},{"location":"patterns/fully-private-cluster/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/gitops-getting-started-argocd/","title":"GitOps Getting Started (ArgoCD)","text":""},{"location":"patterns/gitops-getting-started-argocd/#argocd-on-amazon-eks","title":"ArgoCD on Amazon EKS","text":"<p>This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD, employing the GitOps Bridge Pattern.</p> <p></p> <p>The GitOps Bridge Pattern enables Kubernetes administrators to utilize Infrastructure as Code (IaC) and GitOps tools for deploying Kubernetes Addons and Workloads. Addons often depend on Cloud resources that are external to the cluster. The configuration metadata for these external resources is required by the Addons' Helm charts. While IaC is used to create these cloud resources, it is not used to install the Helm charts. Instead, the IaC tool stores this metadata either within GitOps resources in the cluster or in a Git repository. The GitOps tool then extracts these metadata values and passes them to the Helm chart during the Addon installation process. This mechanism forms the bridge between IaC and GitOps, hence the term \"GitOps Bridge.\"</p> <p>Additional examples available on the GitOps Bridge Pattern:</p> <ul> <li>argocd-ingress</li> <li>aws-secrets-manager</li> <li>crossplane</li> <li>external-secrets</li> <li>multi-cluster/distributed</li> <li>multi-cluster/hub-spoke</li> <li>multi-cluster/hub-spoke-shared</li> <li>private-git</li> </ul>"},{"location":"patterns/gitops-getting-started-argocd/#prerequisites","title":"Prerequisites","text":"<p>Before you begin, make sure you have the following command line tools installed:</p> <ul> <li>git</li> <li>terraform</li> <li>kubectl</li> <li>argocd</li> </ul>"},{"location":"patterns/gitops-getting-started-argocd/#optional-fork-the-gitops-git-repositories","title":"(Optional) Fork the GitOps git repositories","text":"<p>See the appendix section Fork GitOps Repositories for more info on the terraform variables to override.</p>"},{"location":"patterns/gitops-getting-started-argocd/#deploy-the-eks-cluster","title":"Deploy the EKS Cluster","text":"<p>Initialize Terraform and deploy the EKS cluster:</p> <pre><code>terraform init\nterraform apply -target=\"module.vpc\" -auto-approve\nterraform apply -target=\"module.eks\" -auto-approve\nterraform apply -auto-approve\n</code></pre> <p>To retrieve <code>kubectl</code> config, execute the terraform output command:</p> <pre><code>terraform output -raw configure_kubectl\n</code></pre> <p>The expected output will have two lines you run in your terminal</p> <pre><code>export KUBECONFIG=\"/tmp/getting-started-gitops\"\naws eks --region us-west-2 update-kubeconfig --name getting-started-gitops\n</code></pre> <p>The first line sets the <code>KUBECONFIG</code> environment variable to a temporary file that includes the cluster name. The second line uses the <code>aws</code> CLI to populate that temporary file with the <code>kubectl</code> configuration. This approach offers the advantage of not altering your existing <code>kubectl</code> context, allowing you to work in other terminal windows without interference.</p> <p>Terraform will add GitOps Bridge Metadata to the ArgoCD secret. The annotations contain metadata for the addons' Helm charts and ArgoCD ApplicationSets.</p> <pre><code>kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster -o json | jq '.items[0].metadata.annotations'\n</code></pre> <p>The output looks like the following:</p> <pre><code>{\n  \"addons_repo_basepath\": \"argocd/\",\n  \"addons_repo_path\": \"bootstrap/control-plane/addons\",\n  \"addons_repo_revision\": \"main\",\n  \"addons_repo_url\": \"https://github.com/aws-samples/eks-blueprints-add-ons\",\n  \"aws_account_id\": \"0123456789\",\n  \"aws_cluster_name\": \"getting-started-gitops\",\n  \"aws_load_balancer_controller_iam_role_arn\": \"arn:aws:iam::0123456789:role/alb-controller\",\n  \"aws_load_balancer_controller_namespace\": \"kube-system\",\n  \"aws_load_balancer_controller_service_account\": \"aws-load-balancer-controller-sa\",\n  \"aws_region\": \"us-west-2\",\n  \"aws_vpc_id\": \"vpc-001d3f00151bbb731\",\n  \"cluster_name\": \"in-cluster\",\n  \"environment\": \"dev\",\n  \"workload_repo_basepath\": \"patterns/gitops/\",\n  \"workload_repo_path\": \"getting-started-argocd/k8s\",\n  \"workload_repo_revision\": \"main\",\n  \"workload_repo_url\": \"https://github.com/csantanapr/terraform-aws-eks-blueprints\"\n}\n</code></pre> <p>The labels offer a straightforward way to enable or disable an addon in ArgoCD for the cluster.</p> <pre><code>kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster -o json | jq '.items[0].metadata.labels' | grep -v false | jq .\n</code></pre> <p>The output looks like the following:</p> <pre><code>{\n  \"argocd.argoproj.io/secret-type\": \"cluster\",\n  \"aws_cluster_name\": \"getting-started-gitops\",\n  \"cluster_name\": \"in-cluster\",\n  \"enable_argocd\": \"true\",\n  \"enable_aws_load_balancer_controller\": \"true\",\n  \"enable_metrics_server\": \"true\",\n  \"environment\": \"dev\",\n  \"kubernetes_version\": \"1.28\"\n}\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#deploy-the-addons","title":"Deploy the Addons","text":"<p>Bootstrap the addons using ArgoCD:</p> <pre><code>kubectl apply -f bootstrap/addons.yaml\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#monitor-gitops-progress-for-addons","title":"Monitor GitOps Progress for Addons","text":"<p>Wait until all the ArgoCD applications' <code>HEALTH STATUS</code> is <code>Healthy</code>. Use <code>Ctrl+C</code> or <code>Cmd+C</code> to exit the <code>watch</code> command. ArgoCD Applications can take a couple of minutes in order to achieve the Healthy status.</p> <pre><code>kubectl get applications -n argocd -w\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                                            SYNC STATUS   HEALTH STATUS\naddon-in-cluster-argo-cd                        Synced        Healthy\naddon-in-cluster-aws-load-balancer-controller   Synced        Healthy\naddon-in-cluster-metrics-server                 Synced        Healthy\ncluster-addons                                  Synced        Healthy\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#verify-the-addons","title":"Verify the Addons","text":"<p>Verify that the addons are ready:</p> <pre><code>kubectl get deployment -n kube-system \\\n  aws-load-balancer-controller \\\n  metrics-server\nkubectl get deploy -n argocd \\\n  argo-cd-argocd-applicationset-controller \\\n  argo-cd-argocd-repo-server \\\n  argo-cd-argocd-server\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                                       READY   UP-TO-DATE   AVAILABLE   AGE\naws-load-balancer-controller               2/2     2            2           7m21s\nmetrics-server                             1/1     1            1           7m41s\nargo-cd-argocd-applicationset-controller   1/1     1            1           109m\nargo-cd-argocd-repo-server                 1/1     1            1           109m\nargo-cd-argocd-server                      1/1     1            1           109m\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#optional-access-argocd","title":"(Optional) Access ArgoCD","text":"<p>Access to the ArgoCD's UI is completely optional, if you want to do it, run the commands shown in the Terraform output as the example below:</p> <pre><code>terraform output -raw access_argocd\n</code></pre> <p>The expected output should contain the <code>kubectl</code> config followed by <code>kubectl</code> command to retrieve the URL, username, password to login into ArgoCD UI or CLI.</p> <pre><code>echo \"ArgoCD Username: admin\"\necho \"ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template=\"{{index .data.password | base64decode}}\")\"\necho \"ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\"\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#deploy-the-workloads","title":"Deploy the Workloads","text":"<p>Deploy a sample application located in k8s/game-2048.yaml using ArgoCD:</p> <pre><code>kubectl apply -f bootstrap/workloads.yaml\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#monitor-gitops-progress-for-workloads","title":"Monitor GitOps Progress for Workloads","text":"<p>Wait until all the ArgoCD applications' <code>HEALTH STATUS</code> is <code>Healthy</code>. Use <code>Ctrl+C</code> or <code>Cmd+C</code> to exit the <code>watch</code> command. ArgoCD Applications can take a couple of minutes in order to achieve the Healthy status.</p> <pre><code>watch kubectl get -n argocd applications workloads\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME        SYNC STATUS   HEALTH STATUS\nworkloads   Synced        Healthy\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#verify-the-application","title":"Verify the Application","text":"<p>Verify that the application configuration is present and the pod is running:</p> <pre><code>kubectl get -n game-2048 deployments,service,ep,ingress\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                        READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/game-2048   1/1     1            1           7h59m\n\nNAME                TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE\nservice/game-2048   ClusterIP   172.20.155.47   &lt;none&gt;        80/TCP    7h59m\n\nNAME                  ENDPOINTS       AGE\nendpoints/game-2048   10.0.13.64:80   7h59m\n\nNAME                CLASS   HOSTS   ADDRESS                              PORTS   AGE\ningress/game-2048   alb     *       k8s-&lt;&gt;.us-west-2.elb.amazonaws.com   80      7h59m\n</code></pre> <p>AWS Load Balancer can take a couple of minutes in order to be created.</p> <p>Run the following command and wait until and event for ingress <code>game-2048</code> contains <code>Successfully reconciled</code>. Use <code>Ctrl+C</code> or <code>Cmd+C</code>to exit the <code>watch</code> command.</p> <pre><code>kubectl events -n game-2048 --for ingress/game-2048 --watch\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>LAST SEEN   TYPE     REASON                   OBJECT              MESSAGE\n11m         Normal   SuccessfullyReconciled   Ingress/game-2048   Successfully reconciled\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#access-the-application-using-aws-load-balancer","title":"Access the Application using AWS Load Balancer","text":"<p>Verify the application endpoint health using <code>wget</code>:</p> <pre><code>kubectl exec -n game-2048 deploy/game-2048 -- \\\nwget -S --spider $(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>  HTTP/1.1 200 OK\n  Date: Wed, 01 Nov 2023 22:44:57 GMT\n  Content-Type: text/html\n  Content-Length: 3988\n</code></pre> <p>A success response should contain <code>HTTP/1.1 200 OK</code>.</p> <p>Retrieve the ingress URL to access the application in your local web browser.</p> <pre><code>echo \"Application URL: http://$(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\"\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#container-metrics","title":"Container Metrics","text":"<p>Check the application's CPU and memory metrics:</p> <pre><code>kubectl top pods -n game-2048\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                         CPU(cores)   MEMORY(bytes)\ngame-2048-66fb78b995-h1bjv   1m           2Mi\n</code></pre> <p>Check the CPU and memory metrics for all pods for Addons and Workloads:</p> <pre><code>kubectl top pods -A\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAMESPACE     NAME                                                        CPU(cores)   MEMORY(bytes)\nargocd        argo-cd-argocd-application-controller-0                     43m          138Mi\nargocd        argo-cd-argocd-applicationset-controller-5db688844c-79skp   1m           25Mi\nargocd        argo-cd-argocd-dex-server-cd48d7bc-x7flf                    1m           16Mi\nargocd        argo-cd-argocd-notifications-controller-7d7ccc6b9d-dg9r6    1m           17Mi\nargocd        argo-cd-argocd-redis-7f89c69877-6m2cj                       2m           3Mi\nargocd        argo-cd-argocd-repo-server-644b9b5668-m9ddg                 8m           62Mi\nargocd        argo-cd-argocd-server-57cbbd6f94-lp4wx                      2m           26Mi\ngame-2048     game-2048-66fb78b995-h1bjv                                  1m           2Mi\nkube-system   aws-load-balancer-controller-8488df87c-4nxv6                2m           26Mi\nkube-system   aws-load-balancer-controller-8488df87c-zs4p6                1m           19Mi\nkube-system   aws-node-ck6vq                                              3m           57Mi\nkube-system   aws-node-fv2sg                                              3m           56Mi\nkube-system   coredns-59754897cf-5r2xp                                    1m           13Mi\nkube-system   coredns-59754897cf-fn7jb                                    1m           13Mi\nkube-system   kube-proxy-lz2dc                                            1m           11Mi\nkube-system   kube-proxy-pd2lm                                            1m           12Mi\nkube-system   metrics-server-5b76987ff-5g1sv                              4m           17Mi\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#destroy-the-eks-cluster","title":"Destroy the EKS Cluster","text":"<p>To tear down all the resources and the EKS cluster, run the following command:</p> <pre><code>./destroy.sh\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#appendix","title":"Appendix","text":""},{"location":"patterns/gitops-getting-started-argocd/#fork-gitops-repositories","title":"Fork GitOps Repositories","text":"<p>To modify the <code>values.yaml</code> file for addons or the workload manifest files (.ie yaml), you'll need to fork two repositories: aws-samples/eks-blueprints-add-ons for addons and github.com/aws-ia/terraform-aws-eks-blueprints for workloads located in this pattern directory.</p> <p>After forking, update the following environment variables to point to your forks, replacing the default values.</p> <pre><code>export TF_VAR_gitops_addons_org=https://github.com/aws-samples\nexport TF_VAR_gitops_addons_repo=eks-blueprints-add-ons\nexport TF_VAR_gitops_addons_revision=main\n\nexport TF_VAR_gitops_workload_org=https://github.com/aws-ia\nexport TF_VAR_gitops_workload_repo=terraform-aws-eks-blueprints\nexport TF_VAR_gitops_workload_revision=main\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/","title":"GitOps Multi-Cluster Hub-Spoke Topology (ArgoCD)","text":""},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#multi-cluster-centralized-hub-spoke-topology","title":"Multi-Cluster centralized hub-spoke topology","text":"<p>This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD in a Multi-Cluster Hub-Spoke topology, employing the GitOps Bridge Pattern.</p> <p></p> <p>This example deploys ArgoCD on the Hub cluster (i.e. management/control-plane cluster). The spoke clusters are registered as remote clusters in the Hub Cluster's ArgoCD The ArgoCD on the Hub Cluster deploys addons and workloads to the spoke clusters</p> <p>Each spoke cluster gets deployed an app of apps ArgoCD Application with the name <code>workloads-${env}</code></p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#prerequisites","title":"Prerequisites","text":"<p>Before you begin, make sure you have the following command line tools installed:</p> <ul> <li>git</li> <li>terraform</li> <li>kubectl</li> <li>argocd</li> </ul>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#optional-fork-the-gitops-git-repositories","title":"(Optional) Fork the GitOps git repositories","text":"<p>See the appendix section Fork GitOps Repositories for more info on the terraform variables to override.</p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#deploy-the-hub-eks-cluster","title":"Deploy the Hub EKS Cluster","text":"<p>Change directory to <code>hub</code></p> <pre><code>cd hub\n</code></pre> <p>Initialize Terraform and deploy the EKS cluster:</p> <pre><code>terraform init\nterraform apply -target=\"module.vpc\" -auto-approve\nterraform apply -target=\"module.eks\" -auto-approve\nterraform apply -auto-approve\n</code></pre> <p>To retrieve <code>kubectl</code> config, execute the terraform output command:</p> <pre><code>terraform output -raw configure_kubectl\n</code></pre> <p>The expected output will have two lines you run in your terminal</p> <pre><code>export KUBECONFIG=\"/tmp/hub-spoke\"\naws eks --region us-west-2 update-kubeconfig --name getting-started-gitops --alias hub\n</code></pre> <p>The first line sets the <code>KUBECONFIG</code> environment variable to a temporary file that includes the cluster name. The second line uses the <code>aws</code> CLI to populate that temporary file with the <code>kubectl</code> configuration. This approach offers the advantage of not altering your existing <code>kubectl</code> context, allowing you to work in other terminal windows without interference.</p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#deploy-argocd-apps-of-applicationsets-for-addons","title":"Deploy ArgoCD Apps of ApplicationSets for Addons","text":"<p>This command verifies the initial ArgoCD installation, ArgoCD will be re-configured when the addons are deployed and configured from git. <pre><code>kubectl --context hub get all -n argocd\n</code></pre> This command creates the application set manifest to deploy the addons. <pre><code>kubectl --context hub apply -n argocd -f ../hub/bootstrap/addons.yaml\n</code></pre> The application sets defined here will then deploy addons to any spoke clusters provisioned later using Terraform</p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#monitor-gitops-progress-for-addons-on-hub-eks-cluster","title":"Monitor GitOps Progress for Addons on Hub EKS Cluster","text":"<p>Wait until all the ArgoCD applications' <code>HEALTH STATUS</code> is <code>Healthy</code>. Use <code>Ctrl+C</code> or <code>Cmd+C</code> to exit the <code>watch</code> command. ArgoCD Applications can take a couple of minutes in order to achieve the Healthy status.</p> <pre><code>kubectl --context hub get applications -n argocd -w\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                                            SYNC STATUS   HEALTH STATUS\naddon-in-cluster-argo-cd                        Synced        Healthy\naddon-in-cluster-aws-load-balancer-controller   Synced        Healthy\naddon-in-cluster-metrics-server                 Synced        Healthy\ncluster-addons                                  Synced        Healthy\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#optional-access-argocd","title":"(Optional) Access ArgoCD","text":"<p>Access to the ArgoCD's UI is completely optional, if you want to do it, run the commands shown in the Terraform output as the example below:</p> <pre><code>terraform output -raw access_argocd\n</code></pre> <p>The expected output should contain the <code>kubectl</code> config followed by <code>kubectl</code> command to retrieve the URL, username, password to login into ArgoCD UI or CLI.</p> <pre><code>echo \"ArgoCD Username: admin\"\necho \"ArgoCD Password: $(kubectl --context hub get secrets argocd-initial-admin-secret -n argocd --template=\"{{index .data.password | base64decode}}\")\"\necho \"ArgoCD URL: https://$(kubectl --context hub get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\"\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#deploy-the-spoke-eks-cluster","title":"Deploy the Spoke EKS Cluster","text":"<p>Use the <code>deploy.sh</code> script to create terraform workspace, initialize Terraform, and deploy the EKS clusters:</p> <p>You may want to create few spoke environments to validate multi-cluster hub spoke to avoid quota limits</p> <pre><code>cd ../spokes\n./deploy.sh dev\n./deploy.sh staging\n./deploy.sh prod\n</code></pre> <p>Each environment uses a Terraform workspace</p> <p>To retrieve <code>kubectl</code> config, execute the terraform output command:</p> <pre><code>terraform workspace select dev\nterraform output -raw configure_kubectl\n</code></pre> <pre><code>terraform workspace select staging\nterraform output -raw configure_kubectl\n</code></pre> <pre><code>terraform workspace select prod\nterraform output -raw configure_kubectl\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#verify-argocd-cluster-secret-for-spokes-have-the-correct-iam-role-to-be-assume-by-hub-cluster","title":"Verify ArgoCD Cluster Secret for Spokes have the correct IAM Role to be assume by Hub Cluster","text":"<pre><code>for i in dev staging prod ; do echo $i &amp;&amp; kubectl --context hub get secret -n argocd spoke-$i --template='{{index .data.config | base64decode}}' ; done\n</code></pre> <p>The output have a section <code>awsAuthConfig</code> with the <code>clusterName</code> and the <code>roleARN</code> that has write access to the spoke cluster</p> <pre><code>{\n  \"tlsClientConfig\": {\n    \"insecure\": false,\n    \"caData\" : \"LS0tL....\"\n  },\n  \"awsAuthConfig\" : {\n    \"clusterName\": \"hub-spoke-dev\",\n    \"roleARN\": \"arn:aws:iam::0123456789:role/hub-spoke-dev-argocd-spoke\"\n  }\n}\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#verify-the-addons-on-spoke-clusters","title":"Verify the Addons on Spoke Clusters","text":"<p>The addons on the spoke clusters are deployed using the Application Sets created on the EKS Hub Cluster. Verify that the addons are ready:</p> <pre><code>for i in dev staging prod ; do echo $i &amp;&amp; kubectl --context $i get deployment -n kube-system ; done\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#deploy-the-sample-application-to-eks-spoke-clusters","title":"Deploy the sample application to EKS Spoke Clusters","text":"<p>This command will deploy the application using kubectl to all clusters connected to the hub cluster, using the manifest files in ./hub/bootstrap/workloads.yaml. <pre><code>kubectl --context hub apply -n argocd -f ../hub/bootstrap/workloads.yaml\n</code></pre></p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#monitor-gitops-progress-for-workloads-from-hub-cluster-run-on-hub-cluster-context","title":"Monitor GitOps Progress for Workloads from Hub Cluster (run on Hub Cluster context)","text":"<p>Watch until all the Workloads ArgoCD Applications are <code>Healthy</code></p> <pre><code>kubectl --context hub get -n argocd applications -w\n</code></pre> <p>Wait until the ArgoCD Applications <code>HEALTH STATUS</code> is <code>Healthy</code>. Crl+C to exit the <code>watch</code> command</p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#verify-the-application","title":"Verify the Application","text":"<p>Verify that the application configuration is present and the pod is running:</p> <pre><code>for i in dev staging prod ; do echo $i &amp;&amp; kubectl --context $i get all -n workload ; done\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#container-metrics","title":"Container Metrics","text":"<p>Check the application's CPU and memory metrics:</p> <pre><code>for i in dev staging prod ; do echo $i &amp;&amp; kubectl --context $i top pods -n workload ; done\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#destroy-the-spoke-eks-clusters","title":"Destroy the Spoke EKS Clusters","text":"<p>To tear down all the resources and the EKS cluster, run the following command:</p> <pre><code>./destroy.sh dev\n./destroy.sh staging\n./destroy.sh prod\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#destroy-the-hub-eks-clusters","title":"Destroy the Hub EKS Clusters","text":"<p>To tear down all the resources and the EKS cluster, run the following command: Destroy Hub Clusters</p> <pre><code>cd ../hub\n./destroy.sh\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#appendix","title":"Appendix","text":""},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#fork-gitops-repositories","title":"Fork GitOps Repositories","text":"<p>To modify the <code>values.yaml</code> file or the helm chart version for addons, you'll need to fork the repository aws-samples/eks-blueprints-add-ons.</p> <p>After forking, update the following environment variables to point to your forks, replacing the default values.</p> <pre><code>export TF_VAR_gitops_addons_org=https://github.com/aws-samples\nexport TF_VAR_gitops_addons_repo=eks-blueprints-add-ons\nexport TF_VAR_gitops_addons_revision=main\n</code></pre>"},{"location":"patterns/ipv6-eks-cluster/","title":"IPv6 Networking","text":""},{"location":"patterns/ipv6-eks-cluster/#amazon-eks-cluster-w-ipv6-networking","title":"Amazon EKS Cluster w/ IPv6 Networking","text":"<p>This pattern demonstrates an EKS cluster that utilizes IPv6 networking.</p>"},{"location":"patterns/ipv6-eks-cluster/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/ipv6-eks-cluster/#validate","title":"Validate","text":"<ol> <li> <p>Test by listing all the pods running currently; the <code>IP</code> should be an IPv6 address.</p> <pre><code>kubectl get pods -A -o wide\n</code></pre> <pre><code># Output should look like below\nNAMESPACE     NAME                       READY   STATUS    RESTARTS   AGE     IP                                       NODE                                        NOMINATED NODE   READINESS GATES\nkube-system   aws-node-1hd2s             1/1     Running   0          3m5s    2600:1f13:6c4:a703:ecf8:3ac1:76b0:9303   ip-10-0-10-183.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   aws-node-2mdg             1/1     Running   0          3m21s   2600:1f13:6c4:a705:a929:f8d4:9350:1b20   ip-10-0-12-188.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   coredns-799c5565b4-6xrc   1/1     Running   0          10m     2600:1f13:6c4:a705:1bda::                ip-10-0-12-188.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   coredns-799c5565b4-fjq4q   1/1     Running   0          10m     2600:1f13:6c4:a705:1bda::1               ip-10-0-12-188.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   kube-proxy-58tp7           1/1     Running   0          4m25s   2600:1f13:6c4:a703:ecf8:3ac1:76b0:9303   ip-10-0-10-183.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   kube-proxy-3kgw           1/1     Running   0          4m25s   2600:1f13:6c4:a705:a929:f8d4:9350:1b20   ip-10-0-12-188.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\n</code></pre> </li> <li> <p>Test by listing all the nodes running currently; the <code>INTERNAL-IP</code> should be an IPv6 address.</p> <pre><code>kubectl nodes -A -o wide\n</code></pre> <pre><code># Output should look like below\nNAME                                        STATUS   ROLES    AGE     VERSION               INTERNAL-IP                              EXTERNAL-IP   OS-IMAGE         KERNEL-VERSION                 CONTAINER-RUNTIME\nip-10-0-10-183.us-west-2.compute.internal   Ready    &lt;none&gt;   4m57s   v1.24.7-eks-fb459a0   2600:1f13:6c4:a703:ecf8:3ac1:76b0:9303   &lt;none&gt;        Amazon Linux 2   5.4.226-129.415.amzn2.x86_64   containerd://1.6.6\nip-10-0-12-188.us-west-2.compute.internal   Ready    &lt;none&gt;   4m57s   v1.24.7-eks-fb459a0   2600:1f13:6c4:a705:a929:f8d4:9350:1b20   &lt;none&gt;        Amazon Linux 2   5.4.226-129.415.amzn2.x86_64   containerd://1.6.6\n</code></pre> </li> </ol>"},{"location":"patterns/ipv6-eks-cluster/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/istio/","title":"Istio","text":""},{"location":"patterns/istio/#amazon-eks-cluster-w-istio","title":"Amazon EKS Cluster w/ Istio","text":"<p>This example shows how to provision an EKS cluster with Istio.</p> <ul> <li>Deploy EKS Cluster with one managed node group in an VPC</li> <li>Add node_security_group rules for port access required for Istio communication</li> <li>Install Istio using Helm resources in Terraform</li> <li>Install Istio Ingress Gateway using Helm resources in Terraform</li> <li>This step deploys a Service of type <code>LoadBalancer</code> that creates an AWS Network Load Balancer.</li> <li>Deploy/Validate Istio communication using sample application</li> </ul> <p>Refer to the documentation on Istio concepts.</p>"},{"location":"patterns/istio/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and run the following command to deploy this pattern.</p> <pre><code>terraform init\nterraform apply --auto-approve\n</code></pre> <p>Once the resources have been provisioned, you will need to replace the <code>istio-ingress</code> pods due to a <code>istiod</code> dependency issue. Use the following command to perform a rolling restart of the <code>istio-ingress</code> pods:</p> <pre><code>kubectl rollout restart deployment istio-ingress -n istio-ingress\n</code></pre>"},{"location":"patterns/istio/#observability-add-ons","title":"Observability Add-ons","text":"<p>Use the following code snippet to add the Istio Observability Add-ons on the EKS cluster with deployed Istio.</p> <pre><code>for ADDON in kiali jaeger prometheus grafana\ndo\n    ADDON_URL=\"https://raw.githubusercontent.com/istio/istio/release-1.20/samples/addons/$ADDON.yaml\"\n    kubectl apply -f $ADDON_URL\ndone\n</code></pre>"},{"location":"patterns/istio/#validate","title":"Validate","text":"<ol> <li> <p>List out all pods and services in the <code>istio-system</code> namespace:</p> <pre><code>kubectl get pods,svc -n istio-system\nkubectl get pods,svc -n istio-ingress\n</code></pre> <pre><code>NAME                             READY   STATUS    RESTARTS   AGE\npod/grafana-7d4f5589fb-4xj9m     1/1     Running   0          4m14s\npod/istiod-ff577f8b8-c8ssk       1/1     Running   0          4m40s\npod/jaeger-58c79c85cd-n7bkx      1/1     Running   0          4m14s\npod/kiali-749d76d7bb-8kjg7       1/1     Running   0          4m14s\npod/prometheus-5d5d6d6fc-s1txl   2/2     Running   0          4m15s\n\nNAME                       TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                 AGE\nservice/grafana            ClusterIP   172.20.141.12    &lt;none&gt;        3000/TCP                                4m14s\nservice/istiod             ClusterIP   172.20.172.70    &lt;none&gt;        15010/TCP,15012/TCP,443/TCP,15014/TCP   4m40s\nservice/jaeger-collector   ClusterIP   172.20.223.28    &lt;none&gt;        14268/TCP,14250/TCP,9411/TCP            4m15s\nservice/kiali              ClusterIP   172.20.182.231   &lt;none&gt;        20001/TCP,9090/TCP                      4m15s\nservice/prometheus         ClusterIP   172.20.89.64     &lt;none&gt;        9090/TCP                                4m14s\nservice/tracing            ClusterIP   172.20.253.201   &lt;none&gt;        80/TCP,16685/TCP                        4m14s\nservice/zipkin             ClusterIP   172.20.221.157   &lt;none&gt;        9411/TCP                                4m15s\n\nNAME                                 READY   STATUS    RESTARTS   AGE\npod/istio-ingress-6f7c5dffd8-g1szr   1/1     Running   0          4m28s\n\nNAME                    TYPE           CLUSTER-IP      EXTERNAL-IP                                                                     PORT(S)                                      AGE\nservice/istio-ingress   LoadBalancer   172.20.104.27   k8s-istioing-istioing-844c89b6c2-875b8c9a4b4e9365.elb.us-west-2.amazonaws.com   15021:32760/TCP,80:31496/TCP,443:32534/TCP   4m28s\n</code></pre> </li> <li> <p>Verify all the Helm releases installed in the <code>istio-system</code> and <code>istio-ingress</code> namespaces:</p> <pre><code>helm list -n istio-system\n</code></pre> <pre><code>NAME           NAMESPACE    REVISION UPDATED                              STATUS   CHART          APP VERSION\nistio-base    istio-system 1        2023-07-19 11:05:41.599921 -0700 PDT deployed base-1.18.1    1.18.1\nistiod        istio-system 1        2023-07-19 11:05:48.087616 -0700 PDT deployed istiod-1.18.1  1.18.1\n</code></pre> <pre><code>helm list -n istio-ingress\n</code></pre> <pre><code>NAME           NAMESPACE    REVISION UPDATED                              STATUS   CHART          APP VERSION\nistio-ingress istio-ingress 1        2023-07-19 11:06:03.41609 -0700 PDT  deployed gateway-1.18.1 1.18.1\n</code></pre> </li> </ol>"},{"location":"patterns/istio/#observability-add-ons_1","title":"Observability Add-ons","text":"<p>Validate the setup of the observability add-ons by running the following commands and accessing each of the service endpoints using this URL of the form http://localhost:\\ where <code>&lt;port&gt;</code> is one of the port number for the corresponding service. <pre><code># Visualize Istio Mesh console using Kiali\nkubectl port-forward svc/kiali 20001:20001 -n istio-system\n\n# Get to the Prometheus UI\nkubectl port-forward svc/prometheus 9090:9090 -n istio-system\n\n# Visualize metrics in using Grafana\nkubectl port-forward svc/grafana 3000:3000 -n istio-system\n\n# Visualize application traces via Jaeger\nkubectl port-forward svc/jaeger 16686:16686 -n istio-system\n</code></pre>"},{"location":"patterns/istio/#example","title":"Example","text":"<ol> <li> <p>Create the <code>sample</code> namespace and enable the sidecar injection on it</p> <pre><code>kubectl create namespace sample\nkubectl label namespace sample istio-injection=enabled\n</code></pre> <pre><code>namespace/sample created\nnamespace/sample labeled\n</code></pre> </li> <li> <p>Deploy <code>helloworld</code> app</p> <pre><code>cat &lt;&lt;EOF &gt; helloworld.yaml\napiVersion: v1\nkind: Service\nmetadata:\n  name: helloworld\n  labels:\n    app: helloworld\n    service: helloworld\nspec:\n  ports:\n  - port: 5000\n    name: http\n  selector:\n    app: helloworld\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: helloworld-v1\n  labels:\n    app: helloworld\n    version: v1\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app: helloworld\n      version: v1\n  template:\n    metadata:\n      labels:\n        app: helloworld\n        version: v1\n    spec:\n      containers:\n      - name: helloworld\n        image: docker.io/istio/examples-helloworld-v1\n        resources:\n          requests:\n            cpu: \"100m\"\n        imagePullPolicy: IfNotPresent #Always\n        ports:\n        - containerPort: 5000\nEOF\n\nkubectl apply -f helloworld.yaml -n sample\n</code></pre> <pre><code>service/helloworld created\ndeployment.apps/helloworld-v1 created\n</code></pre> </li> <li> <p>Deploy <code>sleep</code> app that we will use to connect to <code>helloworld</code> app</p> <pre><code>cat &lt;&lt;EOF &gt; sleep.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: sleep\n---\napiVersion: v1\nkind: Service\nmetadata:\n  name: sleep\n  labels:\n    app: sleep\n    service: sleep\nspec:\n  ports:\n  - port: 80\n    name: http\n  selector:\n    app: sleep\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: sleep\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app: sleep\n  template:\n    metadata:\n      labels:\n        app: sleep\n    spec:\n      terminationGracePeriodSeconds: 0\n      serviceAccountName: sleep\n      containers:\n      - name: sleep\n        image: curlimages/curl\n        command: [\"/bin/sleep\", \"infinity\"]\n        imagePullPolicy: IfNotPresent\n        volumeMounts:\n        - mountPath: /etc/sleep/tls\n          name: secret-volume\n      volumes:\n      - name: secret-volume\n        secret:\n          secretName: sleep-secret\n          optional: true\nEOF\n\nkubectl apply -f sleep.yaml -n sample\n</code></pre> <pre><code>serviceaccount/sleep created\nservice/sleep created\ndeployment.apps/sleep created\n</code></pre> </li> <li> <p>Check all the pods in the <code>sample</code> namespace</p> <pre><code>kubectl get pods -n sample\n</code></pre> <pre><code>NAME                           READY   STATUS    RESTARTS   AGE\nhelloworld-v1-b6c45f55-bx2xk   2/2     Running   0          50s\nsleep-9454cc476-p2zxr          2/2     Running   0          15s\n</code></pre> </li> <li> <p>Connect to <code>helloworld</code> app from <code>sleep</code> app and verify if the connection uses envoy proxy</p> <pre><code>kubectl exec -n sample -c sleep \\\n    \"$(kubectl get pod -n sample -l \\\n    app=sleep -o jsonpath='{.items[0].metadata.name}')\" \\\n    -- curl -v helloworld.sample:5000/hello\n</code></pre> <pre><code>* processing: helloworld.sample:5000/hello\n...\n* Connection #0 to host helloworld.sample left intact\n</code></pre> </li> </ol>"},{"location":"patterns/istio/#destroy","title":"Destroy","text":"<p>The AWS Load Balancer Controller add-on asynchronously reconciles resource deletions. During stack destruction, the istio ingress resource and the load balancer controller add-on are deleted in quick succession, preventing the removal of some of the AWS resources associated with the ingress gateway load balancer like, the frontend and the backend security groups. This causes the final <code>terraform destroy -auto-approve</code> command to timeout and fail with VPC dependency errors like below:</p> <pre><code>\u2502 Error: deleting EC2 VPC (vpc-XXXX): operation error EC2: DeleteVpc, https response error StatusCode: 400, RequestID: XXXXX-XXXX-XXXX-XXXX-XXXXXX, api error DependencyViolation: The vpc 'vpc-XXXX' has dependencies and cannot be deleted.\n</code></pre> <p>A possible workaround is to manually uninstall the <code>istio-ingress</code> helm chart.</p> <pre><code>terraform destroy -target='module.eks_blueprints_addons.helm_release.this[\"istio-ingress\"]' -auto-approve\n</code></pre> <p>Once the chart is uninstalled move on to destroy the stack.</p> <pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/karpenter/","title":"Karpenter","text":""},{"location":"patterns/karpenter/#karpenter","title":"Karpenter","text":"<p>This pattern demonstrates how to provision Karpenter on a serverless cluster (serverless data plane) using Fargate Profiles.</p>"},{"location":"patterns/karpenter/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/karpenter/#validate","title":"Validate","text":"<ol> <li> <p>Test by listing the nodes in the cluster. You should see four Fargate nodes in the cluster:</p> <pre><code>kubectl get nodes\n\nNAME                                                STATUS   ROLES    AGE     VERSION\nfargate-ip-10-0-11-195.us-west-2.compute.internal   Ready    &lt;none&gt;   5m20s   v1.28.2-eks-f8587cb\nfargate-ip-10-0-27-183.us-west-2.compute.internal   Ready    &lt;none&gt;   5m2s    v1.28.2-eks-f8587cb\nfargate-ip-10-0-4-169.us-west-2.compute.internal    Ready    &lt;none&gt;   5m3s    v1.28.2-eks-f8587cb\nfargate-ip-10-0-44-106.us-west-2.compute.internal   Ready    &lt;none&gt;   5m12s   v1.28.2-eks-f8587cb\n</code></pre> </li> <li> <p>Provision the Karpenter <code>EC2NodeClass</code> and <code>NodePool</code> resources which provide Karpenter the necessary configurations to provision EC2 resources:</p> <pre><code>kubectl apply -f karpenter.yaml\n</code></pre> </li> <li> <p>Once the Karpenter resources are in place, Karpenter will provision the necessary EC2 resources to satisfy any pending pods in the scheduler's queue. You can demonstrate this with the example deployment provided. First deploy the example deployment which has the initial number replicas set to 0:</p> <pre><code>kubectl apply -f example.yaml\n</code></pre> </li> <li> <p>When you scale the example deployment, you should see Karpenter respond by quickly provisioning EC2 resources to satisfy those pending pod requests:</p> <pre><code>kubectl scale deployment inflate --replicas=3\n</code></pre> </li> <li> <p>Listing the nodes should now show some EC2 compute that Karpenter has created for the example deployment:</p> <pre><code>kubectl get nodes\n\nNAME                                                STATUS   ROLES    AGE   VERSION\nfargate-ip-10-0-11-195.us-west-2.compute.internal   Ready    &lt;none&gt;   13m   v1.28.2-eks-f8587cb\nfargate-ip-10-0-27-183.us-west-2.compute.internal   Ready    &lt;none&gt;   12m   v1.28.2-eks-f8587cb\nfargate-ip-10-0-4-169.us-west-2.compute.internal    Ready    &lt;none&gt;   12m   v1.28.2-eks-f8587cb\nfargate-ip-10-0-44-106.us-west-2.compute.internal   Ready    &lt;none&gt;   13m   v1.28.2-eks-f8587cb\nip-10-0-32-199.us-west-2.compute.internal           Ready    &lt;none&gt;   29s   v1.28.2-eks-a5df82a # &lt;== EC2 created by Karpenter\n</code></pre> </li> </ol>"},{"location":"patterns/karpenter/#destroy","title":"Destroy","text":"<p>Scale down the deployment to de-provision Karpenter created resources first:</p> <pre><code>kubectl delete -f example.yaml\n</code></pre> <pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/kubecost/","title":"Kubecost","text":""},{"location":"patterns/kubecost/#kubecost-with-aws-cloud-billing-integration","title":"Kubecost with AWS Cloud Billing Integration","text":"<p>This pattern demonstrates how to install and configure Kubecost with AWS CUR report. The terraform code was created following the official Kubecost documentation for aws cloud billing integration.</p>"},{"location":"patterns/kubecost/#prerequisites","title":"Prerequisites","text":"<p>You need a valid Kubecost token. To generate one, follow the instructions here.</p>"},{"location":"patterns/kubecost/#deploy","title":"Deploy","text":"<pre><code>terraform init\nterraform apply -target=\"module.vpc\" -auto-approve\nterraform apply -target=\"module.eks\" -auto-approve\nterraform apply -auto-approve --var=\"kubecost_token=&lt;your-kubecost-token&gt;\"\n</code></pre> <p>Once all of the resources have successfully been provisioned, the following command can be used to update the <code>kubeconfig</code> on your local machine and allow you to interact with your EKS Cluster using <code>kubectl</code>.</p> <pre><code>aws eks --region &lt;REGION&gt; update-kubeconfig --name &lt;CLUSTER_NAME&gt;\n</code></pre> <p>This command will create a S3 bucket with prefix <code>kubecost-</code> and a Cost and Usage Report (CUR). Within 24h The CUR will generate a CloudFormation template file called <code>crawler-cfn.yml</code> in the S3 bucket. Once that file is generated, navigate to:</p> <pre><code>cd run-me-in-24h/\n</code></pre> <p>To download and apply the CloudFormation template, run:</p> <pre><code>terraform init\nterraform apply --auto-approve\n</code></pre>"},{"location":"patterns/kubecost/#kubecost-ui","title":"Kubecost UI","text":"<p>To access the Kubecost UI run:</p> <pre><code>echo http://$(kubectl -n kubecost get svc cost-analyzer-cost-analyzer \\\n    -o jsonpath='{.status.loadBalancer.ingress[0].hostname}'):9090/\n</code></pre> <p>and then navigate to the output URL.</p> <p>Navigate to Settings -&gt; Diagnostics -&gt; View Full Diagnostics</p> <p>Expected result:</p> <p>Note</p> <p>Spot Data Feed is included in Savings Plan, Reserved Instance, and Out-Of-Cluster.</p>"},{"location":"patterns/kubecost/#destroy","title":"Destroy","text":"<p>First destroy the CloudFormation template:</p> <pre><code>cd run-me-in-24h/\nterraform destroy --auto-approve\ncd ..\nterraform destroy -target=\"module.eks_blueprints_addon\" --var=\"kubecost_token=&lt;your-kubecost-token&gt;\" -auto-approve\nterraform destroy -target=\"module.eks_blueprints_addons\"\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre>"},{"location":"patterns/ml-capacity-block/","title":"ML Capacity Block Reservation (CBR)","text":""},{"location":"patterns/ml-capacity-block/#eks-w-ml-capacity-block-reservation-cbr","title":"EKS w/ ML Capacity Block Reservation (CBR)","text":"<p>This pattern demonstrates how to consume/utilize ML capacity block reservations (CBR) with Amazon EKS. The solution is comprised of primarily 2 components:</p> <p>Warning</p> <p>The use of self-managed node group(s) are required at this time to support capacity block reservations within EKS. This pattern will be updated to demonstrate EKS managed node groups once support has been implemented by the EKS service.</p> <ol> <li> <p>The self-managed node group that will utilize the CBR should have the subnets provided to it restricted to the availability zone where the CBR has been allocated. For example - if the CBR is allocated to <code>us-west-2b</code>, the node group should only have subnet IDs provided to it that reside in <code>us-west-2b</code>. If the subnets that reside in other AZs are provided, its possible to encounter an error such as <code>InvalidParameterException: The following supplied instance types do not exist ...</code>. It is not guaranteed that this error will always be shown, and may appear random since the underlying autoscaling group(s) will provision nodes into different AZs at random. It will only occur when the underlying autoscaling group tries to provision instances into an AZ where capacity is not allocated and there is insufficient on-demand capacity for the desired instance type.</p> </li> <li> <p>The launch template utilized should specify the <code>instance_market_options</code> and <code>capacity_reservation_specification</code> arguments. This is how the CBR is utilized by the node group (i.e. - tells the autoscaling group to launch instances utilizing provided capacity reservation).</p> </li> </ol> <p>Links:</p> <ul> <li>EKS - Capacity Blocks for ML</li> <li>EC2 - Capacity Blocks for ML</li> </ul>"},{"location":"patterns/ml-capacity-block/#code","title":"Code","text":"<pre><code>################################################################################\n# Required Input\n################################################################################\n\n# See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/capacity-blocks-using.html\n# on how to obtain a ML capacity block reservation. Once acquired, you can provide\n# the reservation ID through this input to deploy the pattern\nvariable \"capacity_reservation_id\" {\n  description = \"The ID of the ML capacity block reservation to use for the node group\"\n  type        = string\n}\n\n################################################################################\n# Cluster\n################################################################################\n\nmodule \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n  version = \"~&gt; 20.9\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.29\"\n\n  # Give the Terraform identity admin access to the cluster\n  # which will allow it to deploy resources into the cluster\n  enable_cluster_creator_admin_permissions = true\n  cluster_endpoint_public_access           = true\n\n  cluster_addons = {\n    coredns    = {}\n    kube-proxy = {}\n    vpc-cni    = {}\n  }\n\n  # Add security group rules on the node group security group to\n  # allow EFA traffic\n  enable_efa_support = true\n\n  vpc_id     = module.vpc.vpc_id\n  subnet_ids = module.vpc.private_subnets\n\n  eks_managed_node_groups = {\n    # This node group is for core addons such as CoreDNS\n    default = {\n      instance_types = [\"m5.large\"]\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 2\n    }\n  }\n\n  # Note: ML capacity block reservations are only supported\n  # on self-managed node groups at this time\n  self_managed_node_groups = {\n    odcr = {\n      # The EKS AL2 GPU AMI provides all of the necessary components\n      # for accelerated workloads w/ EFA\n      ami_type      = \"AL2_x86_64_GPU\"\n      instance_type = \"p5.48xlarge\"\n\n      pre_bootstrap_user_data = &lt;&lt;-EOT\n        # Mount instance store volumes in RAID-0 for kubelet and containerd\n        # https://github.com/awslabs/amazon-eks-ami/blob/master/doc/USER_GUIDE.md#raid-0-for-kubelet-and-containerd-raid0\n        /bin/setup-local-disks raid0\n\n        # Ensure only GPU workloads are scheduled on this node group\n        export KUBELET_EXTRA_ARGS='--node-labels=vpc.amazonaws.com/efa.present=true,nvidia.com/gpu.present=true \\\n          --register-with-taints=nvidia.com/gpu=true:NoSchedule'\n\n      EOT\n\n      min_size     = 2\n      max_size     = 2\n      desired_size = 2\n\n      # This will:\n      # 1. Create a placement group to place the instances close to one another\n      # 2. Ignore subnets that reside in AZs that do not support the instance type\n      # 3. Expose all of the available EFA interfaces on the launch template\n      enable_efa_support = true\n\n      # ML capacity block reservation\n      instance_market_options = {\n        market_type = \"capacity-block\"\n      }\n      capacity_reservation_specification = {\n        capacity_reservation_target = {\n          capacity_reservation_id = var.capacity_reservation_id\n        }\n      }\n    }\n  }\n\n  tags = local.tags\n}\n\n################################################################################\n# Helm charts\n################################################################################\n\nresource \"helm_release\" \"nvidia_device_plugin\" {\n  name             = \"nvidia-device-plugin\"\n  repository       = \"https://nvidia.github.io/k8s-device-plugin\"\n  chart            = \"nvidia-device-plugin\"\n  version          = \"0.14.5\"\n  namespace        = \"nvidia-device-plugin\"\n  create_namespace = true\n  wait             = false\n\n  values = [\n    &lt;&lt;-EOT\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n            - matchExpressions:\n              - key: 'nvidia.com/gpu.present'\n                operator: In\n                values:\n                - 'true'\n    EOT\n  ]\n}\n\nresource \"helm_release\" \"aws_efa_device_plugin\" {\n  name       = \"aws-efa-k8s-device-plugin\"\n  repository = \"https://aws.github.io/eks-charts\"\n  chart      = \"aws-efa-k8s-device-plugin\"\n  version    = \"v0.4.4\"\n  namespace  = \"kube-system\"\n  wait       = false\n\n  values = [\n    &lt;&lt;-EOT\n      nodeSelector:\n        vpc.amazonaws.com/efa.present: 'true'\n      tolerations:\n        - key: nvidia.com/gpu\n          operator: Exists\n          effect: NoSchedule\n    EOT\n  ]\n}\n</code></pre>"},{"location":"patterns/ml-capacity-block/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/ml-capacity-block/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/multi-tenancy-with-teams/","title":"Multi-Tenancy w/ Teams","text":""},{"location":"patterns/multi-tenancy-with-teams/#multi-tenancy-w-teams","title":"Multi-Tenancy w/ Teams","text":"<p>This pattern demonstrates how to provision and configure a multi-tenancy Amazon EKS cluster with safeguards for resource consumption and namespace isolation.</p> <p>This example solution provides:</p> <ul> <li>Two development teams - <code>team-red</code> and <code>team-blue</code> - isolated to their respective namespaces</li> <li>An admin team with privileged access to the cluster (<code>team-admin</code>)</li> </ul>"},{"location":"patterns/multi-tenancy-with-teams/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/multi-tenancy-with-teams/#validate","title":"Validate","text":"<p>TODO</p> <p>Add in validation steps</p>"},{"location":"patterns/multi-tenancy-with-teams/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/nvidia-gpu-efa/","title":"NVIDIA GPUs with EFA","text":""},{"location":"patterns/nvidia-gpu-efa/#eks-cluster-w-nvidia-gpus-and-efa-for-machine-learning","title":"EKS Cluster w/ NVIDIA GPUs and EFA for Machine Learning","text":"<p>This pattern demonstrates an Amazon EKS Cluster with an EFA-enabled nodegroup that utilizes <code>p5.48xlarge</code> instances with H100 NVIDIA GPUs used in distributed, multi-node machine learning workloads.</p> <p>The following components are demonstrated in this pattern:</p> <ul> <li>A \"default\" node group that supports addons and components that do not require GPUs nor EFA devices. Any pods that do not tolerate the taints of the GPU node group will be scheduled on instances within this node group.</li> <li>A node group of <code>p5.48xlarge</code> instances with</li> <li>all x32 EFA network interfaces enabled</li> <li>provisioned within a placement group so that the instances are provisioned close to one another in a single availability zone that supports the instance type.</li> <li>a common NVIDIA taint of <code>\"nvidia.com/gpu:NoSchedule\"</code> to ensure only the intended applications are allowed to run on the nodes created</li> <li>two labels to identify that this nodegroup supports NVIDIA GPUs and EFA devices and allow pods to use node selectors with these labels</li> <li>the NVME instance store volumes are mounted in a RAID-0 array to provide a single, large, high-performance storage volume for the GPU workloads</li> <li>kubelet and containerd are configured to utilize the RAID-0 volume, allowing kubelet to discover the additional storage as ephemeral storage that can be utilized by pods</li> <li>A Helm chart deployment for the NVIDIA device plugin to expose and mount the GPUs provided by the instances to the pods that request them</li> <li>A Helm chart deployment for the EFA device plugin to expose and mount the EFA network interfaces provided by the instances to the pods that request them. Since the EFA network interfaces are only found on the instances that provide NVIDIA GPUs in this pattern, we do not apply an additional taint for the EFA network interfaces to avoid over-constraining.</li> </ul>"},{"location":"patterns/nvidia-gpu-efa/#code","title":"Code","text":"<pre><code>################################################################################\n# Cluster\n################################################################################\n\nmodule \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n  version = \"~&gt; 20.9\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.29\"\n\n  # Give the Terraform identity admin access to the cluster\n  # which will allow it to deploy resources into the cluster\n  enable_cluster_creator_admin_permissions = true\n  cluster_endpoint_public_access           = true\n\n  cluster_addons = {\n    coredns    = {}\n    kube-proxy = {}\n    vpc-cni    = {}\n  }\n\n  # Add security group rules on the node group security group to\n  # allow EFA traffic\n  enable_efa_support = true\n\n  vpc_id     = module.vpc.vpc_id\n  subnet_ids = module.vpc.private_subnets\n\n  eks_managed_node_groups = {\n    nvidia-efa = {\n      # The EKS AL2 GPU AMI provides all of the necessary components\n      # for accelerated workloads w/ EFA\n      ami_type       = \"AL2_x86_64_GPU\"\n      instance_types = [\"p5.48xlarge\"]\n\n      pre_bootstrap_user_data = &lt;&lt;-EOT\n        #!/usr/bin/env bash\n\n        # Mount instance store volumes in RAID-0 for kubelet and containerd\n        # https://github.com/awslabs/amazon-eks-ami/blob/master/doc/USER_GUIDE.md#raid-0-for-kubelet-and-containerd-raid0\n        /bin/setup-local-disks raid0\n      EOT\n\n      min_size     = 2\n      max_size     = 2\n      desired_size = 2\n\n      # This will:\n      # 1. Create a placement group to place the instances close to one another\n      # 2. Ignore subnets that reside in AZs that do not support the instance type\n      # 3. Expose all of the available EFA interfaces on the launch template\n      enable_efa_support = true\n\n      labels = {\n        \"vpc.amazonaws.com/efa.present\" = \"true\"\n        \"nvidia.com/gpu.present\"        = \"true\"\n      }\n\n      taints = {\n        # Ensure only GPU workloads are scheduled on this node group\n        gpu = {\n          key    = \"nvidia.com/gpu\"\n          value  = \"true\"\n          effect = \"NO_SCHEDULE\"\n        }\n      }\n    }\n\n    # This node group is for core addons such as CoreDNS\n    default = {\n      instance_types = [\"m5.large\"]\n\n      # Default AMI has only 8GB of storage\n      block_device_mappings = {\n        xvda = {\n          device_name = \"/dev/xvda\"\n          ebs = {\n            volume_size           = 128\n            volume_type           = \"gp3\"\n            delete_on_termination = true\n          }\n        }\n      }\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 2\n    }\n  }\n\n  tags = local.tags\n}\n\n################################################################################\n# Helm charts\n################################################################################\n\nresource \"helm_release\" \"nvidia_device_plugin\" {\n  name             = \"nvidia-device-plugin\"\n  repository       = \"https://nvidia.github.io/k8s-device-plugin\"\n  chart            = \"nvidia-device-plugin\"\n  version          = \"0.14.5\"\n  namespace        = \"nvidia-device-plugin\"\n  create_namespace = true\n  wait             = false\n\n  values = [\n    &lt;&lt;-EOT\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n            - matchExpressions:\n              - key: 'nvidia.com/gpu.present'\n                operator: In\n                values:\n                - 'true'\n    EOT\n  ]\n}\n\nresource \"helm_release\" \"aws_efa_device_plugin\" {\n  name       = \"aws-efa-k8s-device-plugin\"\n  repository = \"https://aws.github.io/eks-charts\"\n  chart      = \"aws-efa-k8s-device-plugin\"\n  version    = \"v0.4.4\"\n  namespace  = \"kube-system\"\n  wait       = false\n\n  values = [\n    &lt;&lt;-EOT\n      nodeSelector:\n        vpc.amazonaws.com/efa.present: 'true'\n      tolerations:\n        - key: nvidia.com/gpu\n          operator: Exists\n          effect: NoSchedule\n    EOT\n  ]\n}\n</code></pre>"},{"location":"patterns/nvidia-gpu-efa/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/nvidia-gpu-efa/#validate","title":"Validate","text":"<p>Note</p> <p>The following steps are shown with <code>g5.8xlarge</code> for frugality. Values shown below will change based on the instance type selected (i.e. - <code>p5.48xlarge</code> has 8 GPUs and 32 EFA interfaces)</p> <ol> <li> <p>List the nodes by instance type:</p> <pre><code>kubectl get nodes -o yaml | grep instance-type | grep node | grep -v f:\n</code></pre> <pre><code>node.kubernetes.io/instance-type: g5.8xlarge\nnode.kubernetes.io/instance-type: m5.large\nnode.kubernetes.io/instance-type: m5.large\nnode.kubernetes.io/instance-type: g5.8xlarge\n</code></pre> <p>You should see two EFA-enabled (in this example <code>g5.8xlarge</code>) nodes in the list.</p> </li> <li> <p>Deploy Kubeflow MPI Operator</p> <p>Kubeflow MPI Operator is required for running MPIJobs on EKS. We will use an MPIJob to test EFA. To deploy the MPI operator execute the following:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/kubeflow/mpi-operator/v0.3.0/deploy/v2beta1/mpi-operator.yaml\n</code></pre> <pre><code>namespace/mpi-operator created\ncustomresourcedefinition.apiextensions.k8s.io/mpijobs.kubeflow.org created\nserviceaccount/mpi-operator created\nclusterrole.rbac.authorization.k8s.io/kubeflow-mpijobs-admin created\nclusterrole.rbac.authorization.k8s.io/kubeflow-mpijobs-edit created\nclusterrole.rbac.authorization.k8s.io/kubeflow-mpijobs-view created\nclusterrole.rbac.authorization.k8s.io/mpi-operator created\nclusterrolebinding.rbac.authorization.k8s.io/mpi-operator created\ndeployment.apps/mpi-operator created\n</code></pre> <p>In addition to deploying the operator, please apply a patch to the mpi-operator clusterrole to allow the mpi-operator service account access to <code>leases</code> resources in the <code>coordination.k8s.io</code> apiGroup.</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/aws-samples/aws-do-eks/main/Container-Root/eks/deployment/kubeflow/mpi-operator/clusterrole-mpi-operator.yaml\n</code></pre> <pre><code>clusterrole.rbac.authorization.k8s.io/mpi-operator configured\n</code></pre> </li> <li> <p>EFA test</p> <p>The results should shown that two EFA adapters are available (one for each worker pod)</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/aws-samples/aws-do-eks/main/Container-Root/eks/deployment/efa-device-plugin/test-efa.yaml\n</code></pre> <pre><code>mpijob.kubeflow.org/efa-info-test created\n</code></pre> <p>Once the test launcher pod enters status <code>Running</code> or <code>Completed</code>, see the test logs using the command below:</p> <pre><code>kubectl logs -f $(kubectl get pods | grep launcher | cut -d ' ' -f 1)\n</code></pre> <pre><code>Warning: Permanently added 'efa-info-test-worker-1.efa-info-test-worker.default.svc,10.11.13.224' (ECDSA) to the list of known hosts.\nWarning: Permanently added 'efa-info-test-worker-0.efa-info-test-worker.default.svc,10.11.4.63' (ECDSA) to the list of known hosts.\n[1,1]&lt;stdout&gt;:provider: efa\n[1,1]&lt;stdout&gt;:    fabric: efa\n[1,1]&lt;stdout&gt;:    domain: rdmap197s0-rdm\n[1,1]&lt;stdout&gt;:    version: 116.10\n[1,1]&lt;stdout&gt;:    type: FI_EP_RDM\n[1,1]&lt;stdout&gt;:    protocol: FI_PROTO_EFA\n[1,0]&lt;stdout&gt;:provider: efa\n[1,0]&lt;stdout&gt;:    fabric: efa\n[1,0]&lt;stdout&gt;:    domain: rdmap197s0-rdm\n[1,0]&lt;stdout&gt;:    version: 116.10\n[1,0]&lt;stdout&gt;:    type: FI_EP_RDM\n[1,0]&lt;stdout&gt;:    protocol: FI_PROTO_EFA\n</code></pre> </li> <li> <p>EFA NCCL test</p> <p>To run the EFA NCCL test please execute the following kubectl command:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/aws-samples/aws-do-eks/main/Container-Root/eks/deployment/efa-device-plugin/test-nccl-efa.yaml\n</code></pre> <pre><code>mpijob.kubeflow.org/test-nccl-efa created\n</code></pre> <p>Once the launcher pod enters <code>Running</code> or <code>Completed</code> state, execute the following to see the test logs:</p> <pre><code>kubectl logs -f $(kubectl get pods | grep launcher | cut -d ' ' -f 1)\n</code></pre> <pre><code>[1,0]&lt;stdout&gt;:test-nccl-efa-worker-0:21:21 [0] NCCL INFO NET/OFI Selected Provider is efa (found 1 nics)\n[1,0]&lt;stdout&gt;:test-nccl-efa-worker-0:21:21 [0] NCCL INFO Using network AWS Libfabric\n[1,0]&lt;stdout&gt;:NCCL version 2.12.7+cuda11.4\n</code></pre> <p>Columns 8 and 12 in the output table show the in-place and out-of-place bus bandwidth calculated for the data size listed in column 1. In this case it is 3.13 and 3.12 GB/s respectively. Your actual results may be slightly different. The calculated average bus bandwidth is displayed at the bottom of the log when the test finishes after it reaches the max data size, specified in the mpijob manifest. In this result the average bus bandwidth is 1.15 GB/s.</p> <pre><code>[1,0]&lt;stdout&gt;:#       size         count      type   redop    root     time   algbw   busbw #wrong     time   algbw   busbw #wrong\n[1,0]&lt;stdout&gt;:#        (B)    (elements)                               (us)  (GB/s)  (GB/s)            (us)  (GB/s)  (GB/s)\n...\n[1,0]&lt;stdout&gt;:      262144         65536     float     sum      -1    195.0    1.34    1.34      0    194.0    1.35    1.35      0\n[1,0]&lt;stdout&gt;:      524288        131072     float     sum      -1    296.9    1.77    1.77      0    291.1    1.80    1.80      0\n[1,0]&lt;stdout&gt;:     1048576        262144     float     sum      -1    583.4    1.80    1.80      0    579.6    1.81    1.81      0\n[1,0]&lt;stdout&gt;:     2097152        524288     float     sum      -1    983.3    2.13    2.13      0    973.9    2.15    2.15      0\n[1,0]&lt;stdout&gt;:     4194304       1048576     float     sum      -1   1745.4    2.40    2.40      0   1673.2    2.51    2.51      0\n...\n[1,0]&lt;stdout&gt;:# Avg bus bandwidth    : 1.15327\n</code></pre> </li> </ol>"},{"location":"patterns/nvidia-gpu-efa/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/private-public-ingress/","title":"Private and Public Ingress","text":""},{"location":"patterns/private-public-ingress/#amazon-eks-private-and-public-ingress-example","title":"Amazon EKS Private and Public Ingress example","text":"<p>This example demonstrates how to provision an Amazon EKS cluster with two  ingress-nginx controllers; one to expose applications publicly and the other to expose applications internally. It also assigns security groups to the Network Load Balancers used to expose the internal and external ingress controllers.</p> <p>This solution:</p> <ul> <li>Installs an ingress-nginx controller for public traffic</li> <li>Installs an ingress-nginx controller for internal traffic</li> </ul> <p>To expose your application services via an <code>Ingress</code> resource with this solution you can set the respective <code>ingressClassName</code> as either <code>ingress-nginx-external</code> or <code>ingress-nginx-internal</code>.</p> <p>Refer to the documentation for <code>AWS Load Balancer controller</code> configuration options.</p>"},{"location":"patterns/private-public-ingress/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/private-public-ingress/#validate","title":"Validate","text":"<p>TODO</p> <p>Add in validation steps</p>"},{"location":"patterns/private-public-ingress/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/privatelink-access/","title":"PrivateLink Access","text":""},{"location":"patterns/privatelink-access/#private-eks-cluster-access-via-aws-privatelink","title":"Private EKS cluster access via AWS PrivateLink","text":"<p>This pattern demonstrates how to access a private EKS cluster using AWS PrivateLink.</p> <p>Refer to the documentation for further details on  <code>AWS PrivateLink</code>.</p>"},{"location":"patterns/privatelink-access/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and follow the steps below to deploy this pattern.</p> <pre><code>terraform init\nterraform apply -target=module.eventbridge -target=module.nlb --auto-approve\nterraform apply --auto-approve\n</code></pre>"},{"location":"patterns/privatelink-access/#test-access-to-eks-kubernetes-api-server-endpoint","title":"Test access to EKS Kubernetes API server endpoint","text":"<p>Of the output values, the value <code>ssm_test</code> is provided to aid in quickly testing the connectivity from the client EC2 instance to the private EKS cluster via AWS PrivateLink. Copy the output value, which looks like the snippet shown below (as an example) and paste it into your terminal to execute and check the connectivity. If configured correctly, the value returned should be <code>ok</code>.</p> <pre><code>COMMAND=\"curl -ks https://9A85B21811733524E3ABCDFEA8714642.gr7.us-west-2.eks.amazonaws.com/readyz\"\n\nCOMMAND_ID=$(aws ssm send-command --region us-west-2 \\\n   --document-name \"AWS-RunShellScript\" \\\n   --parameters \"commands=[$COMMAND]\" \\\n   --targets \"Key=instanceids,Values=i-0a45eff73ba408575\" \\\n   --query 'Command.CommandId' \\\n   --output text)\n\naws ssm get-command-invocation --region us-west-2 \\\n   --command-id $COMMAND_ID \\\n   --instance-id i-0a45eff73ba408575 \\\n   --query 'StandardOutputContent' \\\n   --output text\n</code></pre>"},{"location":"patterns/privatelink-access/#test-access-to-eks-kubernetes-api-with-kubectl","title":"Test access to EKS Kubernetes API with <code>kubectl</code>","text":"<p>Perform the following steps to access the cluster with <code>kubectl</code> from the provided Client EC2 instance.</p>"},{"location":"patterns/privatelink-access/#log-into-the-client-ec2-instance","title":"Log into the Client EC2 instance","text":"<p>Start a new SSM session on the Client EC2 instance using the provided <code>ssm_start_session</code> output value. It should look similar to the snippet shown below. Copy the output value and paste it into your terminal to execute. Your terminal will now be connected to the Client EC2 instance.</p> <pre><code>aws ssm start-session --region us-west-2 --target i-0280cf604085f4a44\n</code></pre>"},{"location":"patterns/privatelink-access/#update-kubeconfig","title":"Update Kubeconfig","text":"<p>On the Client EC2 machine, run the following command to update the local <code>~/.kube/config</code> file to enable access to the cluster:</p> <pre><code>aws eks update-kubeconfig --region us-west-2 --name privatelink-access\n</code></pre>"},{"location":"patterns/privatelink-access/#test-complete-access-with-kubectl","title":"Test complete access with <code>kubectl</code>","text":"<p>Test access by listing the pods running on the cluster:</p> <pre><code>kubectl get pods -A\n</code></pre> <pre><code>NAMESPACE     NAME                       READY   STATUS    RESTARTS   AGE\nkube-system   aws-node-4f8g8             1/1     Running   0          1m\nkube-system   coredns-6ff9c46cd8-59sqp   1/1     Running   0          1m\nkube-system   coredns-6ff9c46cd8-1npb   1/1     Running   0          2m\nkube-system   kube-proxy-mm2zc           1/1     Running   0          1m\n</code></pre>"},{"location":"patterns/privatelink-access/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/sso-iam-identity-center/","title":"SSO - IAM Identity Center","text":""},{"location":"patterns/sso-iam-identity-center/#iam-identity-center-single-sign-on-for-amazon-eks-cluster-with-cluster-access-manager","title":"IAM Identity Center Single Sign-On for Amazon EKS Cluster with Cluster Access Manager","text":"<p>This example demonstrates how to deploy an Amazon EKS cluster that is deployed on the AWS Cloud, integrated with IAM Identity Center (former AWS SSO) as an the Identity Provider (IdP) for Single Sign-On (SSO) authentication having Cluster Access Manager as the entry point API. The configuration to provide authorization for users is simplified using a combination of Access Entries and Kubernetes Role-based access control (RBAC).</p>"},{"location":"patterns/sso-iam-identity-center/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p> <p>This patterns uses aws Identity datasource <code>data \"aws_ssoadmin_instances\" \"this\" {}</code> to retrieve your identity center configuration. You can check that the datasource can works by using the aws cli:</p> <pre><code>aws identitystore list-instances\n</code></pre> <p>You may need to specify the appropriate AWS region in either the previous command and terraform (in main.tf) for the pattern to succeed. If the previous command is empty, you may want to properly configure AWS identity center within your account.</p>"},{"location":"patterns/sso-iam-identity-center/#validate","title":"Validate","text":"<p>After the <code>terraform</code> commands are executed successfully, check if the newly created users are active.</p> <p>To do that use the link provided in the email invite - if you added a valid email address for your users either in your Terraform code or IAM Identity Center Console - or go to the IAM Identity Center Console, in the Users dashboard on the left hand side menu, then select the user, and click on Reset password button on the upper right corner. Choose the option to Generate a one-time password and share the password with the user.</p> <p>With the active users, use one of the <code>terraform output</code> examples to configure your AWS credentials for SSO, as shown in the examples below. After you choose the SSO registration scopes, your browser windows will appear and request to login using your IAM Identity Center username and password.</p>"},{"location":"patterns/sso-iam-identity-center/#admin-user-example","title":"Admin user example","text":"<pre><code>configure_sso_admin = &lt;&lt;EOT\n  # aws configure sso --profile EKSClusterAdmin\n      SSO session name (Recommended): &lt;SESSION_NAME&gt;\n      SSO start URL [None]: https://d-1234567890.awsapps.com/start\n      SSO region [None]: us-west-2\n      SSO registration scopes [sso:account:access]:\n      Attempting to automatically open the SSO authorization page in your default browser.\n      If the browser does not open or you wish to use a different device to authorize this request, open the following URL:\n\n      https://device.sso.us-west-2.amazonaws.com/\n\n      Then enter the code:\n\n      The only AWS account available to you is: 123456789012\n      Using the account ID 123456789012\n      The only role available to you is: EKSClusterAdmin\n      Using the role name EKSClusterAdmin\n      CLI default client Region [us-west-2]: us-west-2\n      CLI default output format [json]: json\n      CLI profile name [EKSClusterAdmin-123456789012]:\n\n      To use this profile, specify the profile name using --profile, as shown:\n\n  # aws eks --region us-west-2 update-kubeconfig --name iam-identity-center --profile EKSClusterAdmin\n\nEOT\n</code></pre>"},{"location":"patterns/sso-iam-identity-center/#read-only-user-example","title":"Read-only user example","text":"<pre><code>configure_sso_user = &lt;&lt;EOT\n  # aws configure sso --profile EKSClusterUser\n      SSO session name (Recommended): &lt;SESSION_NAME&gt;\n      SSO start URL [None]: https://d-1234567890.awsapps.com/start\n      SSO region [None]: us-west-2\n      SSO registration scopes [sso:account:access]:\n      Attempting to automatically open the SSO authorization page in your default browser.\n      If the browser does not open or you wish to use a different device to authorize this request, open the following URL:\n\n      https://device.sso.us-west-2.amazonaws.com/\n\n      Then enter the code:\n\n      The only AWS account available to you is: 123456789012\n      Using the account ID 123456789012\n      The only role available to you is: EKSClusterUser\n      Using the role name EKSClusterUser\n      CLI default client Region [us-west-2]: us-west-2\n      CLI default output format [json]: json\n      CLI profile name [EKSClusterUser-123456789012]:\n\n      To use this profile, specify the profile name using --profile, as shown:\n\n  # aws eks --region us-west-2 update-kubeconfig --name iam-identity-center --profile EKSClusterUser\n\nEOT\n</code></pre> <p>With the <code>kubeconfig</code> configured, you'll be able to run <code>kubectl</code> commands in your Amazon EKS Cluster with the impersonated user. The read-only user has a <code>cluster-viewer</code> Kubernetes role bound to it's group, whereas the admin user, has the <code>admin</code> Kubernetes role bound to it's group.</p> <pre><code>kubectl get pods -A\nNAMESPACE          NAME                        READY   STATUS    RESTARTS   AGE\namazon-guardduty   aws-guardduty-agent-bl2v2   1/1     Running   0          3h54m\namazon-guardduty   aws-guardduty-agent-s2vcx   1/1     Running   0          3h54m\namazon-guardduty   aws-guardduty-agent-w8gfc   1/1     Running   0          3h54m\nkube-system        aws-node-m9hmd              1/1     Running   0          3h53m\nkube-system        aws-node-w42b8              1/1     Running   0          3h53m\nkube-system        aws-node-wm6rm              1/1     Running   0          3h53m\nkube-system        coredns-6ff9c46cd8-94jlr    1/1     Running   0          3h59m\nkube-system        coredns-6ff9c46cd8-n2mrb    1/1     Running   0          3h59m\nkube-system        kube-proxy-7fb86            1/1     Running   0          3h54m\nkube-system        kube-proxy-p4f5g            1/1     Running   0          3h54m\nkube-system        kube-proxy-q1fmc            1/1     Running   0          3h54m\n</code></pre> <p>If not revoked after the cluster creation, it's possible to use the <code>configure_kubectl</code> output to assume the Cluster creator role with <code>cluster-admin</code> access.</p> <pre><code>configure_kubectl = \"aws eks --region us-west-2 update-kubeconfig --name iam-identity-center\"\n</code></pre>"},{"location":"patterns/sso-iam-identity-center/#destroy","title":"Destroy","text":"<p>If you revoked the Cluster creator <code>cluster-admin</code> permission, you may need to re-associate the <code>AmazonEKSClusterAdminPolicy</code> access entry to run <code>terraform destroy</code>.</p> <pre><code>terraform destroy -target module.developers_team -auto-approve\nterraform destroy -target module.eks -auto-approve\nterraform destroy -auto-approve\n</code></pre>"},{"location":"patterns/sso-okta/","title":"SSO - Okta","text":""},{"location":"patterns/sso-okta/#okta-single-sign-on-for-amazon-eks-cluster","title":"Okta Single Sign-On for Amazon EKS Cluster","text":"<p>This example demonstrates how to deploy an Amazon EKS cluster that is deployed on the AWS Cloud, integrated with Okta as an the Identity Provider (IdP) for Single Sign-On (SSO) authentication. The configuration for authorization is done using Kubernetes Role-based access control (RBAC).</p>"},{"location":"patterns/sso-okta/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/sso-okta/#validate","title":"Validate","text":"<p>After the <code>terraform</code> commands are executed successfully, check if the newly created users are active.</p> <p>To do that use the link provided in the email invite if you added a valid email address for your users, or go to the Okta Admin Dashboard, select the user, and click on Set Password and Activate button.</p> <p>With the active users, use the <code>terraform output</code> example to setup your <code>kubeconfig</code> profile to authenticate through Okta.</p> <pre><code>configure_kubeconfig = &lt;&lt;EOT\n    kubectl config set-credentials oidc \\\n      --exec-api-version=client.authentication.k8s.io/v1beta1 \\\n      --exec-command=kubectl \\\n      --exec-arg=oidc-login \\\n      --exec-arg=get-token \\\n      --exec-arg=--oidc-issuer-url=https://dev-ORGID.okta.com/oauth2/1234567890abcdefghij \\\n      --exec-arg=--oidc-client-id=1234567890abcdefghij\n      --exec-arg=--oidc-extra-scope=\"email offline_access profile openid\"\n</code></pre> <p>With the <code>kubeconfig</code> configured, you'll be able to run <code>kubectl</code> commands in your Amazon EKS Cluster using the <code>--user</code> cli option to impersonate the Okta authenticated user. When <code>kubectl</code> command is issued with the <code>--user</code> option for the first time, your browser window will open and require you to authenticate.</p> <p>The read-only user has a <code>cluster-viewer</code> Kubernetes role bound to it's group, whereas the admin user, has the <code>admin</code> Kubernetes role bound to it's group.</p> <pre><code>kubectl get pods -A\nNAMESPACE          NAME                        READY   STATUS    RESTARTS   AGE\namazon-guardduty   aws-guardduty-agent-bl2v2   1/1     Running   0          3h54m\namazon-guardduty   aws-guardduty-agent-s1vcx   1/1     Running   0          3h54m\namazon-guardduty   aws-guardduty-agent-w8gfc   1/1     Running   0          3h54m\nkube-system        aws-node-m9hmd              1/1     Running   0          3h53m\nkube-system        aws-node-w42b8              1/1     Running   0          3h53m\nkube-system        aws-node-wm6rm              1/1     Running   0          3h53m\nkube-system        coredns-6ff9c46cd8-94jlr    1/1     Running   0          3h59m\nkube-system        coredns-6ff9c46cd8-nw2rb    1/1     Running   0          3h59m\nkube-system        kube-proxy-7fb86            1/1     Running   0          3h54m\nkube-system        kube-proxy-p4f5g            1/1     Running   0          3h54m\nkube-system        kube-proxy-qk2mc            1/1     Running   0          3h54m\n</code></pre> <p>You can also use the <code>configure_kubectl</code> output to assume the Cluster creator role with <code>cluster-admin</code> access.</p> <pre><code>configure_kubectl = \"aws eks --region us-west-2 update-kubeconfig --name okta\"\n</code></pre> <p>It's also possible to pre-configure your <code>kubeconfig</code> using the <code>okta_login</code> output. This will also require you to authenticate in a browser window.</p> <pre><code>okta_login = \"kubectl oidc-login setup --oidc-issuer-url=https://dev-ORGID.okta.com/oauth2/1234567890abcdefghij--oidc-client-id=1234567890abcdefghij\"\n</code></pre>"},{"location":"patterns/sso-okta/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/stateful/","title":"Stateful","text":""},{"location":"patterns/stateful/#amazon-eks-cluster-for-stateful-workloads","title":"Amazon EKS Cluster for Stateful Workloads","text":""},{"location":"patterns/stateful/#features","title":"Features","text":"<p>Please note: not all of the features listed below are required for stateful workloads on EKS. We are simply grouping together a set of features that are commonly encountered when managing stateful workloads. Users are encouraged to only enable the features that are required for their workload(s) and use case(s).</p>"},{"location":"patterns/stateful/#velero","title":"velero","text":"<p>(From the project documentation) <code>velero</code> (formerly Heptio Ark) gives you tools to back up and restore your Kubernetes cluster resources and persistent volumes. You can run Velero with a public cloud platform or on-premises. Velero lets you:</p> <ul> <li>Take backups of your cluster and restore in case of loss.</li> <li>Migrate cluster resources to other clusters.</li> <li>Replicate your production cluster to development and testing clusters.</li> </ul>"},{"location":"patterns/stateful/#ebs-efs-csi-drivers","title":"EBS &amp; EFS CSI Drivers","text":"<ul> <li>A second storage class for <code>gp3</code> backed volumes has been added and made the default over the EKS default <code>gp2</code> storage class (<code>gp2</code> storage class remains in the cluster for use, but it is no longer the default storage class)</li> <li>A standard implementation of the EFS CSI driver</li> </ul>"},{"location":"patterns/stateful/#eks-managed-nodegroup-w-multiple-volumes","title":"EKS Managed Nodegroup w/ Multiple Volumes","text":"<p>An EKS managed nodegroup that utilizes multiple EBS volumes. The primary use case demonstrated in this example is a second volume that is dedicated to the <code>containerd</code> runtime to ensure the root volume is not filled up nor has its I/O exhausted to ensure the instance does not reach a degraded state. The <code>containerd</code> directories are mapped to this volume. You can read more about this recommendation in our EKS best practices guide and refer to the <code>containerd</code> documentation for more information. The update for <code>containerd</code> to use the second volume is managed through the provided user data.</p> <p>In addition, the following properties are configured on the nodegroup volumes:</p> <ul> <li>EBS encryption using a customer managed key (CMK)</li> <li>Configuring the volumes to use GP3 storage</li> </ul>"},{"location":"patterns/stateful/#eks-managed-nodegroup-w-instance-store-volumes","title":"EKS Managed Nodegroup w/ Instance Store Volume(s)","text":"<p>An EKS managed nodegroup that utilizes EC2 instances with ephemeral instance store(s). Instance stores are ideal for temporary storage of information that changes frequently, such as buffers, caches, scratch data, and other temporary content, or for data that is replicated across a fleet of instances. You can read more about instance stores in the AWS documentation; and be sure to check out the <code>Block device mapping instance store caveats</code> section as well which covers why the example has provided user data for mounting the instance store(s). The size and number of instance stores will vary based on the EC2 instance type and class.</p> <p>In addition, the following properties are configured on the nodegroup volumes:</p> <ul> <li>EBS encryption using a customer managed key (CMK)</li> <li>Configuring the volumes to use GP3 storage</li> </ul>"},{"location":"patterns/stateful/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/stateful/#validate","title":"Validate","text":"<p>The following command will update the <code>kubeconfig</code> on your local machine and allow you to interact with your EKS Cluster using <code>kubectl</code> to validate the Velero deployment.</p> <ol> <li> <p>Run <code>update-kubeconfig</code> command:</p> <pre><code>aws eks --region &lt;REGION&gt; update-kubeconfig --name &lt;CLUSTER_NAME&gt;\n</code></pre> </li> <li> <p>List the storage classes to view that <code>efs</code>, <code>gp2</code>, and <code>gp3</code> classes are present and <code>gp3</code> is the default storage class</p> <pre><code>kubectl get storageclasses\n</code></pre> <pre><code>NAME            PROVISIONER             RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE\nefs             efs.csi.aws.com         Delete          Immediate              true                   2m19s\ngp2             kubernetes.io/aws-ebs   Delete          WaitForFirstConsumer   false                  15m\ngp3 (default)   ebs.csi.aws.com         Delete          WaitForFirstConsumer   true                   2m19s\n</code></pre> </li> <li> <p>From an instance launched with instance store(s), check that the instance store has been mounted correctly. To verify, first install the <code>nvme-cli</code> tool and then use it to verify. To verify, you can access the instance using SSM Session Manager:</p> <pre><code># Install the nvme-cli tool\nsudo yum install nvme-cli -y\n\n# Show NVMe volumes attached\nsudo nvme list\n</code></pre> <pre><code># Notice the model is `EC2 NVMe Instance Storage` for the instance store\nNode             SN                   Model                                    Namespace Usage                      Format           FW Rev\n---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------\n/dev/nvme0n1     vol0546d3c3b0af0bf6d Amazon Elastic Block Store               1          25.77  GB /  25.77  GB    512   B +  0 B   1.0\n/dev/nvme1n1     AWS24BBF51AF55097008 Amazon EC2 NVMe Instance Storage         1          75.00  GB /  75.00  GB    512   B +  0 B   0\n\n# Show disks, their partitions and mounts\nsudo lsblk\n\n# Output should look like below\nNAME          MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT\nnvme0n1       259:0    0   24G  0 disk\n\u251c\u2500nvme0n1p1   259:2    0   24G  0 part /\n\u2514\u2500nvme0n1p128 259:3    0    1M  0 part\nnvme1n1       259:1    0 69.9G  0 disk /local1 # &lt;--- this is the instance store\n</code></pre> </li> <li> <p>From an instance launched with multiple volume(s), check that the instance store has been mounted correctly. To verify, first install the <code>nvme-cli</code> tool and then use it to verify. To verify, you can access the instance using SSM Session Manager:</p> <pre><code># Install the nvme-cli tool\nsudo yum install nvme-cli -y\n\n# Show NVMe volumes attached\nsudo nvme list\n</code></pre> <pre><code># /dev/nvme0n1 is the root volume and /dev/nvme1n1 is the second, additional volume\nNode             SN                   Model                                    Namespace Usage                      Format           FW Rev\n---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------\n/dev/nvme0n1     vol0cd37dab9e4a5c184 Amazon Elastic Block Store               1          68.72  GB /  68.72  GB    512   B +  0 B   1.0\n/dev/nvme1n1     vol0ad3629c159ee869c Amazon Elastic Block Store               1          25.77  GB /  25.77  GB    512   B +  0 B   1.0\n</code></pre> </li> <li> <p>From the same instance used in step 4, check that the containerd directories are using the second <code>/dev/nvme1n1</code> volume:</p> <pre><code>df /var/lib/containerd/\n</code></pre> <pre><code># Output should look like below, which shows the directory on the\n# /dev/nvme1n1 volume and NOT on /dev/nvme0n1 (root volume)\nFilesystem     1K-blocks    Used Available Use% Mounted on\n/dev/nvme1n1    24594768 2886716  20433380  13% /var/lib/containerd\n</code></pre> <pre><code>df /run/containerd/\n</code></pre> <pre><code># Output should look like below, which shows the directory on the\n# /dev/nvme1n1 volume and NOT on /dev/nvme0n1 (root volume)\nFilesystem     1K-blocks    Used Available Use% Mounted on\n/dev/nvme1n1    24594768 2886716  20433380  13% /run/containerd\n</code></pre> </li> <li> <p>Test by listing velero resources provisioned:</p> <pre><code>kubectl get all -n velero\n\n# Output should look similar to below\nNAME                         READY   STATUS    RESTARTS   AGE\npod/velero-b4d8fd5c7-5smp6   1/1     Running   0          112s\n\nNAME             TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE\nservice/velero   ClusterIP   172.20.217.203   &lt;none&gt;        8085/TCP   114s\n\nNAME                     READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/velero   1/1     1            1           114s\n\nNAME                               DESIRED   CURRENT   READY   AGE\nreplicaset.apps/velero-b4d8fd5c7   1         1         1       114s\n</code></pre> </li> <li> <p>Get backup location using velero CLI</p> <pre><code>velero backup-location get\n\n# Output should look similar to below\nNAME      PROVIDER   BUCKET/PREFIX             PHASE       LAST VALIDATED                  ACCESS MODE   DEFAULT\ndefault   aws        velero-ss1wm44hv1fzb32d   Available   2022-05-22 10:53:26 -0400 EDT   ReadWrite     true\n</code></pre> </li> </ol>"},{"location":"patterns/stateful/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/targeted-odcr/","title":"Targeted On-Demand Capacity Reservation (ODCR)","text":""},{"location":"patterns/targeted-odcr/#eks-cluster-with-on-demand-capacity-reservations-odcr","title":"EKS Cluster with On-demand Capacity Reservations (ODCR)","text":"<p>This pattern demonstrates how to consume/utilize on-demand capacity reservations (ODCRs) with Amazon EKS. The solution is comprised of primarily 3 components:</p> <ol> <li> <p>The node group that will utilize the ODCRs should have the subnets provided to it restricted to the availability zone where the ODCR(s) capacity is allocated. For example - if the ODCR(s) are allocated to <code>us-west-2b</code>, the node group should only have subnet IDs provided to it that reside in <code>us-west-2b</code>. If the subnets that reside in other AZs are provided, its possible to encounter an error such as <code>InvalidParameterException: The following supplied instance types do not exist ...</code>. It is not guaranteed that this error will always be shown, and may appear random since the underlying autoscaling group(s) will provision nodes into different AZs at random. It will only occur when the underlying autoscaling group tries to provision instances into an AZ where capacity is not allocated and there is insufficient on-demand capacity for the desired instance type.</p> </li> <li> <p>A custom launch template is required in order to specify the <code>capacity_reservation_specification</code> arguments. This is how the ODCRs are integrated into the node group (i.e. - tells the autoscaling group to utilize the provided capacity reservation(s)).</p> <p>Info</p> <p>By default, the <code>terraform-aws-eks</code> module creates and utilizes a custom launch template with EKS managed node groups which means users just need to supply the <code>capacity_reservation_specification</code> in their node group definition.</p> </li> <li> <p>A resource group will need to be created for the capacity reservations. The resource group acts like a container, allowing for ODCRs to be added or removed as needed to adjust the available capacity. Utilizing the resource group allows for this additional capacity to be adjusted without any modification or disruption to the existing node group or launch template. As soon as the ODCR has been associated to the resource group, the node group can scale up to start utilizing that capacity.</p> </li> </ol> <p>Links:</p> <ul> <li>Tutorial: Launch On-Demand Instances using targeted Capacity Reservations</li> <li>Target a group of Amazon EC2 On-Demand Capacity Reservations</li> </ul>"},{"location":"patterns/targeted-odcr/#code","title":"Code","text":"<pre><code>################################################################################\n# Cluster\n################################################################################\n\nmodule \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n  version = \"~&gt; 20.9\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.29\"\n\n  # To facilitate easier interaction for demonstration purposes\n  cluster_endpoint_public_access = true\n\n  # Gives Terraform identity admin access to the cluster\n  enable_cluster_creator_admin_permissions = true\n\n  cluster_addons = {\n    coredns    = {}\n    kube-proxy = {}\n    vpc-cni    = {}\n  }\n\n  vpc_id     = module.vpc.vpc_id\n  subnet_ids = module.vpc.private_subnets\n\n  eks_managed_node_group_defaults = {\n    iam_role_additional_policies = {\n      # Not required, but used in the example to access the nodes to inspect drivers and devices\n      AmazonSSMManagedInstanceCore = \"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore\"\n    }\n  }\n\n  eks_managed_node_groups = {\n    odcr = {\n      instance_types = [\"t3.micro\", \"t3.small\"]\n\n      min_size     = 4\n      max_size     = 5\n      desired_size = 2\n\n      # First subnet is in the \"${local.region}a\" availability zone\n      # where the capacity reservation is created\n      subnet_ids = [element(module.vpc.private_subnets, 0)]\n      capacity_reservation_specification = {\n        capacity_reservation_target = {\n          capacity_reservation_resource_group_arn = aws_resourcegroups_group.odcr.arn\n        }\n      }\n    }\n  }\n\n  tags = local.tags\n}\n\n################################################################################\n# Resource Group\n################################################################################\n\nresource \"aws_resourcegroups_group\" \"odcr\" {\n  name        = \"${local.name}-p5-odcr\"\n  description = \"P5 instance on-demand capacity reservations\"\n\n  configuration {\n    type = \"AWS::EC2::CapacityReservationPool\"\n  }\n\n  configuration {\n    type = \"AWS::ResourceGroups::Generic\"\n\n    parameters {\n      name   = \"allowed-resource-types\"\n      values = [\"AWS::EC2::CapacityReservation\"]\n    }\n  }\n}\n\nresource \"aws_resourcegroups_resource\" \"odcr_1\" {\n  group_arn = aws_resourcegroups_group.odcr.arn\n  # Replace the following with the ARN of the capacity reservation\n  # provided by AWS when supplied with a capacity reservation\n  resource_arn = aws_ec2_capacity_reservation.micro.arn\n}\n\nresource \"aws_resourcegroups_resource\" \"odcr_2\" {\n  group_arn = aws_resourcegroups_group.odcr.arn\n  # Replace the following with the ARN of the capacity reservation\n  # provided by AWS when supplied with a capacity reservation\n  resource_arn = aws_ec2_capacity_reservation.small.arn\n}\n\n################################################################################\n# Capacity Reservation\n# These are created for the example, but are not necessary when\n# AWS EC2 provides you with a capacity reservation ID\n################################################################################\n\nresource \"aws_ec2_capacity_reservation\" \"micro\" {\n  instance_type           = \"t3.micro\"\n  instance_platform       = \"Linux/UNIX\"\n  availability_zone       = \"${local.region}a\"\n  instance_count          = 2\n  instance_match_criteria = \"targeted\"\n\n  # Just for example - 30 minutes from time of creation\n  end_date      = timeadd(timestamp(), \"30m\")\n  end_date_type = \"limited\"\n}\n\nresource \"aws_ec2_capacity_reservation\" \"small\" {\n  instance_type           = \"t3.small\"\n  instance_platform       = \"Linux/UNIX\"\n  availability_zone       = \"${local.region}a\"\n  instance_count          = 2\n  instance_match_criteria = \"targeted\"\n\n  # Just for example - 30 minutes from time of creation\n  end_date      = timeadd(timestamp(), \"30m\")\n  end_date_type = \"limited\"\n}\n\n################################################################################\n# Kubectl Output\n################################################################################\n\noutput \"configure_kubectl\" {\n  description = \"Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig\"\n  value       = \"aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}\"\n}\n</code></pre>"},{"location":"patterns/targeted-odcr/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/targeted-odcr/#validate","title":"Validate","text":"<ol> <li> <p>Navigate to the EC2 console page - on the left hand side, click on <code>Capacity Reservations</code>under the <code>Instances</code> section. You should see the capacity reservation(s) that have been created similar to the screenshot below. For this example, you can see that <code>Available capacity</code> column is empty, which means that the capacity reservations have been fully utilized by the example (as expected).</p> <p> </p> </li> <li> <p>Click on one of the capacity reservation IDs to view the details of the capacity reservation. You should see the details of the capacity reservation similar to the screenshot below. For this example, you can see that <code>Available capacity</code> is <code>0 instances</code>, which means that the capacity reservation has been fully utilized by the example (as expected).</p> <p> </p> </li> </ol>"},{"location":"patterns/targeted-odcr/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/tls-with-aws-pca-issuer/","title":"TLS w/ AWS PCA Issuer","text":""},{"location":"patterns/tls-with-aws-pca-issuer/#tls-with-aws-pca-issuer","title":"TLS with AWS PCA Issuer","text":"<p>This pattern demonstrates how to enable TLS with AWS PCA issuer on an Amazon EKS cluster.</p>"},{"location":"patterns/tls-with-aws-pca-issuer/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/tls-with-aws-pca-issuer/#validate","title":"Validate","text":"<ol> <li> <p>List all the pods running in <code>aws-privateca-issuer</code> and <code>cert-manager</code> Namespace.</p> <pre><code>kubectl get pods -n aws-privateca-issuer\nkubectl get pods -n cert-manager\n</code></pre> </li> <li> <p>View the <code>certificate</code> status in the <code>default</code> Namespace. It should be in <code>Ready</code> state, and be pointing to a <code>secret</code> created in the same Namespace.</p> <pre><code>kubectl get certificate -o wide\n</code></pre> <pre><code>NAME      READY   SECRET                  ISSUER                    STATUS                                          AGE\nexample   True    example-clusterissuer   tls-with-aws-pca-issuer   Certificate is up to date and has not expired   41m\n</code></pre> <pre><code>kubectl get secret example-clusterissuer\n</code></pre> <pre><code>NAME                    TYPE                DATA   AGE\nexample-clusterissuer   kubernetes.io/tls   3      43m\n</code></pre> </li> </ol>"},{"location":"patterns/tls-with-aws-pca-issuer/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/vpc-lattice/","title":"Amazon VPC Lattice","text":""},{"location":"patterns/vpc-lattice/#amazon-vpc-lattice","title":"Amazon VPC Lattice","text":"<p>This folder contains use case-driven patterns covering different aspects of the Amazon VPC Lattice service.</p>"},{"location":"patterns/vpc-lattice/#use-cases","title":"Use cases","text":"<ul> <li> <p>Simple Client to Server Communication</p> <p>This pattern describes how to expose a simple API within an Amazon EKS cluster deployed in VPC A to a client application hosted in VPC B through Amazon VPC Lattice.</p> </li> </ul>"},{"location":"patterns/vpc-lattice/#supporting-resources","title":"Supporting resources","text":"<ul> <li>Documentation</li> <li>AWS Gateway API Controller</li> </ul>"},{"location":"patterns/wireguard-with-cilium/","title":"Wireguard /w Cilium","text":""},{"location":"patterns/wireguard-with-cilium/#transparent-encryption-with-cilium-and-wireguard","title":"Transparent Encryption with Cilium and Wireguard","text":"<p>This pattern demonstrates Cilium configured in CNI chaining mode with the VPC CNI and with Wireguard transparent encryption enabled on an Amazon EKS cluster.</p> <ul> <li>Cilium CNI Chaining Documentation</li> <li>Cilium Wireguard Encryption Documentation</li> </ul>"},{"location":"patterns/wireguard-with-cilium/#focal-points","title":"Focal Points","text":"<ul> <li><code>eks.tf</code> contains the cluster configuration and the deployment of Cilium.<ul> <li>There are no specific requirements from an EKS perspective, other than the Linux Kernel version used by the OS must be 5.10+</li> </ul> </li> <li><code>example.yaml</code> provides a sample application used to demonstrate the encrypted connectivity. This is optional and not required for the pattern.</li> </ul>"},{"location":"patterns/wireguard-with-cilium/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/wireguard-with-cilium/#validate","title":"Validate","text":"<ol> <li> <p>Deploy the example pods:</p> <pre><code>kubectl apply -f example.yaml\n</code></pre> <pre><code>pod/server created\nservice/server created\npod/client created\n</code></pre> </li> <li> <p>Get the Cilium status from one of the Cilium pods.</p> <p>Under the <code>Encryption</code> field, it should state <code>Wireguard</code> with a PubKey. <code>NodeEncryption: Disabled</code> is expected since <code>NodeEncryption</code> was not enabled via the Helm values provided.</p> <pre><code>kubectl -n kube-system exec -ti ds/cilium -- cilium status\n</code></pre> <pre><code>Defaulted container \"cilium-agent\" out of: cilium-agent, config (init), mount-cgroup (init), apply-sysctl-overwrites (init), mount-bpf-fs (init), clean-cilium-state (init), install-cni-binaries (init)\nKVStore:                 Ok   Disabled\nKubernetes:              Ok   1.28+ (v1.28.1-eks-43840fb) [linux/amd64]\nKubernetes APIs:         [\"EndpointSliceOrEndpoint\", \"cilium/v2::CiliumClusterwideNetworkPolicy\", \"cilium/v2::CiliumEndpoint\", \"cilium/v2::CiliumNetworkPolicy\", \"cilium/v2::CiliumNode\", \"cilium/v2alpha1::CiliumCIDRGroup\", \"core/v1::Namespace\", \"core/v1::Pods\", \"core/v1::Service\", \"networking.k8s.io/v1::NetworkPolicy\"]\nKubeProxyReplacement:    False   [eth0 10.0.21.109 (Direct Routing), eth1 10.0.27.0]\nHost firewall:           Disabled\nCNI Chaining:            aws-cni\nCilium:                  Ok   1.14.2 (v1.14.2-a6748946)\nNodeMonitor:             Listening for events on 2 CPUs with 64x4096 of shared memory\nCilium health daemon:    Ok\nIPAM:                    IPv4: 1/254 allocated from 10.0.0.0/24,\nIPv4 BIG TCP:            Disabled\nIPv6 BIG TCP:            Disabled\nBandwidthManager:        Disabled\nHost Routing:            Legacy\nMasquerading:            Disabled\nController Status:       24/24 healthy\nProxy Status:            No managed proxy redirect\nGlobal Identity Range:   min 256, max 65535\nHubble:                  Ok          Current/Max Flows: 410/4095 (10.01%), Flows/s: 1.59   Metrics: Disabled\nEncryption:              Wireguard   [NodeEncryption: Disabled, cilium_wg0 (Pubkey: /yuqsZyG91AzVIkZ3AIq8qjQ0gGKQd6GWcRYh4LYpko=, Port: 51871, Peers: 1)]\nCluster health:                      Probe disabled\n</code></pre> </li> <li> <p>Open a shell inside the cilium container</p> <pre><code>kubectl -n kube-system exec -ti ds/cilium -- bash\n</code></pre> </li> <li> <p>Install <code>tcpdump</code></p> <pre><code>apt-get update\napt-get install -y tcpdump\n</code></pre> </li> <li> <p>Start a packet capture on <code>cilium_wg0</code> and verify you see payload in clear text, it means the traffic is encrypted with wireguard</p> <pre><code>tcpdump -A -c 40 -i cilium_wg0 | grep \"Welcome to nginx!\"\n</code></pre> <pre><code>tcpdump: verbose output suppressed, use -v[v]... for full protocol decode\nlistening on cilium_wg0, link-type RAW (Raw IP), snapshot length 262144 bytes\n&lt;title&gt;Welcome to nginx!&lt;/title&gt;\n&lt;h1&gt;Welcome to nginx!&lt;/h1&gt;\n40 packets captured\n40 packets received by filter\n0 packets dropped by kernel\n</code></pre> <p>Exit</p> <p>Exit the container shell by typing <code>exit</code> before continuing to next step</p> </li> <li> <p>Deploy the Cilium connectivity resources to check and evaluate connectivity:</p> <pre><code>kubectl create ns cilium-test\nkubectl apply -n cilium-test -f https://raw.githubusercontent.com/cilium/cilium/v1.14.1/examples/kubernetes/connectivity-check/connectivity-check.yaml\n</code></pre> <pre><code>deployment.apps/echo-a created\ndeployment.apps/echo-b created\ndeployment.apps/echo-b-host created\ndeployment.apps/pod-to-a created\ndeployment.apps/pod-to-external-1111 created\ndeployment.apps/pod-to-a-denied-cnp created\ndeployment.apps/pod-to-a-allowed-cnp created\ndeployment.apps/pod-to-external-fqdn-allow-google-cnp created\ndeployment.apps/pod-to-b-multi-node-clusterip created\ndeployment.apps/pod-to-b-multi-node-headless created\ndeployment.apps/host-to-b-multi-node-clusterip created\ndeployment.apps/host-to-b-multi-node-headless created\ndeployment.apps/pod-to-b-multi-node-nodeport created\ndeployment.apps/pod-to-b-intra-node-nodeport created\nservice/echo-a created\nservice/echo-b created\nservice/echo-b-headless created\nservice/echo-b-host-headless created\nciliumnetworkpolicy.cilium.io/pod-to-a-denied-cnp created\nciliumnetworkpolicy.cilium.io/pod-to-a-allowed-cnp created\nciliumnetworkpolicy.cilium.io/pod-to-external-fqdn-allow-google-cnp created\n</code></pre> </li> <li> <p>View the logs of any of the connectivity tests to view the results:</p> <pre><code>kubectl logs &lt;cilium test pod&gt; -n cilium-test\n</code></pre> <pre><code>\\{^_^}/ hi!\n\nLoading /default.json\nDone\n\nResources\nhttp://:8080/private\nhttp://:8080/public\n\nHome\nhttp://:8080\n\nType s + enter at any time to create a snapshot of the database\nWatching...\n\nGET /public 200 7.063 ms - 57\nGET /public 200 3.126 ms - 57\nGET /public 200 3.039 ms - 57\nGET /public 200 2.776 ms - 57\nGET /public 200 3.087 ms - 57\nGET /public 200 2.781 ms - 57\nGET /public 200 2.977 ms - 57\nGET /public 200 2.596 ms - 57\nGET /public 200 2.991 ms - 57\nGET /public 200 2.708 ms - 57\nGET /public 200 3.066 ms - 57\nGET /public 200 2.616 ms - 57\nGET /public 200 2.875 ms - 57\nGET /public 200 2.689 ms - 57\nGET /public 200 2.800 ms - 57\nGET /public 200 2.556 ms - 57\nGET /public 200 3.238 ms - 57\nGET /public 200 2.538 ms - 57\nGET /public 200 2.890 ms - 57\nGET /public 200 2.666 ms - 57\nGET /public 200 2.729 ms - 57\nGET /public 200 2.580 ms - 57\nGET /public 200 2.919 ms - 57\nGET /public 200 2.630 ms - 57\nGET /public 200 2.857 ms - 57\nGET /public 200 2.716 ms - 57\nGET /public 200 1.693 ms - 57\nGET /public 200 2.715 ms - 57\nGET /public 200 2.729 ms - 57\nGET /public 200 2.655 ms - 57\n</code></pre> </li> </ol>"},{"location":"patterns/wireguard-with-cilium/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/vpc-lattice/client-server-communication/","title":"Amazon VPC Lattice Client-server Communication","text":""},{"location":"patterns/vpc-lattice/client-server-communication/#amazon-vpc-lattice-simple-client-to-server-communication","title":"Amazon VPC Lattice - Simple Client to Server Communication","text":"<p>This pattern demonstrates how to expose an EKS cluster hosted application to an internal consumer through Amazon VPC Lattice.</p>"},{"location":"patterns/vpc-lattice/client-server-communication/#scenario","title":"Scenario","text":"<p>With this solution we showcase how to configure Amazon VPC Lattice using the AWS Gateway API Controller in order to manage Amazon VPC Lattice resources through native K8S Gateway API objects. This pattern deploys two distinct VPCs with a client application running in one of them and a server application in the other. The server application is deployed inside an EKS cluster and made exposed to the client application through Amazon VPC Lattice which establishes connectivity between the two applications. Further we demonstrate how to configure a custom domain name for the exposed service using Amazon Route53 and the external-dns project.</p> <p></p>"},{"location":"patterns/vpc-lattice/client-server-communication/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/vpc-lattice/client-server-communication/#validate","title":"Validate","text":"<p>In order to test the connectivity between the client and server, please follow the steps outlined below:</p> <ol> <li>Login to the management console of your AWS account and navigate to the EC2 service</li> <li>Select your the EC2 Instance with the name client, click Connect, choose Session Manager and click Connect</li> <li> <p>Within the console test the connectivity to the server application by entering the following command:</p> <pre><code>$ curl -i http://server.example.com\nHTTP/1.1 200 OK\ndate: Thu, 14 Dec 2023 08:29:39 GMT\ncontent-length: 54\ncontent-type: text/plain; charset=utf-8\n\nRequesting to Pod(server-6f487b9bcd-5qm4v): server pod\n</code></pre> </li> </ol>"},{"location":"patterns/vpc-lattice/client-server-communication/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"snippets/ipv4-prefix-delegation/","title":"IPv4 Prefix Delegation","text":"<p>The configuration snippet below shows how to enable prefix delegation to increase the number of available IP addresses on the provisioned EC2 nodes.</p> <ul> <li>Documentation</li> <li>Blog post</li> </ul>"},{"location":"snippets/ipv4-prefix-delegation/#vpc-cni-configuration","title":"VPC CNI Configuration","text":"<p>In this example, the <code>vpc-cni</code> addon is configured using <code>before_compute = true</code>. This is done to ensure the <code>vpc-cni</code> is created and updated before any EC2 instances are created so that the desired settings have applied before they will be referenced. With this configuration, you will now see that nodes created will have <code>--max-pods 110</code> configured do to the use of prefix delegation being enabled on the <code>vpc-cni</code>.</p> <p>If you find that your nodes are not being created with the correct number of max pods (i.e. - for <code>m5.large</code>, if you are seeing a max pods of 29 instead of 110), most likely the <code>vpc-cni</code> was not configured before the EC2 instances.</p> <pre><code>module \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n\n  # Truncated for brevity\n  ...\n\n  cluster_addons = {\n    vpc-cni = {\n      before_compute = true\n      most_recent    = true # To ensure access to the latest settings provided\n      configuration_values = jsonencode({\n        env = {\n          ENABLE_PREFIX_DELEGATION = \"true\"\n          WARM_PREFIX_TARGET       = \"1\"\n        }\n      })\n    }\n  }\n\n  ...\n}\n</code></pre> <p>When enabled, inspect one of the <code>aws-node-*</code> (AWS VPC CNI) pods to ensure prefix delegation is enabled and warm prefix target is 1:</p> <pre><code>kubectl describe ds -n kube-system aws-node | grep ENABLE_PREFIX_DELEGATION: -A 3\n</code></pre> <p>Output should look similar to below (truncated for brevity):</p> <pre><code>    ENABLE_PREFIX_DELEGATION:               true # &lt;- this should be set to true\n    WARM_ENI_TARGET:                        1\n    WARM_PREFIX_TARGET:                     1 # &lt;- this should be set to 1\n    ...\n</code></pre>"},{"location":"snippets/vpc-cni-custom-networking/","title":"VPC CNI Custom Networking","text":"<p>Custom networking addresses the IP exhaustion issue by assigning the node and Pod IPs from secondary VPC address spaces (CIDR). Custom networking support supports ENIConfig custom resource. The ENIConfig includes an alternate subnet CIDR range (carved from a secondary VPC CIDR), along with the security group(s) that the Pods will belong to. When custom networking is enabled, the VPC CNI creates secondary ENIs in the subnet defined under ENIConfig. The CNI assigns Pods an IP addresses from a CIDR range defined in a ENIConfig CRD.</p> <p>Since the primary ENI is not used by custom networking, the maximum number of Pods you can run on a node is lower. The host network Pods continue to use IP address assigned to the primary ENI. Additionally, the primary ENI is used to handle source network translation and route Pods traffic outside the node.</p> <ul> <li>Documentation</li> <li>Best Practices Guide</li> </ul>"},{"location":"snippets/vpc-cni-custom-networking/#vpc-cni-configuration","title":"VPC CNI Configuration","text":"<p>In this example, the <code>vpc-cni</code> addon is configured using <code>before_compute = true</code>. This is done to ensure the <code>vpc-cni</code> is created and updated before any EC2 instances are created so that the desired settings have applied before they will be referenced. With this configuration, you will now see that nodes created will have <code>--max-pods 110</code> configured do to the use of prefix delegation being enabled on the <code>vpc-cni</code>.</p> <p>If you find that your nodes are not being created with the correct number of max pods (i.e. - for <code>m5.large</code>, if you are seeing a max pods of 29 instead of 110), most likely the <code>vpc-cni</code> was not configured before the EC2 instances.</p>"},{"location":"snippets/vpc-cni-custom-networking/#components","title":"Components","text":"<p>To enable VPC CNI custom networking, you must configuring the following components:</p> <ol> <li> <p>Create a VPC with additional CIDR block associations. These additional CIDR blocks will be used to create subnets for the VPC CNI custom networking:</p> <pre><code>module \"vpc\" {\nsource  = \"terraform-aws-modules/vpc/aws\"\n\n# Truncated for brevity\n...\n\nsecondary_cidr_blocks = [local.secondary_vpc_cidr] # can add up to 5 total CIDR blocks\n\nazs = local.azs\nprivate_subnets = concat(\n   [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)],\n   [for k, v in local.azs : cidrsubnet(local.secondary_vpc_cidr, 2, k)]\n)\n\n...\n}\n</code></pre> </li> <li> <p>Specify the VPC CNI custom networking configuration in the <code>vpc-cni</code> addon configuration:</p> <pre><code>module \"eks\" {\nsource  = \"terraform-aws-modules/eks/aws\"\n\n# Truncated for brevity\n...\n\ncluster_addons = {\n   vpc-cni = {\n      before_compute = true\n      most_recent    = true # To ensure access to the latest settings provided\n      configuration_values = jsonencode({\n      env = {\n         AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG = \"true\"\n         ENI_CONFIG_LABEL_DEF               = \"topology.kubernetes.io/zone\"\n      })\n   }\n}\n\n...\n}\n</code></pre> </li> <li> <p>Create the <code>ENIConfig</code> custom resource for each subnet that you want to deploy pods into:</p> <pre><code>resource \"kubectl_manifest\" \"eni_config\" {\nfor_each = zipmap(local.azs, slice(module.vpc.private_subnets, 3, 6))\n\nyaml_body = yamlencode({\n   apiVersion = \"crd.k8s.amazonaws.com/v1alpha1\"\n   kind       = \"ENIConfig\"\n   metadata = {\n      name = each.key\n   }\n   spec = {\n      securityGroups = [\n      module.eks.node_security_group_id,\n      ]\n      subnet = each.value\n   }\n})\n}\n</code></pre> </li> </ol> <p>Once those settings have been successfully applied, you can verify if custom networking is enabled correctly by inspecting one of the <code>aws-node-*</code> (AWS VPC CNI) pods:</p> <pre><code>kubectl describe pod aws-node-ttg4h -n kube-system\n\n# Output should look similar below (truncated for brevity)\n  Environment:\n    ADDITIONAL_ENI_TAGS:                    {}\n    AWS_VPC_CNI_NODE_PORT_SUPPORT:          true\n    AWS_VPC_ENI_MTU:                        9001\n    AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:     false\n    AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG:     true # &lt;- this should be set to true\n    AWS_VPC_K8S_CNI_EXTERNALSNAT:           false\n    AWS_VPC_K8S_CNI_LOGLEVEL:               DEBUG\n    ...\n</code></pre>"},{"location":"v4-to-v5/addons/","title":"Migrate to EKS Blueprints Addons Module","text":"<p>Please consult the docs/v4-to-v5/example directory for reference configurations. If you find a bug, please open an issue with supporting configuration to reproduce.</p>"},{"location":"v4-to-v5/addons/#this-guide-is-under-active-development","title":"\u26a0\ufe0f This guide is under active development.","text":""},{"location":"v4-to-v5/addons/#list-of-backwards-incompatible-changes","title":"List of backwards incompatible changes","text":"<p>-</p>"},{"location":"v4-to-v5/addons/#additional-changes","title":"Additional changes","text":""},{"location":"v4-to-v5/addons/#added","title":"Added","text":"<p>-</p>"},{"location":"v4-to-v5/addons/#modified","title":"Modified","text":"<p>-</p>"},{"location":"v4-to-v5/addons/#removed","title":"Removed","text":"<p>-</p>"},{"location":"v4-to-v5/addons/#variable-and-output-changes","title":"Variable and output changes","text":"<ol> <li> <p>Removed variables:</p> <p>-</p> </li> <li> <p>Renamed variables:</p> <p>-</p> </li> <li> <p>Added variables:</p> <p>-</p> </li> <li> <p>Removed outputs:</p> <p>-</p> </li> <li> <p>Renamed outputs:</p> <p>-</p> </li> <li> <p>Added outputs:</p> <p>-</p> </li> </ol>"},{"location":"v4-to-v5/addons/#upgrade-migrations","title":"Upgrade Migrations","text":""},{"location":"v4-to-v5/addons/#before-v4x-example","title":"Before - v4.x Example","text":"<pre><code>module \"eks_blueprints_addons\" {\n  source = \"github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1\"\n\n  eks_cluster_id       = module.eks.cluster_name\n  eks_cluster_endpoint = module.eks.cluster_endpoint\n  eks_oidc_provider    = module.eks.oidc_provider\n  eks_cluster_version  = module.eks.cluster_version\n\n  # TODO\n\n}\n</code></pre>"},{"location":"v4-to-v5/addons/#after-v5x-example","title":"After - v5.x Example","text":"<pre><code>module \"eks_blueprints_addons\" {\n  source  = \"aws-ia/eks-blueprints-addons/aws\"\n  version = \"~&gt; 1.0\"\n\n  cluster_name      = module.eks.cluster_name\n  cluster_endpoint  = module.eks.cluster_endpoint\n  cluster_version   = module.eks.cluster_version\n  oidc_provider_arn = module.eks.oidc_provider_arn\n\n  # TODO\n\n}\n</code></pre>"},{"location":"v4-to-v5/addons/#diff-of-before-vs-after","title":"Diff of Before vs After","text":"<pre><code>module \"eks_blueprints_addons\" {\n-  source = \"github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1\"\n+  source  = \"aws-ia/eks-blueprints-addons/aws\"\n+  version = \"~&gt; 1.0\"\n\n  # TODO\n}\n</code></pre>"},{"location":"v4-to-v5/addons/#state-move-commands","title":"State Move Commands","text":"<p>In conjunction with the changes above, users can elect to move their external capacity provider(s) under this module using the following move command. Command is shown using the values from the example shown above, please update to suit your configuration names:</p> <pre><code>terraform state mv 'xxx' 'yyy'\n</code></pre>"},{"location":"v4-to-v5/cluster/","title":"Migrate to EKS Module v19.x","text":"<p>Please consult the docs/v4-to-v5/example directory for reference configurations. If you find a bug, please open an issue with supporting configuration to reproduce.</p>"},{"location":"v4-to-v5/cluster/#backwards-incompatible-changes","title":"Backwards incompatible changes","text":"<ul> <li>The cluster module provided in EKS Blueprints is being removed entirely from the project. Instead, users are encouraged to use the <code>terraform-aws-eks</code> module for creating and managing their EKS cluster in Terraform.</li> <li>The KMS module provided in EKS Blueprints has been removed. Users can leverage the KMS creation/management functionality provided by the <code>terraform-aws-eks</code> module or utilize the standalone <code>terraform-aws-kms</code> module.</li> <li>The EMR on EKS module provided in EKS Blueprints has been removed. Instead, users are encouraged to use the <code>terraform-aws-emr</code> virtual cluster sub-module for creating and managing their EMR on EKS virtual cluster in Terraform.</li> <li>The teams multi-tenancy module provided in EKS Blueprints has been removed. Instead, users are encouraged to use the <code>terraform-aws-eks-blueprints-teams</code> module for creating and managing their multi-tenancy constructions within their EKS clusters in Terraform.</li> </ul>"},{"location":"v4-to-v5/cluster/#additional-changes","title":"Additional changes","text":""},{"location":"v4-to-v5/cluster/#added","title":"Added","text":"<ul> <li>N/A</li> </ul>"},{"location":"v4-to-v5/cluster/#modified","title":"Modified","text":"<ul> <li>N/A</li> </ul>"},{"location":"v4-to-v5/cluster/#removed","title":"Removed","text":"<ul> <li>All noted above under <code>Backwards incompatible changes</code></li> </ul>"},{"location":"v4-to-v5/cluster/#variable-and-output-changes","title":"Variable and output changes","text":"<p>Since the change is to replace the EKS Blueprints cluster module with the <code>terraform-aws-eks</code> module, there aren't technically any variable or output changes other than their removal. Please consult the <code>terraform-aws-eks</code> module for its respective variables/outputs.</p> <ol> <li> <p>Removed variables:</p> <ul> <li>All</li> </ul> </li> <li> <p>Renamed variables:</p> <ul> <li>None</li> </ul> </li> <li> <p>Added variables:</p> <ul> <li>None</li> </ul> </li> <li> <p>Removed outputs:</p> <ul> <li>All</li> </ul> </li> <li> <p>Renamed outputs:</p> <ul> <li>None</li> </ul> </li> <li> <p>Added outputs:</p> <ul> <li>None</li> </ul> </li> </ol>"},{"location":"v4-to-v5/cluster/#upgrade-migrations","title":"Upgrade Migrations","text":""},{"location":"v4-to-v5/cluster/#before-v432-example","title":"Before - v4.32 Example","text":"<pre><code>module \"eks\" {\n  source = \"github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.32.1\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.26\"\n\n  vpc_id                          = module.vpc.vpc_id\n  private_subnet_ids              = module.vpc.private_subnets\n  cluster_endpoint_private_access = true\n\n  map_roles = [\n    {\n      rolearn  = data.aws_caller_identity.current.arn\n      username = \"me\"\n      groups   = [\"system:masters\"]\n    },\n  ]\n\n  managed_node_groups = {\n    managed = {\n      node_group_name = \"managed\"\n      instance_types  = [\"m5.large\"]\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n      k8s_labels = {\n        Which = \"managed\"\n      }\n    }\n  }\n\n  fargate_profiles = {\n    fargate = {\n      fargate_profile_name = \"fargate\"\n      fargate_profile_namespaces = [{\n        namespace = \"default\"\n\n        k8s_labels = {\n          Which = \"fargate\"\n        }\n      }]\n      subnet_ids = module.vpc.private_subnets\n    }\n  }\n\n  self_managed_node_groups = {\n    self_managed = {\n      node_group_name    = \"self_managed\"\n      instance_type      = \"m5.large\"\n      launch_template_os = \"amazonlinux2eks\"\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n      k8s_labels = {\n        Which = \"self-managed\"\n      }\n    }\n  }\n\n  tags = {\n    Blueprint  = local.name\n    GithubRepo = \"github.com/aws-ia/terraform-aws-eks-blueprints\"\n  }\n}\n</code></pre>"},{"location":"v4-to-v5/cluster/#after-v50-example","title":"After - v5.0 Example","text":"<p>Any of the values that are marked with <code># Backwards compat</code> are provided to demonstrate configuration level changes to reduce the number of Terraform changes when migrating to the EKS module.</p> <pre><code>module \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n  version = \"~&gt; 19.13\"\n\n  cluster_name                   = local.name\n  cluster_version                = \"1.26\"\n  cluster_endpoint_public_access = true # Backwards compat\n  cluster_enabled_log_types      = [\"api\", \"audit\", \"authenticator\", \"controllerManager\", \"scheduler\"] # Backwards compat\n\n  iam_role_name            = \"${local.name}-cluster-role\" # Backwards compat\n  iam_role_use_name_prefix = false                        # Backwards compat\n\n  kms_key_aliases = [local.name] # Backwards compat\n\n  vpc_id     = module.vpc.vpc_id\n  subnet_ids = module.vpc.private_subnets\n\n  manage_aws_auth_configmap = true\n  aws_auth_roles = [\n    {\n      rolearn  = data.aws_caller_identity.current.arn\n      username = \"me\"\n      groups   = [\"system:masters\"]\n    },\n  ]\n\n  eks_managed_node_groups = {\n    managed = {\n      iam_role_name              = \"${local.name}-managed\" # Backwards compat\n      iam_role_use_name_prefix   = false                   # Backwards compat\n      use_custom_launch_template = false                   # Backwards compat\n\n      instance_types = [\"m5.large\"]\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n      labels = {\n        Which = \"managed\"\n      }\n    }\n  }\n\n  fargate_profiles = {\n    fargate = {\n      iam_role_name            = \"${local.name}-fargate\" # Backwards compat\n      iam_role_use_name_prefix = false                   # Backwards compat\n\n      selectors = [{\n        namespace = \"default\"\n        labels = {\n          Which = \"fargate\"\n        }\n      }]\n    }\n  }\n\n  self_managed_node_groups = {\n    self_managed = {\n      name            = \"${local.name}-self_managed\" # Backwards compat\n      use_name_prefix = false                        # Backwards compat\n\n      iam_role_name            = \"${local.name}-self_managed\" # Backwards compat\n      iam_role_use_name_prefix = false                        # Backwards compat\n\n      launch_template_name            = \"self_managed-${local.name}\" # Backwards compat\n      launch_template_use_name_prefix = false                        # Backwards compat\n\n      instance_type = \"m5.large\"\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n      labels = {\n        Which = \"self-managed\"\n      }\n    }\n  }\n\n  tags = {\n    Blueprint  = local.name\n    GithubRepo = \"github.com/aws-ia/terraform-aws-eks-blueprints\"\n  }\n}\n</code></pre>"},{"location":"v4-to-v5/cluster/#diff-of-before-vs-after","title":"Diff of Before vs After","text":"<pre><code>module \"eks\" {\n-  source = \"github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.32.1\"\n+  source  = \"terraform-aws-modules/eks/aws\"\n+  version = \"~&gt; 19.13\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.26\"\n\n  vpc_id                          = module.vpc.vpc_id\n  private_subnet_ids              = module.vpc.private_subnets\n+  cluster_endpoint_public_access  = true\n-  cluster_endpoint_private_access = true\n\n-  map_roles = [\n+  aws_auth_roles = [\n    {\n      rolearn  = data.aws_caller_identity.current.arn\n      username = \"me\"\n      groups   = [\"system:masters\"]\n    },\n  ]\n\n-  managed_node_groups = {\n+  eks_managed_node_groups = {\n    managed = {\n-      node_group_name = \"managed\"\n      instance_types  = [\"m5.large\"]\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n-      k8s_labels = {\n+      labels = {\n        Which = \"managed\"\n      }\n    }\n  }\n\n  fargate_profiles = {\n    fargate = {\n-      fargate_profile_name = \"fargate\"\n-      fargate_profile_namespaces = [{\n+      selectors = [{\n        namespace = \"default\"\n\n-        k8s_labels = {\n+        labels = {\n          Which = \"fargate\"\n        }\n      }]\n-      subnet_ids = module.vpc.private_subnets\n    }\n  }\n\n  self_managed_node_groups = {\n    self_managed = {\n-      node_group_name    = \"self_managed\"\n      instance_type      = \"m5.large\"\n-      launch_template_os = \"amazonlinux2eks\"\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n-      k8s_labels = {\n+      labels = {\n        Which = \"self-managed\"\n      }\n    }\n  }\n\n  tags = {\n    Blueprint  = local.name\n    GithubRepo = \"github.com/aws-ia/terraform-aws-eks-blueprints\"\n  }\n}\n</code></pre>"},{"location":"v4-to-v5/cluster/#state-move-commands","title":"State Move Commands","text":"<p>The following Terraform state move commands are provided to aid in migrating the control plane and data plane components.</p> <pre><code># This is not removing the configmap from the cluster -\n# it will be adopted by the new module\nterraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]'\n\n# Cluster\nterraform state mv 'module.eks.module.aws_eks.aws_eks_cluster.this[0]' 'module.eks.aws_eks_cluster.this[0]'\n\n# Cluster IAM role\nterraform state mv 'module.eks.module.aws_eks.aws_iam_role.this[0]' 'module.eks.aws_iam_role.this[0]'\nterraform state mv 'module.eks.module.aws_eks.aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSClusterPolicy\"]' 'module.eks.aws_iam_role_policy_attachment.this[\"AmazonEKSClusterPolicy\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSVPCResourceController\"]' 'module.eks.aws_iam_role_policy_attachment.this[\"AmazonEKSVPCResourceController\"]'\n\n# Cluster primary security group tags\n# Note: This will depend on the tags applied to the module - here we\n#       are demonstrating the two tags used in the configuration above\nterraform state mv 'module.eks.module.aws_eks.aws_ec2_tag.cluster_primary_security_group[\"Blueprint\"]' 'module.eks.aws_ec2_tag.cluster_primary_security_group[\"Blueprint\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_ec2_tag.cluster_primary_security_group[\"GithubRepo\"]' 'module.eks.aws_ec2_tag.cluster_primary_security_group[\"GithubRepo\"]'\n\n# Cluster security group\nterraform state mv 'module.eks.module.aws_eks.aws_security_group.cluster[0]' 'module.eks.aws_security_group.cluster[0]'\n\n# Cluster security group rules\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.cluster[\"ingress_nodes_443\"]' 'module.eks.aws_security_group_rule.cluster[\"ingress_nodes_443\"]'\n\n# Node security group\nterraform state mv 'module.eks.module.aws_eks.aws_security_group.node[0]' 'module.eks.aws_security_group.node[0]'\n\n# Node security group rules\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.node[\"ingress_cluster_443\"]' 'module.eks.aws_security_group_rule.node[\"ingress_cluster_443\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.node[\"ingress_cluster_kubelet\"]' 'module.eks.aws_security_group_rule.node[\"ingress_cluster_kubelet\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.node[\"ingress_self_coredns_tcp\"]' 'module.eks.aws_security_group_rule.node[\"ingress_self_coredns_tcp\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.node[\"ingress_self_coredns_udp\"]' 'module.eks.aws_security_group_rule.node[\"ingress_self_coredns_udp\"]'\n\n# OIDC provider\nterraform state mv 'module.eks.module.aws_eks.aws_iam_openid_connect_provider.oidc_provider[0]' 'module.eks.aws_iam_openid_connect_provider.oidc_provider[0]'\n\n# Fargate profile(s)\n# Note: This demonstrates migrating one profile that is stored under the\n#       key `fargate` in the module definition. The same set of steps would\n#       need to be performed for each profile, changing only the key name\nterraform state mv 'module.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_eks_fargate_profile.eks_fargate' 'module.eks.module.fargate_profile[\"fargate\"].aws_eks_fargate_profile.this[0]'\nterraform state mv 'module.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_iam_role.fargate[0]' 'module.eks.module.fargate_profile[\"fargate\"].aws_iam_role.this[0]'\nterraform state mv 'module.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_iam_role_policy_attachment.fargate_pod_execution_role_policy[\"arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy\"]' 'module.eks.module.fargate_profile[\"fargate\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy\"]'\n\n# Managed nodegroup(s)\n# Note: This demonstrates migrating one nodegroup that is stored under the\n#       key `managed` in the module definition. The same set of steps would\n#       need to be performed for each nodegroup, changing only the key name\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_eks_node_group.managed_ng' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_eks_node_group.this[0]'\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role.managed_ng[0]' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_iam_role.this[0]'\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role_policy_attachment.managed_ng[\"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\"]' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\"]'\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role_policy_attachment.managed_ng[\"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy\"]' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy\"]'\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role_policy_attachment.managed_ng[\"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy\"]' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy\"]'\n\n# Self-managed nodegroup(s)\n# Note: This demonstrates migrating one nodegroup that is stored under the\n#       key `self_managed` in the module definition. The same set of steps would\n#       need to be performed for each nodegroup, changing only the key name\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_autoscaling_group.self_managed_ng' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_autoscaling_group.this[0]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_instance_profile.self_managed_ng[0]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_instance_profile.this[0]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role.self_managed_ng[0]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_role.this[0]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role_policy_attachment.self_managed_ng[\"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\"]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\"]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role_policy_attachment.self_managed_ng[\"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy\"]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy\"]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role_policy_attachment.self_managed_ng[\"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy\"]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy\"]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].module.launch_template_self_managed_ng.aws_launch_template.this[\"self-managed-node-group\"]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_launch_template.this[0]'\n\n# Secrets KMS key\nterraform state mv ' module.eks.module.kms[0].aws_kms_key.this' 'module.eks.module.kms.aws_kms_key.this[0]'\nterraform state mv 'module.eks.module.kms[0].aws_kms_alias.this' 'module.eks.module.kms.aws_kms_alias.this[\"migration\"]'\n\n# Cloudwatch Log Group\nterraform import 'module.eks.aws_cloudwatch_log_group.this[0]' /aws/eks/migration/cluster\n</code></pre>"},{"location":"v4-to-v5/cluster/#removed-resources","title":"Removed Resources","text":"<p>The following resources will be destroyed when migrating from EKS Blueprints v4.32.1 cluster to the v19.x EKS cluster:</p> <pre><code>module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_instance_profile.managed_ng[0]\n</code></pre> <ul> <li>It is not directly used and was intended to be used by Karpenter. The https://github.com/aws-ia/terraform-aws-eks-blueprints-addons module provides its own resource for creating an IAM instance profile for Karpenter</li> </ul> <pre><code>module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role_policy_attachment.managed_ng[\"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore\"]\n</code></pre> <ul> <li>IAM policy is not required by EKS - users can re-add this policy at their discretion</li> </ul> <pre><code>module.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_iam_policy.cwlogs[0]\nmodule.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_iam_role_policy_attachment.cwlogs[0]\n</code></pre> <ul> <li>Policy is not required by EKS</li> </ul> <pre><code>module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role_policy_attachment.self_managed_ng[\"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore\"]\n</code></pre> <ul> <li>IAM policy is not required by EKS - users can re-add this policy at their discretion</li> </ul>"},{"location":"v4-to-v5/motivation/","title":"Direction for v5 of Terraform EKS Blueprints","text":""},{"location":"v4-to-v5/motivation/#what-has-worked","title":"What Has Worked","text":"<ul> <li> <p>EKS Blueprints was started to make it easier for customers to adopt Amazon Elastic Kubernetes Service (EKS) in a shorter period of time. The project has been quite successful in this regard - hearing from customers stating that EKS Blueprints has helped them get from zero to one or more clusters running with applications in less than 1-2 weeks.</p> </li> <li> <p>EKS Blueprints has also been successful in providing working examples to users that demonstrate common architectural patterns and workload solutions. Some popular examples include:</p> </li> <li>Spark on EKS</li> <li>Karpenter on EKS Fargate</li> <li>Transparent encryption with Wireguard and Cilium</li> <li>Fully serverless cluster with EKS Fargate</li> </ul>"},{"location":"v4-to-v5/motivation/#what-has-not","title":"What Has Not","text":"<ul> <li> <p>Scaling and managing addons that are created through EKS Blueprints. With almost 1,200 projects on the CNCF roadmap, the number of various ways and methods that a project allows for deploying onto a cluster (i.e. - Datadog offers 5 different Helm charts for its service, Prometheus hosts over 30 Helm charts for its services), as well as the number of different tools used to provision addons (i.e. - Terraform, ArgoCD, FluxCD, etc.), supporting both the number of addons and their different forms has been extremely challenging for the team. In addition to managing just the sheer number of addons, supporting the different configurations that users wish to have exposed in conjunction with testing and validating those various configurations is only compounded by the number of addons and their methods of creation.</p> </li> <li> <p>Managing resources provisioned on the cluster using Terraform. Terraform is a fantastic tool for provisioning infrastructure and it is the tool of choice for many customers when it comes to creating resources in AWS. However, there are a number of downsides with Terraform when it comes to provisioning resources on a Kubernetes cluster. These include:</p> </li> <li> <p>Ordering of dependencies when relationships live outside of Terraform's HCL syntax. Terraform wants to evaluate the current state of what it controls and be able to plan a series of actions to align the current state with the desired state in one action. It does this once for each <code>terraform plan</code> or <code>terraform apply</code>, and if any issues are encountered, it simply fails and halts execution. When Terraform cannot infer the ordering of dependencies across resources (i.e. - through passing outputs of parent resources to arguments of child resources using the Terraform <code>&lt;resource&gt;.&lt;name&gt;.&lt;attribute&gt;</code> syntax), it will view this as no relationship between the resources and attempt to execute their provisioning in parallel and asynchronously. Any resources that are left waiting for a dependency will eventually timeout and fail, causing Terraform itself to timeout and fail the apply. This is where the reconciliation loop of a Kubernetes controller or operator on the cluster is better suited - continuously trying to reconcile the state over and over again as dependencies are eventually resolved. (To be clear - the issue of dependency ordering still exists, but the controller/operator will keep retrying and on each retry, some resources will succeed which will move the execution along with each cycle until everything is fully deployed. Terraform could do this if it kept re-trying, but it does not do this today)</p> </li> <li> <p>Publicly exposing access to the EKS endpoints in order to provision resources defined outside of the VPC onto the cluster. When using Terraform, the resource provisioning operation is a \"push\" model where Terraform will send requests to the EKS API Server to create resources. Coupled with the fact that the Terraform operation typically resides outside of the VPC where the cluster is running, this results in users enabling public access to the EKS endpoints to provision resources. However, the more widely accepted approach by the Kubernetes community has been the adoption of GitOps which uses a \"pull\" based model, where an operator or controller running on the cluster will pull the resource definitions from a Git repository and reconcile state from within the cluster itself. This approach is more secure as it does not require public access to the EKS endpoints and instead relies on the cluster's internal network to communicate with the EKS API Server.</p> </li> <li> <p>The nesting of multiple sub-modules in conjunction with the necessity to even require a module to be able to support an addon. When we compare and contrast the Terraform approach to addons versus the GitOps approach, the Terraform approach has a glaring disadvantage - the need to create a module that wraps the addon's Helm chart in order to provision the addon via Terraform. As opposed to the GitOps approach, where users simply consume the charts from where they are stored as needed. This creates a bottleneck on the team to review, test, and validate each new addon as well as the overhead then added for maintaining and updating those addons going forward. This also opens up more areas where breaking changes are encountered which is compounded by the fact that Terraform addons are grouped under an \"umbrella\" module which obfuscates versioning.</p> </li> <li> <p>Being able to support a combination of various tools, modules, frameworks, etc., to meet the needs of customers. The <code>terraform-aws-eks</code> was created long before EKS Blueprints, and many customers had already adopted this module for creating their clusters. In addition, Amazon has since adopted the <code>eksctl</code> as the official CLI for Amazon EKS. When EKS Blueprints was first announced, many customers raised questions asking if they needed to abandon their current clusters created through those other tools in order to adopt EKS Blueprints. The answer is no - users can and should be able to use their existing clusters while EKS Blueprints can help augment that process through its supporting modules (addons, teams, etc.). This left the team with the question - why create a Terraform module for creating an EKS cluster when the <code>terraform-aws-eks</code> already exists and the EKS Blueprints implementation already uses that module for creating the control plane and security groups?</p> </li> </ul>"},{"location":"v4-to-v5/motivation/#what-is-changing","title":"What Is Changing","text":"<p>The direction for EKS Blueprints in v5 will shift from providing an all-encompassing, monolithic \"framework\" and instead focus more on how users can organize a set of modular components to create the desired solution on Amazon EKS. This will allow customers to use the components of their choosing in a way that is more familiar to them and their organization instead of having to adopt and conform to a framework.</p> <p>With this shift in direction, the cluster definition will be removed from the project and instead examples will reference the <code>terraform-aws-eks</code> module for cluster creation. The remaining modules will be moved out to their own respective repositories as standalone projects. This leaves the EKS Blueprint project as the canonical place where users can receive guidance on how to configure their clusters to meet a desired architecture, how best to setup their clusters following well-architected practices, as well as references on the various ways that different workloads can be deployed on Amazon EKS.</p>"},{"location":"v4-to-v5/motivation/#notable-changes","title":"Notable Changes","text":"<ol> <li>EKS Blueprints will remove its Amazon EKS cluster Terraform module components (control plane, EKS managed node group, self-managed node group, and Fargate profile modules) from the project. In its place, users are encouraged to utilize the <code>terraform-aws-eks</code> module which meets or exceeds nearly all of the functionality of the EKS Blueprints v4.x cluster module. This includes the Terraform code contained at the root of the project as well as the <code>aws-eks-fargate-profiles</code>, <code>aws-eks-managed-node-groups</code>, <code>aws-eks-self-managed-node-groups</code>, and <code>launch-templates</code> modules which will all be removed from the project.</li> <li>The <code>aws-kms</code> module will be removed entirely. This was consumed in the root project module for cluster secret encryption. In its place, users can utilize the KMS key creation functionality of the <code>terraform-aws-eks</code> module or the <code>terraform-aws-kms</code> module if they wish to control the key separately from the cluster itself.</li> <li>The <code>emr-on-eks</code> module will be removed entirely; its replacement can be found in the new external module <code>terraform-aws-emr</code>.</li> <li>The <code>irsa</code> and <code>helm-addon</code> modules will be removed entirely; we have released a new external module <code>terraform-aws-eks-blueprints-addon</code> that is available on the Terraform registry that replicates/replaces the functionality of these two modules. This will now allow users, as well as partners, to create their own addons that are not natively supported by EKS Blueprints more easily and following the same process as EKS Blueprints.</li> <li>The <code>aws-eks-teams</code> module will be removed entirely; its replacement will be the new external module <code>terraform-aws-eks-blueprints-teams</code> that incorporates the changes customers have been asking for in https://github.com/aws-ia/terraform-aws-eks-blueprints/issues/842</li> <li>The integration between Terraform and ArgoCD has been removed in the initial release of v5. The team is currently investigating better patterns and solutions in conjunction with the ArgoCD and FluxCD teams that will provide a better, more integrated experience when using a GitOps based approach for cluster management. This will be released in a future version of EKS Blueprints v5 and is tracked here</li> </ol>"},{"location":"v4-to-v5/motivation/#resulting-project-structure","title":"Resulting Project Structure","text":"<p>Previously under the v4.x structure, the EKS Blueprint project was comprised of various repositories across multiple AWS organizations that looked roughly like the following:</p>"},{"location":"v4-to-v5/motivation/#v4x-structure","title":"v4.x Structure","text":"<pre><code>\u251c\u2500\u2500 aws-ia/\n|   \u251c\u2500\u2500 terraform-aws-eks-ack-addons/\n|   \u2514\u2500\u2500 terraform-aws-eks-blueprints/\n|       \u251c\u2500\u2500 aws-auth-configmap.tf\n|       \u251c\u2500\u2500 data.tf\n|       \u251c\u2500\u2500 eks-worker.tf\n|       \u251c\u2500\u2500 locals.tf\n|       \u251c\u2500\u2500 main.tf\n|       \u251c\u2500\u2500 outputs.tf\n|       \u251c\u2500\u2500 variables.tf\n|       \u251c\u2500\u2500 versions.tf\n|       \u251c\u2500\u2500 examples/\n|       \u2514\u2500\u2500 modules\n|           \u251c\u2500\u2500 aws-eks-fargate-profiles/\n|           \u251c\u2500\u2500 aws-eks-managed-node-groups/\n|           \u251c\u2500\u2500 aws-eks-self-managed-node-groups/\n|           \u251c\u2500\u2500 aws-eks-teams/\n|           \u251c\u2500\u2500 aws-kms/\n|           \u251c\u2500\u2500 emr-on-eks/\n|           \u251c\u2500\u2500 irsa/\n|           \u251c\u2500\u2500 kubernetes-addons/\n|           \u2514\u2500\u2500 launch-templates/\n\u251c\u2500\u2500 awslabs/\n|   \u251c\u2500\u2500 crossplane-on-eks/\n|   \u2514\u2500\u2500 data-on-eks/\n\u2514\u2500\u2500 aws-samples/\n    \u251c\u2500\u2500 eks-blueprints-add-ons/   # Previously shared with the CDK based EKS Blueprints project\n    \u2514\u2500\u2500 eks-blueprints-workloads/ # Previously shared with the CDK based EKS Blueprints project\n</code></pre> <p>Under th new v5.x structure, the Terraform based EKS Blueprints project will be comprised of the following repositories:</p>"},{"location":"v4-to-v5/motivation/#v5x-structure","title":"v5.x Structure","text":"<pre><code>\u251c\u2500\u2500 aws-ia/\n|   \u251c\u2500\u2500 terraform-aws-eks-ack-addons/\n|   \u251c\u2500\u2500 terraform-aws-eks-blueprints/       # Will contain only example/blueprint implementations; no modules\n|   \u251c\u2500\u2500 terraform-aws-eks-blueprints-addon  # Module for creating Terraform based addon (IRSA + Helm chart)\n|   \u251c\u2500\u2500 terraform-aws-eks-blueprints-addons # Will contain a select set of addons supported by the EKS Blueprints\n|   \u2514\u2500\u2500 terraform-aws-eks-blueprints-teams  # Was previously `aws-eks-teams/` EKS Blueprint sub-module; updated based on customer feedback\n\u2514\u2500\u2500 awslabs/\n    \u251c\u2500\u2500 crossplane-on-eks/\n    \u2514\u2500\u2500 data-on-eks/        # Data related patterns that used to be located in `terraform-aws-eks-blueprints/` are now located here\n</code></pre>"},{"location":"v4-to-v5/motivation/#what-can-users-expect","title":"What Can Users Expect","text":"<p>With these changes, the team intends to provide a better experience for users of the Terraform EKS Blueprints project as well as new and improved reference architectures. Following the v5 changes, the team intends to:</p> <ol> <li>Improved quality of the examples provided - more information on the intent of the example, why it might be useful for users, what scenarios is the pattern applicable, etc. Where applicable, architectural diagrams and supporting material will be provided to highlight the intent of the example and how its constructed.</li> <li>A more clear distinction between a blueprint and a usage reference. For example - the Karpenter on EKS Fargate blueprint should demonstrate all of the various aspects that users should be aware of and consider in order to take full advantage of this pattern (recommended practices, observability, logging, monitoring, security, day 2 operations, etc.); this is what makes it a blueprint. In contrast, a usage reference would be an example that shows how users can pass configuration values to the Karpenter provisioner. This example is less focused on the holistic architecture and more focused on how one might configure Karpenter using the implementation. The EKS Blueprints repository will focus mostly on holistic architecture and patterns, and any usage references should be saved for the repository that contains that implementation definition (i.e. - the <code>terraform-aws-eks-blueprints-addons</code> repository where the addon implementation is defined).</li> <li>Faster, and more responsive feedback. The first part of this is going to be improved documentation on how to contribute which should help clarify whether a contribution is worthy and willing to be accepted by the team before any effort is spent by the contributor. However, the goal of v5 is to focus more on the value added benefits that EKS Blueprints was created to provide as opposed to simply mass producing Helm chart wrappers (addons) and trying to keep up with that operationally intensive process.</li> <li>Lastly, more examples and blueprints that demonstrate various architectures and workloads that run on top of Amazon EKS as well as integrations into other AWS services.</li> </ol>"},{"location":"v4-to-v5/teams/","title":"Migrate to EKS Blueprints Teams Module","text":"<p>Please consult the docs/v4-to-v5/example directory for reference configurations. If you find a bug, please open an issue with supporting configuration to reproduce.</p>"},{"location":"v4-to-v5/teams/#this-guide-is-under-active-development","title":"\u26a0\ufe0f This guide is under active development.","text":""},{"location":"v4-to-v5/teams/#list-of-backwards-incompatible-changes","title":"List of backwards incompatible changes","text":"<p>-</p>"},{"location":"v4-to-v5/teams/#additional-changes","title":"Additional changes","text":""},{"location":"v4-to-v5/teams/#added","title":"Added","text":"<p>-</p>"},{"location":"v4-to-v5/teams/#modified","title":"Modified","text":"<p>-</p>"},{"location":"v4-to-v5/teams/#removed","title":"Removed","text":"<p>-</p>"},{"location":"v4-to-v5/teams/#variable-and-output-changes","title":"Variable and output changes","text":"<ol> <li> <p>Removed variables:</p> <p>-</p> </li> <li> <p>Renamed variables:</p> <p>-</p> </li> <li> <p>Added variables:</p> <p>-</p> </li> <li> <p>Removed outputs:</p> <p>-</p> </li> <li> <p>Renamed outputs:</p> <p>-</p> </li> <li> <p>Added outputs:</p> <p>-</p> </li> </ol>"},{"location":"v4-to-v5/teams/#upgrade-migrations","title":"Upgrade Migrations","text":""},{"location":"v4-to-v5/teams/#before-v4x-example","title":"Before - v4.x Example","text":"<pre><code>module \"eks_blueprints\" {\n  source = \"github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.32.1\"\n\n  # TODO\n\n}\n</code></pre>"},{"location":"v4-to-v5/teams/#after-v5x-example","title":"After - v5.x Example","text":"<pre><code>module \"eks_blueprints_teams\" {\n  source  = \"aws-ia/eks-blueprints-teams/aws\"\n  version = \"~&gt; 1.0\"\n\n  # TODO\n\n}\n</code></pre>"},{"location":"v4-to-v5/teams/#diff-of-before-vs-after","title":"Diff of Before vs After","text":"<pre><code>module \"eks_blueprints_teams\" {\n-  source = \"github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.32.1\"\n+  source  = \"aws-ia/eks-blueprints-teams/aws\"\n+  version = \"~&gt; 1.0\"\n\n  # TODO\n}\n</code></pre>"},{"location":"v4-to-v5/teams/#state-move-commands","title":"State Move Commands","text":"<p>In conjunction with the changes above, users can elect to move their external capacity provider(s) under this module using the following move command. Command is shown using the values from the example shown above, please update to suit your configuration names:</p> <pre><code>terraform state mv 'xxx' 'yyy'\n</code></pre>"},{"location":"v4-to-v5/example/","title":"Migration - v4 to v5","text":""}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Overview","text":""},{"location":"#amazon-eks-blueprints-for-terraform","title":"Amazon EKS Blueprints for Terraform","text":"<p>Welcome to Amazon EKS Blueprints for Terraform!</p> <p>This project contains a collection of Amazon EKS cluster patterns implemented in Terraform that demonstrate how fast and easy it is for customers to adopt Amazon EKS. The patterns can be used by AWS customers, partners, and internal AWS teams to configure and manage complete EKS clusters that are fully bootstrapped with the operational software that is needed to deploy and operate workloads.</p>"},{"location":"#motivation","title":"Motivation","text":"<p>Kubernetes is a powerful and extensible container orchestration technology that allows you to deploy and manage containerized applications at scale. The extensible nature of Kubernetes also allows you to use a wide range of popular open-source tools in Kubernetes clusters. However, With the wide array of tooling and design choices available, configuring an EKS cluster that meets your organization\u2019s specific needs can take a significant amount of time. It involves integrating a wide range of open-source tools and AWS services as well as expertise in AWS and Kubernetes.</p> <p>AWS customers have asked for patterns that demonstrate how to integrate the landscape of Kubernetes tools and make it easy for them to provision complete, opinionated EKS clusters that meet specific application requirements. Customers can utilize EKS Blueprints to configure and deploy purpose built EKS clusters, and start onboarding workloads in days, rather than months.</p>"},{"location":"#consumption","title":"Consumption","text":"<p>EKS Blueprints for Terraform has been designed to be consumed in the following manners:</p> <ol> <li>Reference: Users can refer to the patterns and snippets provided to help guide them to their desired solution. Users will typically view how the pattern or snippet is configured to achieve the desired end result and then replicate that in their environment.</li> <li>Copy &amp; Paste: Users can copy and paste the patterns and snippets into their own environment, using EKS Blueprints as the starting point for their implementation. Users can then adapt the initial pattern to customize it to their specific needs.</li> </ol> <p>EKS Blueprints for Terraform are not intended to be consumed as-is directly from this project. In \"Terraform speak\" - the patterns and snippets provided in this repository are not designed to be consumed as a Terraform module. Therefore, the patterns provided only contain <code>variables</code> when certain information is required to deploy the pattern (i.e. - a Route53 hosted zone ID, or ACM certificate ARN) and generally use local variables. If you wish to deploy the patterns into a different region or with other changes, it is recommended that you make those modifications locally before applying the pattern. EKS Blueprints for Terraform will not expose variables and outputs in the same manner that Terraform modules follow in order to avoid confusion around the consumption model.</p> <p>However, we do have a number of Terraform modules that were created to support EKS Blueprints in addition to the community hosted modules. Please see the respective projects for more details on the modules constructed to support EKS Blueprints for Terraform; those projects are listed below.</p> <ul> <li><code>terraform-aws-eks-blueprint-addon</code> - (Note the singular form) Terraform module which can provision an addon using the Terraform <code>helm_release</code> resource in addition to an IAM role for service account (IRSA).</li> <li><code>terraform-aws-eks-blueprint-addons</code> - (Note the plural form) Terraform module which can provision multiple addons; both EKS addons using the <code>aws_eks_addon</code> resource as well as Helm chart based addons using the <code>terraform-aws-eks-blueprint-addon</code> module.</li> <li><code>terraform-aws-eks-blueprints-teams</code> - Terraform module that creates Kubernetes multi-tenancy resources and configurations, allowing both administrators and application developers to access only the resources which they are responsible for.</li> </ul>"},{"location":"#related-projects","title":"Related Projects","text":"<p>In addition to the supporting EKS Blueprints Terraform modules listed above, there are a number of related projects that users should be aware of:</p> <ol> <li> <p>GitOps</p> <ul> <li><code>terraform-aws-eks-ack-addons</code> -   Terraform module to deploy ACK controllers onto EKS clusters</li> <li><code>crossplane-on-eks</code> - Crossplane Blueprints is an open source repo to bootstrap Amazon EKS clusters and provision AWS resources using a library of Crossplane Compositions (XRs) with Composite Resource Definitions (XRDs).</li> </ul> </li> <li> <p>Data on EKS</p> <ul> <li><code>data-on-eks</code> - A collection of blueprints intended for data workloads on Amazon EKS.</li> <li><code>terraform-aws-eks-data-addons</code> - Terraform module to deploy multiple addons that are specific to data workloads on EKS clusters.</li> </ul> </li> <li> <p>Observability Accelerator</p> <ul> <li><code>terraform-aws-observability-accelerator</code> - A set of opinionated modules to help you set up observability for your AWS environments with AWS-managed observability services such as Amazon Managed Service for Prometheus, Amazon Managed Grafana, AWS Distro for OpenTelemetry (ADOT) and Amazon CloudWatch</li> </ul> </li> <li> <p>Karpenter Blueprints</p> </li> <li><code>karpenter-blueprints</code> - includes a list of common workload scenarios,    some of them go in depth with the explanation of why configuring Karpenter and Kubernetes objects in such a way is important.</li> </ol>"},{"location":"#terraform-caveats","title":"Terraform Caveats","text":"<p>EKS Blueprints for Terraform does not intend to teach users the recommended practices for Terraform nor does it offer guidance on how users should structure their Terraform projects. The patterns provided are intended to show users how they can achieve a defined architecture or configuration in a way that they can quickly and easily get up and running to start interacting with that pattern. Therefore, there are a few caveats users should be aware of when using EKS Blueprints for Terraform:</p> <ol> <li> <p>We recognize that most users will already have an existing VPC in a separate Terraform workspace. However, the patterns provided come complete with a VPC to ensure a stable, deployable example that has been tested and validated.</p> </li> <li> <p>Hashicorp does not recommend providing computed values in provider blocks , which means that the cluster configuration should be defined in a workspace separate from the resources deployed onto the cluster (i.e. - addons). However, to simplify the pattern experience, we have defined everything in one workspace and provided instructions to provision the patterns using a targeted apply approach. Users are encouraged to investigate a Terraform project structure that suites their needs; EKS Blueprints for Terraform does not have an opinion in this matter and will defer to Hashicorp's guidance.</p> </li> <li> <p>Patterns are not intended to be consumed in-place in the same manner that one would consume a module. Therefore, we do not provide variables and outputs to expose various levels of configuration for the examples. Users can modify the pattern locally after cloning to suite their requirements.</p> </li> <li> <p>Please see the FAQ section on authenticating Kubernetes based providers (<code>kubernetes</code>, <code>helm</code>, <code>kubectl</code>) to Amazon EKS clusters regarding the use of static tokens versus dynamic tokens using the <code>awscli</code>.</p> </li> </ol>"},{"location":"#support-feedback","title":"Support &amp; Feedback","text":"<p>EKS Blueprints for Terraform is maintained by AWS Solution Architects. It is not part of an AWS service and support is provided as a best-effort by the EKS Blueprints community. To provide feedback, please use the issues templates provided. If you are interested in contributing to EKS Blueprints, see the Contribution guide.</p>"},{"location":"#security","title":"Security","text":"<p>See CONTRIBUTING for more information.</p>"},{"location":"#license","title":"License","text":"<p>Apache-2.0 Licensed. See LICENSE.</p>"},{"location":"faq/","title":"Frequently Asked Questions","text":""},{"location":"faq/#timeouts-on-destroy","title":"Timeouts on destroy","text":"<p>Customers who are deleting their environments using <code>terraform destroy</code> may see timeout errors when VPCs are being deleted. This is due to a known issue in the vpc-cni</p> <p>Customers may face a situation where ENIs that were attached to EKS managed nodes (same may apply to self-managed nodes) are not being deleted by the VPC CNI as expected which leads to IaC tool failures, such as:</p> <ul> <li>ENIs are left on subnets</li> <li>EKS managed security group which is attached to the ENI can\u2019t be deleted by EKS</li> </ul> <p>The current recommendation is to execute cleanup in the following order:</p> <ol> <li>delete all pods that have been created in the cluster.</li> <li>add delay/ wait</li> <li>delete VPC CNI</li> <li>delete nodes</li> <li>delete cluster</li> </ol>"},{"location":"faq/#leaked-cloudwatch-logs-group","title":"Leaked CloudWatch Logs Group","text":"<p>Sometimes, customers may see the CloudWatch Log Group for EKS cluster being created is left behind after their blueprint has been destroyed using <code>terraform destroy</code>. This happens because even after terraform deletes the CW log group, there\u2019s still logs being processed behind the scene by AWS EKS and service continues to write logs after recreating the log group using the EKS service IAM role which users don't have control over. This results in a terraform failure when the same blueprint is being recreated due to the existing log group left behind.</p> <p>There are two options here:</p> <ol> <li> <p>During cluster creation set <code>var.create_cloudwatch_log_group</code> to <code>false</code>. This will tell the EKS module to not create the log group, but instead let the EKS service create the log group. This means that upon cluster deletion the log group will be left behind but there will not be Terraform failures if you re-create the same cluster as Terraform does not manage the log group creation/deletion anymore.</p> </li> <li> <p>During cluster creation set <code>var.create_cloudwatch_log_group</code> to <code>true</code>. This will tell the EKS module to create the log group via Terraform. The EKS service will detect the log group and will start forwarding the logs for the log types enabled. Upon deletion terraform will delete the log group but depending upon any un-forwarded logs, the EKS service may recreate log group using the service role. This will result in terraform errors if the same blueprint is recreated. To proceed, manually delete the log group using the console or cli rerun the <code>terraform destroy</code>.</p> </li> </ol>"},{"location":"faq/#provider-authentication","title":"Provider Authentication","text":"<p>The chain of events when provisioning an example is typically in the stages of VPC -&gt; EKS cluster -&gt; addons and manifests. Per Terraform's recommendation, it is not recommended to pass an unknown value into provider configurations. However, for the sake of simplicity and ease of use, Blueprints does specify the AWS provider along with the Kubernetes, Helm, and Kubectl providers in order to show the full configuration required for provisioning example. Note - this is the configuration required to provision the example, not necessarily the shape of how the configuration should be structured; users are encouraged to split up EKS cluster creation from addon and manifest provisioning to align with Terraform's recommendations.</p> <p>With that said, the examples here are combining the providers and users can sometimes encounter various issues with the provider authentication methods. There are primarily two methods for authenticating the Kubernetes, Helm, and Kubectl providers to the EKS cluster created:</p> <ol> <li>Using a static token which has a lifetime of 15 minutes per the EKS service documentation.</li> <li>Using the <code>exec()</code> method which will fetch a token at the time of Terraform invocation.</li> </ol> <p>The Kubernetes and Helm providers recommend the <code>exec()</code> method, however this has the caveat that it requires the awscli to be installed on the machine running Terraform AND of at least a minimum version to support the API spec used by the provider (i.e. - <code>\"client.authentication.k8s.io/v1alpha1\"</code>, <code>\"client.authentication.k8s.io/v1beta1\"</code>, etc.). Selecting the appropriate provider authentication method is left up to users, and the examples used in this project will default to using the static token method for ease of use.</p> <p>Users of the static token method should be aware that if they receive a <code>401 Unauthorized</code> message, they might have a token that has expired and will need to run <code>terraform refresh</code> to get a new token. Users of the <code>exec()</code> method should be aware that the <code>exec()</code> method is reliant on the awscli and the associated authentication API version; the awscli version may need to be updated to support a later API version required by the Kubernetes version in use.</p> <p>The following examples demonstrate either method that users can utilize - please refer to the associated provider's documentation for further details on configuration.</p>"},{"location":"faq/#static-token-example","title":"Static Token Example","text":"<pre><code>provider \"kubernetes\" {\n  host                   = module.eks.cluster_endpoint\n  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n  token                  = data.aws_eks_cluster_auth.this.token\n}\n\nprovider \"helm\" {\n  kubernetes {\n    host                   = module.eks.cluster_endpoint\n    cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n    token                  = data.aws_eks_cluster_auth.this.token\n  }\n}\n\nprovider \"kubectl\" {\n  apply_retry_count      = 10\n  host                   = module.eks.cluster_endpoint\n  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n  load_config_file       = false\n  token                  = data.aws_eks_cluster_auth.this.token\n}\n\ndata \"aws_eks_cluster_auth\" \"this\" {\n  name = module.eks.cluster_name\n}\n</code></pre>"},{"location":"faq/#exec-example","title":"<code>exec()</code> Example","text":"<p>Usage of exec plugin for AWS credentials</p> <p>Links to References related to this issue</p> <ul> <li>https://github.com/hashicorp/terraform/issues/29182</li> <li>https://github.com/aws/aws-cli/pull/6476</li> </ul> <pre><code>provider \"kubernetes\" {\n  host                   = module.eks.cluster_endpoint\n  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n\n  exec {\n    api_version = \"client.authentication.k8s.io/v1beta1\"\n    command     = \"aws\"\n    # This requires the awscli to be installed locally where Terraform is executed\n    args = [\"eks\", \"get-token\", \"--cluster-name\", module.eks.cluster_name]\n  }\n}\n\nprovider \"helm\" {\n  kubernetes {\n    host                   = module.eks.cluster_endpoint\n    cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n\n    exec {\n      api_version = \"client.authentication.k8s.io/v1beta1\"\n      command     = \"aws\"\n      # This requires the awscli to be installed locally where Terraform is executed\n      args = [\"eks\", \"get-token\", \"--cluster-name\", module.eks.cluster_name]\n    }\n  }\n}\n\nprovider \"kubectl\" {\n  apply_retry_count      = 5\n  host                   = module.eks.cluster_endpoint\n  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)\n  load_config_file       = false\n\n  exec {\n    api_version = \"client.authentication.k8s.io/v1beta1\"\n    command     = \"aws\"\n    # This requires the awscli to be installed locally where Terraform is executed\n    args = [\"eks\", \"get-token\", \"--cluster-name\", module.eks.cluster_name]\n  }\n}\n</code></pre>"},{"location":"faq/#unable-to-destroy-namespace-created-by-terraform","title":"Unable to destroy namespace created by Terraform","text":"<p>In some cases, when you try to run terraform destroy on kubernetes resources created by Terraform such as namespace, you may end up seeing failures such as timeout and context deadline exceeded failures. Namespace one of those resources we've seen before, the main reason this happens is because orphaned resources created through CRDs of addons (such as ArgoCD, AWS LBC and more) are left behind after the addons are being deleted, this is case by case scenario. For example, with namespaces:</p> <ol> <li> <p>Confirm the namespace is hanging in status <code>Terminating</code></p> <pre><code>kubectl get namespaces\n</code></pre> </li> <li> <p>Check for any orphaned resources in the namespace, make sure to replace  with your namespace: <pre><code>kubectl api-resources --verbs=list --namespaced -o name   | xargs -n 1 kubectl get  \\\n--show-kind --ignore-not-found -n &lt;namespace_name&gt;\n</code></pre> <li> <p>For any of the above output, patch the resource finalize:</p> <pre><code>kubectl patch RESOURCE NAME -p '{\"metadata\":{\"finalizers\":[]}}' --type=merge\n</code></pre> </li> <li> <p>Check the status of the namespace, if needed you may need to patch the namespace finalizers as-well</p> <pre><code>kubectl patch ns &lt;ns-name&gt; -p '{\"spec\":{\"finalizers\":null}}'\n</code></pre> </li>"},{"location":"getting-started/","title":"Getting Started","text":"<p>This getting started guide will help you deploy your first pattern using EKS Blueprints.</p>"},{"location":"getting-started/#prerequisites","title":"Prerequisites","text":"<p>Ensure that you have installed the following tools locally:</p> <ul> <li>awscli</li> <li>kubectl</li> <li>terraform</li> </ul>"},{"location":"getting-started/#deploy","title":"Deploy","text":"<ol> <li> <p>For consuming EKS Blueprints, please see the Consumption section. For exploring and trying out the patterns provided, please clone the project locally to quickly get up and running with a pattern. After cloning the project locally, <code>cd</code> into the pattern directory of your choice.</p> </li> <li> <p>To provision the pattern, the typical steps of execution are as follows:</p> <pre><code>terraform init\nterraform apply -target=\"module.vpc\" -auto-approve\nterraform apply -target=\"module.eks\" -auto-approve\nterraform apply -auto-approve\n</code></pre> <p>For patterns that deviate from this general flow, see the pattern's respective <code>README.md</code> for more details.</p> <p>Terraform targeted apply</p> <p>Please see the Terraform Caveats section for details on the use of targeted Terraform apply's</p> </li> <li> <p>Once all of the resources have successfully been provisioned, the following command can be used to update the <code>kubeconfig</code> on your local machine and allow you to interact with your EKS Cluster using <code>kubectl</code>.</p> <pre><code>aws eks --region &lt;REGION&gt; update-kubeconfig --name &lt;CLUSTER_NAME&gt;\n</code></pre> <p>Pattern Terraform outputs</p> <p>Most examples will output the <code>aws eks update-kubeconfig ...</code> command as part of the Terraform apply output to simplify this process for users</p> <p>Private clusters</p> <p>Clusters that do not enable the clusters public endpoint will require users to access the cluster from within the VPC. For these patterns, a sample EC2 or other means are provided to demonstrate how to access those clusters privately</p> <p>and without exposing the public endpoint. Please see the respective pattern's <code>README.md</code> for more details.</p> </li> <li> <p>Once you have updated your <code>kubeconfig</code>, you can verify that you are able to interact with your cluster by running the following command:</p> <pre><code>kubectl get nodes\n</code></pre> <p>This should return a list of the node(s) running in the cluster created. If any errors are encountered, please re-trace the steps above and consult the pattern's <code>README.md</code> for more details on any additional/specific steps that may be required.</p> </li> </ol>"},{"location":"getting-started/#destroy","title":"Destroy","text":"<p>To teardown and remove the resources created in the pattern, the typical steps of execution are as follows:</p> <pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>Resources created outside of Terraform</p> <p>Depending on the pattern, some resources may have been created that Terraform is not aware of that will cause issues when attempting to clean up the pattern. For example, Karpenter is responsible for creating additional EC2 instances to satisfy the pod scheduling requirements. These instances will not be cleaned up by Terraform and will need to be de-provisioned BEFORE attempting to <code>terraform destroy</code>. This is why it is important that the addons, or any resources provisioned onto the cluster are cleaned up first. Please see the respective pattern's <code>README.md</code> for more details.</p>"},{"location":"_partials/destroy/","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"internal/ci/","title":"E2E tests","text":"<p>We use GitHub Actions to run an end-to-end tests to verify all PRs. The GitHub Actions used are a combination of <code>aws-actions/configure-aws-credentials</code> and <code>hashicorp/setup-terraform@v1</code>.</p>"},{"location":"internal/ci/#setup","title":"Setup","text":"<ol> <li>Use the following CloudFormation template to setup a new IAM role.</li> </ol> <pre><code>Parameters:\n  GitHubOrg:\n    Type: String\n  RepositoryName:\n    Type: String\n  OIDCProviderArn:\n    Description: Arn for the GitHub OIDC Provider.\n    Default: \"\"\n    Type: String\n\nConditions:\n  CreateOIDCProvider: !Equals\n    - !Ref OIDCProviderArn\n    - \"\"\n\nResources:\n  Role:\n    Type: AWS::IAM::Role\n    Properties:\n      AssumeRolePolicyDocument:\n        Statement:\n          - Effect: Allow\n            Action: sts:AssumeRoleWithWebIdentity\n            Principal:\n              Federated: !If\n                - CreateOIDCProvider\n                - !Ref GithubOidc\n                - !Ref OIDCProviderArn\n            Condition:\n              StringLike:\n                token.actions.githubusercontent.com:sub: !Sub repo:${GitHubOrg}/${RepositoryName}:*\n\n  GithubOidc:\n    Type: AWS::IAM::OIDCProvider\n    Condition: CreateOIDCProvider\n    Properties:\n      Url: https://token.actions.githubusercontent.com\n      ClientIdList:\n        - sts.amazonaws.com\n      ThumbprintList:\n        - a031c46782e6e6c662c2c87c76da9aa62ccabd8e\n\nOutputs:\n  Role:\n    Value: !GetAtt Role.Arn\n</code></pre> <ol> <li> <p>Add a permissible IAM Policy to the above create role. For our purpose <code>AdministratorAccess</code> works the best.</p> </li> <li> <p>Setup a GitHub repo secret called <code>ROLE_TO_ASSUME</code> and set it to ARN of the role created in 1.</p> </li> <li> <p>We use an S3 backend for the e2e tests. This allows us to recover from any failures during the <code>apply</code> stage. If you are setting up your own CI pipeline change the s3 bucket name in backend configuration of the example.</p> </li> </ol>"},{"location":"patterns/agones-game-controller/","title":"Agones Game Controller","text":""},{"location":"patterns/agones-game-controller/#amazon-eks-deployment-with-agones-gaming-kubernetes-controller","title":"Amazon EKS Deployment with Agones Gaming Kubernetes Controller","text":"<p>This pattern shows how to deploy and run gaming applications on Amazon EKS using the Agones Kubernetes Controller</p> <p>Agones is an open source Kubernetes controller that provisions and manages dedicated game server processes within Kubernetes clusters using standard Kubernetes tooling and APIs. This model also allows any matchmaker to interact directly with Agones via the Kubernetes API to provision a dedicated game server</p> <p>Amazon GameLift enables developers to deploy, operate, and scale dedicated, low-cost servers in the cloud for session-based, multiplayer games. Built on AWS global computing infrastructure, GameLift helps deliver high-performance, high-reliability, low-cost game servers while dynamically scaling your resource usage to meet worldwide player demand. See below for more information on how GameLift FleetIQ can be integrated with Agones deployed on Amazon EKS.</p> <p>Amazon GameLift FleetIQ optimizes the use of low-cost Spot Instances for cloud-based game hosting with Amazon EC2. With GameLift FleetIQ, you can work directly with your hosting resources in Amazon EC2 and Auto Scaling while taking advantage of GameLift optimizations to deliver inexpensive, resilient game hosting for your players and makes the use of low-cost Spot Instances viable for game hosting</p> <p>This blog walks through the details of deploying EKS Cluster using eksctl and deploy Agones with GameLift FleetIQ.</p>"},{"location":"patterns/agones-game-controller/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/agones-game-controller/#validate","title":"Validate","text":"<ol> <li> <p>Deploy the sample game server</p> <pre><code>kubectl create -f https://raw.githubusercontent.com/googleforgames/agones/release-1.32.0/examples/simple-game-server/gameserver.yaml\nkubectl get gs\n</code></pre> <pre><code>NAME                       STATE   ADDRESS         PORT   NODE                                        AGE\nsimple-game-server-7r6jr   Ready   34.243.345.22   7902   ip-10-1-23-233.eu-west-1.compute.internal   11h\n</code></pre> </li> <li> <p>Test the sample game server using <code>netcat</code></p> <pre><code>echo -n \"UDP test - Hello EKS Blueprints!\" | nc -u 34.243.345.22 7902\n</code></pre> <pre><code>Hello EKS Blueprints!\nACK: Hello EKS Blueprints!\nEXIT\nACK: EXIT\n</code></pre> </li> </ol>"},{"location":"patterns/agones-game-controller/#destroy","title":"Destroy","text":"<p>Delete the resources created by the sample game server first:</p> <pre><code>kubectl -n default delete gs --all || true\n</code></pre> <pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/aws-vpc-cni-network-policy/","title":"AWS VPC CNI Network Policy","text":""},{"location":"patterns/aws-vpc-cni-network-policy/#amazon-eks-cluster-w-network-policies","title":"Amazon EKS Cluster w/ Network Policies","text":"<p>This pattern demonstrates an EKS cluster that uses the native Network Policy support provided by the Amazon VPC CNI (1.14.0 or higher).</p> <ul> <li>Documentation</li> <li>Launch Blog</li> </ul>"},{"location":"patterns/aws-vpc-cni-network-policy/#scenario","title":"Scenario","text":"<p>This pattern deploys an Amazon EKS Cluster with Network Policies support implemented by the Amazon VPC CNI. Further it deploys a simple demo application (distributed as a Helm Chart) and some sample Network Policies to restrict the traffic between different components of the application.</p> <p>For a detailed description of the demo application and the Network Policies, please refer to the Stars demo of network policy section in the official Documentation.</p>"},{"location":"patterns/aws-vpc-cni-network-policy/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/aws-vpc-cni-network-policy/#validate","title":"Validate","text":"<ol> <li> <p>List out the pods running currently:</p> <pre><code>kubectl get pods -A\n</code></pre> <p><code>text    NAMESPACE         NAME                                       READY   STATUS    RESTARTS   AGE [...] client            client-xlf2c                               1/1     Running   0          5m19s [...] management-ui     management-ui-qrb2g                        1/1     Running   0          5m24s stars             backend-sz87q                              1/1     Running   0          5m23s stars             frontend-c1cnf                             1/1     Running   0          5m21s [...]</code></p> <p>In your output, you should see pods in the namespaces shown in the following output. The NAMES of your pods and the number of pods in the READY column are different than those in the following output. Don't continue until you see pods with similar names and they all have Running in the STATUS column.</p> </li> <li> <p>Connect to the management user interface using the EXTERNAL IP of the running service and observe the traffic flow and restrictions based on the Network Policies deployed:</p> <pre><code>kubectl get service/management-ui -n management-ui\n</code></pre> <p>Open the browser based on the URL obtained from the previous step to see the connection map and restrictions put in place by the Network Policies deployed.</p> </li> </ol>"},{"location":"patterns/aws-vpc-cni-network-policy/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/blue-green-upgrade/","title":"Blue/Green Upgrade","text":""},{"location":"patterns/blue-green-upgrade/#bluegreen-migration","title":"Blue/Green Migration","text":"<p>This directory provides a solution based on EKS Blueprint for Terraform that shows how to leverage blue/green or canary application workload migration between EKS clusters, using Amazon Route 53 weighted routing feature. The workloads will be dynamically exposed using AWS LoadBalancer Controller and External DNS add-on.</p> <p>We are leveraging the existing EKS Blueprints Workloads GitHub repository sample to deploy our GitOps ArgoCD applications, which are defined as helm charts. We are leveraging ArgoCD Apps of apps pattern where an ArgoCD Application can also reference other Helm charts to deploy.</p> <p>You can also find more informations in the associated blog post</p>"},{"location":"patterns/blue-green-upgrade/#table-of-content","title":"Table of content","text":"<ul> <li>Blue/Green Migration</li> <li>Table of content</li> <li>Project structure</li> <li>Prerequisites</li> <li>Quick Start<ul> <li>Configure the Stacks</li> <li>Create the environment stack</li> <li>Create the Blue cluster</li> <li>Create the Green cluster</li> </ul> </li> <li>How this work<ul> <li>Watch our Workload: we focus on team-burnham namespace.</li> <li>Using AWS Route53 and External DNS</li> <li>Configure Ingress resources with weighted records</li> </ul> </li> <li>Automate the migration from Terraform</li> <li>Delete the Stack<ul> <li>Delete the EKS Cluster(s)</li> <li>TL;DR</li> </ul> </li> <li>Troubleshoot<ul> <li>External DNS Ownership</li> <li>Check Route 53 Record status</li> <li>Check current resolution and TTL value</li> <li>Get ArgoCD UI Password</li> </ul> </li> </ul>"},{"location":"patterns/blue-green-upgrade/#project-structure","title":"Project structure","text":"<p>See the Architecture of what we are building</p> <p> </p> <p>Our sample is composed of four main directory:</p> <ul> <li>environment \u2192 this stack will create the common VPC and its dependencies used by our EKS clusters: create a Route53 sub domain hosted zone for our sample, a wildcard certificate on Certificate Manager for our applications TLS endpoints, and a SecretManager password for the ArgoCD UIs.</li> <li>modules/eks_cluster \u2192 local module defining the EKS blueprint cluster with ArgoCD add-on which will automatically deploy additional add-ons and our demo workloads</li> <li>eks-blue \u2192 an instance of the eks_cluster module to create blue cluster</li> <li>eks-green \u2192 an instance of the eks_cluster module to create green cluster</li> </ul> <p>So we are going to create 2 EKS clusters, sharing the same VPC, and each one of them will install locally our workloads from the central GitOps repository leveraging ArgoCD add-on. In the GitOps workload repository, we have configured our applications deployments to leverage AWS Load Balancers Controllers annotations, so that applications will be exposed on AWS Load Balancers, created from our Kubernetes manifests. We will have 1 load balancer per cluster for each of our applications.</p> <p>We have configured ExternalDNS add-ons in our two clusters to share the same Route53 Hosted Zone. The workloads in both clusters also share the same Route 53 DNS records, we rely on AWS Route53 weighted records to allow us to configure canary workload migration between our two EKS clusters.</p> <p>We are leveraging the gitops-bridge-argocd-bootstrap terraform module that allow us to dynamically provide metadatas from Terraform to ArgoCD deployed in the cluster. For doing this, the module will extract all metadatas from the terraform-aws-eks-blueprints-addons module, configured to create all resources except installing the addon's Helm chart. The addon Installation will be delegate to ArgoCD Itself using the eks-blueprints-add-ons git repository containing ArgoCD ApplicaitonSets for each supported Addons.</p> <p>The gitops-bridge will create a secret in the EKS cluster containing all metadatas that will be dynamically used by ArgoCD ApplicationSets at deployment time, so that we can adapt their configuration to our EKS cluster context.</p> <p></p> <p>Our objective here is to show you how Application teams and Platform teams can configure their infrastructure and workloads so that application teams are able to deploy autonomously their workloads to the EKS clusters thanks to ArgoCD, and platform team can keep the control of migrating production workloads from one cluster to another without having to synchronized operations with applications teams, or asking them to build a complicated CD pipeline.</p> <p>In this example we show how you can seamlessly migrate your stateless workloads between the 2 clusters for a blue/green or Canary migration, but you can also leverage the same architecture to have your workloads for example separated in different accounts or regions, for either High Availability or Lower latency Access from your customers.</p>"},{"location":"patterns/blue-green-upgrade/#prerequisites","title":"Prerequisites","text":"<ul> <li>Terraform (tested version v1.3.5 on linux)</li> <li>Git</li> <li>AWS CLI</li> <li>AWS test account with administrator role access</li> <li>For working with this repository, you will need an existing Amazon Route 53 Hosted Zone that will be used to create our project hosted zone. It will be provided via the Terraform variable <code>hosted_zone_name</code> defined in terraform.tfvars.example.</li> <li>Before moving to the next step, you will need to register a parent domain with AWS Route 53 (https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html) in case you don\u2019t have one created yet.</li> <li>Accessing GitOps git repositories with SSH access requiring an SSH key for authentication. In this example our workloads repositories are stored in GitHub, you can see in GitHub documentation on how to connect with SSH.</li> <li>Your GitHub private ssh key value is supposed to be stored in plain text in AWS Secret Manager in a secret named <code>github-blueprint-ssh-key</code>, but you can change it using the terraform variable <code>workload_repo_secret</code> in terraform.tfvars.example.</li> <li></li> </ul>"},{"location":"patterns/blue-green-upgrade/#quick-start","title":"Quick Start","text":""},{"location":"patterns/blue-green-upgrade/#configure-the-stacks","title":"Configure the Stacks","text":"<ol> <li>Clone the repository</li> </ol> <pre><code>git clone https://github.com/aws-ia/terraform-aws-eks-blueprints.git\ncd patterns/blue-green-upgrade/\n</code></pre> <ol> <li>Copy the <code>terraform.tfvars.example</code> to <code>terraform.tfvars</code> and symlink it on each <code>environment</code>, <code>eks-blue</code> and <code>eks-green</code> folders, and change region, hosted_zone_name, eks_admin_role_name according to your needs.</li> </ol> <pre><code>cp terraform.tfvars.example terraform.tfvars\nln -s ../terraform.tfvars environment/terraform.tfvars\nln -s ../terraform.tfvars eks-blue/terraform.tfvars\nln -s ../terraform.tfvars eks-green/terraform.tfvars\n</code></pre> <ul> <li>You will need to provide the <code>hosted_zone_name</code> for example <code>my-example.com</code>. Terraform will create a new hosted zone for the project with name: <code>${environment}.${hosted_zone_name}</code> so in our example <code>eks-blueprint.my-example.com</code>.</li> <li>You need to provide a valid IAM role in <code>eks_admin_role_name</code> to have EKS cluster admin rights, generally the one uses in the EKS console.</li> </ul>"},{"location":"patterns/blue-green-upgrade/#create-the-environment-stack","title":"Create the environment stack","text":"<p>More info in the environment Readme</p> <pre><code>cd environment\nterraform init\nterraform apply\n</code></pre> <p>There can be some Warnings due to not declare variables. This is normal and you can ignore them as we share the same <code>terraform.tfvars</code> for the 3 projects by using symlinks for a unique file, and we declare some variables used for the eks-blue and eks-green directory</p>"},{"location":"patterns/blue-green-upgrade/#create-the-blue-cluster","title":"Create the Blue cluster","text":"<p>More info in the eks-blue Readme, you can also see the detailed step in the local module Readme</p> <pre><code>cd eks-blue\nterraform init\nterraform apply\n</code></pre> <p>This can take 8mn for EKS cluster, 15mn</p>"},{"location":"patterns/blue-green-upgrade/#create-the-green-cluster","title":"Create the Green cluster","text":"<pre><code>cd eks-green\nterraform init\nterraform apply\n</code></pre> <p>By default the only differences in the 2 clusters are the values defined in main.tf. We will change those values to upgrade Kubernetes version of new cluster, and to migrate our stateless workloads between clusters.</p>"},{"location":"patterns/blue-green-upgrade/#how-this-work","title":"How this work","text":""},{"location":"patterns/blue-green-upgrade/#watch-our-workload-we-focus-on-team-burnham-namespace","title":"Watch our Workload: we focus on team-burnham namespace.","text":"<p>Our clusters are configured with existing ArgoCD Github repository that is synchronized into each of the clusters:</p> <ul> <li>EKS Blueprints Add-ons repository</li> <li>Workloads repository</li> </ul> <p> </p> <p>We are going to look after one of the application deployed from the workload repository as example to demonstrate our migration automation: the <code>Burnham</code> workload in the team-burnham namespace. We have set up a simple go application than can respond in it's body the name of the cluster it is running on. With this it will be easy to see the current migration on our workload.</p> <pre><code>&lt;head&gt;\n  &lt;title&gt;Hello EKS Blueprint&lt;/title&gt;\n&lt;/head&gt;\n&lt;div class=\"info\"&gt;\n  &lt;h&gt;Hello EKS Blueprint Version 1.4&lt;/h&gt;\n  &lt;p&gt;&lt;span&gt;Server&amp;nbsp;address:&lt;/span&gt; &lt;span&gt;10.0.2.201:34120&lt;/span&gt;&lt;/p&gt;\n  &lt;p&gt;&lt;span&gt;Server&amp;nbsp;name:&lt;/span&gt; &lt;span&gt;burnham-9d686dc7b-dw45m&lt;/span&gt;&lt;/p&gt;\n  &lt;p class=\"smaller\"&gt;&lt;span&gt;Date:&lt;/span&gt; &lt;span&gt;2022.10.13 07:27:28&lt;/span&gt;&lt;/p&gt;\n  &lt;p class=\"smaller\"&gt;&lt;span&gt;URI:&lt;/span&gt; &lt;span&gt;/&lt;/span&gt;&lt;/p&gt;\n  &lt;p class=\"smaller\"&gt;&lt;span&gt;HOST:&lt;/span&gt; &lt;span&gt;burnham.eks-blueprint.mon-domain.com&lt;/span&gt;&lt;/p&gt;\n  &lt;p class=\"smaller\"&gt;&lt;span&gt;CLUSTER_NAME:&lt;/span&gt; &lt;span&gt;eks-blueprint-blue&lt;/span&gt;&lt;/p&gt;\n&lt;/div&gt;\n</code></pre> <p>The application is deployed from our  workload repository manifest <p>Connect to the cluster: Execute one of the EKS cluster login commands from the <code>terraform output</code> command, depending on the IAM role you can assume to access to the cluster. If you want EKS Admin cluster, you can execute the command associated to the eks_blueprints_admin_team_configure_kubectl output. It should be something similar to:</p> <pre><code>aws eks --region eu-west-3 update-kubeconfig --name eks-blueprint-blue  --role-arn arn:aws:iam::0123456789:role/admin-team-20230505075455219300000002\n</code></pre> <p>Note it will allow the role associated to the parameter eks_admin_role_name to assume the role.</p> <p>You can also connect with the user who created the EKS cluster without specifying the <code>--role-arn</code> parameter</p> <p>Next, you can interact with the cluster and see the deployment</p> <pre><code>$ kubectl get deployment -n team-burnham -l app=burnham\nNAME      READY   UP-TO-DATE   AVAILABLE   AGE\nburnham   3/3     3            3           3d18h\n</code></pre> <p>See the pods</p> <pre><code>$ kubectl get pods -n team-burnham -l app=burnham\nNAME                       READY   STATUS    RESTARTS   AGE\nburnham-7db4c6fdbb-82hxn   1/1     Running   0          3d18h\nburnham-7db4c6fdbb-dl59v   1/1     Running   0          3d18h\nburnham-7db4c6fdbb-hpq6h   1/1     Running   0          3d18h\n</code></pre> <p>See the logs:</p> <pre><code>$ kubectl logs -n team-burnham -l app=burnham\n2022/10/10 12:35:40 {url: / }, cluster: eks-blueprint-blue }\n2022/10/10 12:35:49 {url: / }, cluster: eks-blueprint-blue }\n</code></pre> <p>You can make a request to the service, and filter the output to know on which cluster it runs:</p> <pre><code>$ URL=$(echo -n \"https://\" ; kubectl get ing -n team-burnham burnham-ingress -o json | jq \".spec.rules[0].host\" -r)\n$ curl -s $URL | grep CLUSTER_NAME | awk -F \"&lt;span&gt;|&lt;/span&gt;\" '{print $4}'\neks-blueprint-blue\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#using-aws-route53-and-external-dns","title":"Using AWS Route53 and External DNS","text":"<p>We have configured both our clusters to configure the same Amazon Route 53 Hosted Zones. This is done by having the same configuration of ExternalDNS add-on in <code>main.tf</code>:</p> <p>This is the Terraform configuration to configure the ExternalDNS Add-on which is deployed by the Blueprint using ArgoCD. we specify the Route53 zone that external-dns needs to monitor.</p> <pre><code>  enable_external_dns = true\n  external_dns_route53_zone_arns      = [data.aws_route53_zone.sub.arn]\n</code></pre> <p>we also configure the addons_metadata to provide more configurations to external-dns:</p> <pre><code>addons_metadata = merge(\n  ...\n  external_dns_policy        = \"sync\"\n</code></pre> <ul> <li>We use ExternalDNS in <code>sync</code> mode so that the controller can create but also remove DNS records accordingly to service or ingress objects creation.</li> <li>We also configured the <code>txtOwnerId</code> with the name of the cluster, so that each controller will be able to create/update/delete records but only for records which are associated to the proper OwnerId.</li> <li>Each Route53 record will be also associated with a <code>txt</code> record. This record is used to specify the owner of the associated record and is in the form of:</li> </ul> <pre><code>\"heritage=external-dns,external-dns/owner=eks-blueprint-blue,external-dns/resource=ingress/team-burnham/burnham-ingress\"\n</code></pre> <p>So in this example the Owner of the record is the external-dns controller, from the eks-blueprint-blue EKS cluster, and correspond to the Kubernetes ingress resource names burnham-ingress in the team-burnham namespace.</p> <p>Using this feature, and relying on weighted records, we will be able to do blue/green or canary migration by changing the weight of ingress resources defined in each cluster.</p>"},{"location":"patterns/blue-green-upgrade/#configure-ingress-resources-with-weighted-records","title":"Configure Ingress resources with weighted records","text":"<p>Since we have configured ExternalDNS add-on, we can now defined specific annotation in our <code>ingress</code> object. You may already know that our workload are synchronized using ArgoCD from our workload repository sample.</p> <p>We are focusing on the burnham deployment, which is defined here where we configure the <code>burnham-ingress</code> ingress object with:</p> <pre><code>    external-dns.alpha.kubernetes.io/set-identifier: {{ .Values.spec.clusterName }}\n    external-dns.alpha.kubernetes.io/aws-weight: '{{ .Values.spec.ingress.route53_weight }}'\n</code></pre> <p>We rely on two external-dns annotation to configure how the record will be created. the <code>set-identifier</code> annotation will contain the name of the cluster we want to create, which must match the one define in the external-dns <code>txtOwnerId</code> configuration.</p> <p>The <code>aws-weight</code> will be used to configure the value of the weighted record, and it will be deployed from Helm values, that will be injected by Terraform in our example, so that our platform team will be able to control autonomously how and when they want to migrate workloads between the EKS clusters.</p> <p>Amazon Route 53 weighted records works like this:</p> <ul> <li>If we specify a value of 100 in eks-blue cluster and 0 in eks-green cluster, then Route 53 will route all requests to eks-blue cluster.</li> <li>If we specify a value of 0 in eks-blue cluster and 0 in eks-green cluster, then Route 53 will route all requests to eks-green cluster.</li> <li>we can also define any intermediate values like 100 in eks-blue cluster and 100 in eks-green cluster, so we will have 50% on eks-blue and 50% on eks-green.</li> </ul>"},{"location":"patterns/blue-green-upgrade/#automate-the-migration-from-terraform","title":"Automate the migration from Terraform","text":"<p>Now that we have setup our 2 clusters, deployed with ArgoCD and that the weighed records from <code>values.yaml</code> are injected from Terraform, let's see how our Platform team can trigger the workload migration.</p> <ol> <li>At first, 100% of burnham traffic is set to the eks-blue cluster, this is controlled from the <code>eks-blue/main.tf</code> &amp; <code>eks-green/main.tf</code> files with the parameter <code>route53_weight = \"100\"</code>. The same parameter is set to 0 in cluster eks-green.</li> </ol> <p> </p> <p>Which correspond to :</p> <p> </p> <p>All requests to our endpoint should response with <code>eks-blueprint-blue</code> we can test it with the following command:</p> <pre><code>URL=$(echo -n \"https://\" ; kubectl get ing -n team-burnham burnham-ingress -o json | jq \".spec.rules[0].host\" -r)\ncurl -s $URL | grep CLUSTER_NAME | awk -F \"&lt;span&gt;|&lt;/span&gt;\" '{print $4}'\n</code></pre> <p>you should see:</p> <pre><code>eks-blueprint-blue\n</code></pre> <ol> <li>Let's change traffic to 50% eks-blue and 50% eks-green by activating also value 100 in eks-green locals.tf (<code>route53_weight = \"100\"</code>) and let's <code>terraform apply</code> to let terraform update the configuration</li> </ol> <p> </p> <p>Which correspond to :</p> <p> </p> <p>All records have weight of 100, so we will have 50% requests on each clusters.</p> <p>We can check the ratio of requests resolution between both clusters</p> <pre><code>URL=$(echo -n \"https://\" ; kubectl get ing -n team-burnham burnham-ingress -o json | jq \".spec.rules[0].host\" -r)\nrepeat 10 curl -s $URL | grep CLUSTER_NAME | awk -F \"&lt;span&gt;|&lt;/span&gt;\" '{print $4}' &amp;&amp; sleep 60\n</code></pre> <p>Result should be similar to:</p> <pre><code>eks-blueprint-blue\neks-blueprint-blue\neks-blueprint-blue\neks-blueprint-blue\neks-blueprint-green\neks-blueprint-green\neks-blueprint-blue\neks-blueprint-green\neks-blueprint-blue\neks-blueprint-green\n</code></pre> <p>The default TTL is for 60 seconds, and you have 50% chance to have blue or green cluster, then you may need to replay the previous command several times to have an idea of the repartition, which theoretically is 50%</p> <ol> <li>Now that we see that our green cluster is taking requests correctly, we can update the eks-blue cluster configuration to have the weight to 0 and apply again. after a few moment, your route53 records should look like the below screenshot, and all requests should now reach eks-green cluster.</li> </ol> <p> </p> <p>Which correspond to :</p> <p> </p> <p>At this step, once all DNS TTL will be up to date, all the traffic will be coming on the eks-green cluster. You can either, delete the eks-blue cluster, or decide to make upgrades on the blue cluster and send back traffic on eks-blue afterward, or simply keep it as a possibility for rollback if needed.</p> <p>In this sample, we uses a simple terraform variable to control the weight for all applications, we can also choose to have several parameters, let's say one per application, so you can finer control your migration strategy application by application.</p>"},{"location":"patterns/blue-green-upgrade/#delete-the-stack","title":"Delete the Stack","text":""},{"location":"patterns/blue-green-upgrade/#delete-the-eks-clusters","title":"Delete the EKS Cluster(s)","text":"<p>This section, can be executed in either eks-blue or eks-green folders, or in both if you want to delete both clusters.</p> <p>In order to properly destroy the Cluster, we need first to remove the ArgoCD workloads, while keeping the ArgoCD addons. We will also need to remove our Karpenter provisioners, and any other objects you created outside of Terraform that needs to be cleaned before destroying the terraform stack.</p> <p>Why doing this? When we remove an ingress object, we want the associated Kubernetes add-ons like aws load balancer controller and External DNS to correctly free the associated AWS resources. If we directly ask terraform to destroy everything, it can remove first theses controllers without allowing them the time to remove associated aws resources that will still existing in AWS, preventing us to completely delete our cluster.</p>"},{"location":"patterns/blue-green-upgrade/#tldr","title":"TL;DR","text":"<pre><code>../tear-down.sh\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#troubleshoot","title":"Troubleshoot","text":""},{"location":"patterns/blue-green-upgrade/#external-dns-ownership","title":"External DNS Ownership","text":"<p>The Amazon Route 53 records association are controlled by ExternalDNS controller. You can see the logs from the controller to understand what is happening by executing the following command in each cluster:</p> <pre><code>kubectl logs  -n external-dns -l app.kubernetes.io/name=external-dns -f\n</code></pre> <p>In eks-blue cluster, you can see logs like the following, which showcase that the eks-blueprint-blue controller won't make any change in records owned by eks-blueprint-green cluster, the reverse is also true.</p> <pre><code>time=\"2022-10-10T15:46:54Z\" level=debug msg=\"Skipping endpoint skiapp.eks-blueprint.sallaman.people.aws.dev 300 IN CNAME eks-blueprint-green k8s-riker-68438cd99f-893407990.eu-west-1.elb.amazonaws.com [{aws/evaluate-target-health true} {alias true} {aws/weight 100}] because owner id does not match, found: \\\"eks-blueprint-green\\\", required: \\\"eks-blueprint-blue\\\"\"\ntime=\"2022-10-10T15:46:54Z\" level=debug msg=\"Refreshing zones list cache\"\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#check-route-53-record-status","title":"Check Route 53 Record status","text":"<p>We can also use the CLI to see our current Route 53 configuration:</p> <pre><code>export ROOT_DOMAIN=&lt;your-domain-name&gt; # the value you put in hosted_zone_name\nZONE_ID=$(aws route53 list-hosted-zones-by-name --output json --dns-name \"eks-blueprint.${ROOT_DOMAIN}.\" --query \"HostedZones[0].Id\" --out text)\necho $ZONE_ID\naws route53 list-resource-record-sets \\\n  --output json \\\n  --hosted-zone-id $ZONE_ID \\\n  --query \"ResourceRecordSets[?Name == 'burnham.eks-blueprint.$ROOT_DOMAIN.']|[?Type == 'A']\"\n\naws route53 list-resource-record-sets \\\n  --output json \\\n  --hosted-zone-id $ZONE_ID \\\n  --query \"ResourceRecordSets[?Name == 'burnham.eks-blueprint.$ROOT_DOMAIN.']|[?Type == 'TXT']\"\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#check-current-resolution-and-ttl-value","title":"Check current resolution and TTL value","text":"<p>As DNS migration is dependent of DNS caching, normally relying on the TTL, you can use dig to see the current value of the TTL used locally</p> <pre><code>export ROOT_DOMAIN=&lt;your-domain-name&gt; # the value you put for hosted_zone_name\ndig +noauthority +noquestion +noadditional +nostats +ttlunits +ttlid A burnham.eks-blueprint.$ROOT_DOMAIN\n</code></pre>"},{"location":"patterns/blue-green-upgrade/#get-argocd-ui-password","title":"Get ArgoCD UI Password","text":"<p>You can connect to the ArgoCD UI using the service :</p> <pre><code>kubectl get svc -n argocd argo-cd-argocd-server -o json | jq '.status.loadBalancer.ingress[0].hostname' -r\n</code></pre> <p>Then login with admin and get the password from AWS Secret Manager:</p> <pre><code>aws secretsmanager get-secret-value \\\n  --secret-id argocd-admin-secret.eks-blueprint \\\n  --query SecretString \\\n  --output text --region $AWS_REGION\n</code></pre>"},{"location":"patterns/external-secrets/","title":"External Secrets","text":""},{"location":"patterns/external-secrets/#amazon-eks-cluster-w-external-secrets-operator","title":"Amazon EKS Cluster w/ External Secrets Operator","text":"<p>This pattern deploys an EKS Cluster with the External Secrets Operator. The cluster is populated with a ClusterSecretStore and SecretStore example using SecretManager and Parameter Store respectively. A secret for each store is also created. Both stores use IRSA to retrieve the secret values from AWS.</p>"},{"location":"patterns/external-secrets/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/external-secrets/#validate","title":"Validate","text":"<ol> <li> <p>List the secret resources in the <code>external-secrets</code> namespace</p> <pre><code>kubectl get externalsecrets -n external-secrets\nkubectl get secrets -n external-secrets\n</code></pre> </li> </ol>"},{"location":"patterns/external-secrets/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/fargate-serverless/","title":"Fargate Serverless","text":""},{"location":"patterns/fargate-serverless/#serverless-amazon-eks-cluster","title":"Serverless Amazon EKS Cluster","text":"<p>This pattern demonstrates an Amazon EKS Cluster that utilizes Fargate profiles for a serverless data plane.</p>"},{"location":"patterns/fargate-serverless/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/fargate-serverless/#validate","title":"Validate","text":"<ol> <li> <p>List the nodes in in the cluster; you should see Fargate instances:</p> <pre><code>kubectl get nodes\n</code></pre> <pre><code>NAME                                                STATUS   ROLES    AGE   VERSION\nfargate-ip-10-0-17-17.us-west-2.compute.internal    Ready    &lt;none&gt;   25m   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-20-244.us-west-2.compute.internal   Ready    &lt;none&gt;   71s   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-41-143.us-west-2.compute.internal   Ready    &lt;none&gt;   25m   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-44-95.us-west-2.compute.internal    Ready    &lt;none&gt;   25m   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-45-153.us-west-2.compute.internal   Ready    &lt;none&gt;   77s   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-47-31.us-west-2.compute.internal    Ready    &lt;none&gt;   75s   v1.26.3-eks-f4dc2c0\nfargate-ip-10-0-6-175.us-west-2.compute.internal    Ready    &lt;none&gt;   25m   v1.26.3-eks-f4dc2c0\n</code></pre> </li> <li> <p>List the pods. All the pods should reach a status of <code>Running</code> after approximately 60 seconds:</p> <pre><code>kubectl get pods -A\n</code></pre> <pre><code>NAMESPACE       NAME                                            READY   STATUS    RESTARTS   AGE\napp-2048        app-2048-65bd744dfb-7g9rx                       1/1     Running   0          2m34s\napp-2048        app-2048-65bd744dfb-n1cbm                       1/1     Running   0          2m34s\napp-2048        app-2048-65bd744dfb-z4b6z                       1/1     Running   0          2m34s\nkube-system     aws-load-balancer-controller-6cbdb58654-f1skt   1/1     Running   0          26m\nkube-system     aws-load-balancer-controller-6cbdb58654-sc7dk   1/1     Running   0          26m\nkube-system     coredns-7b7bddbc85-j1bv6                        1/1     Running   0          26m\nkube-system     coredns-7b7bddbc85-rg2zq                        1/1     Running   0          26m\n</code></pre> </li> <li> <p>Validate the <code>aws-logging</code> configMap for Fargate Fluentbit was created:</p> <pre><code>kubectl -n aws-observability get configmap aws-logging -o yaml\n</code></pre> <pre><code>apiVersion: v1\ndata:\n  filters.conf: |\n    [FILTER]\n      Name parser\n      Match *\n      Key_Name log\n      Parser regex\n      Preserve_Key True\n      Reserve_Data True\n  flb_log_cw: \"true\"\n  output.conf: |\n    [OUTPUT]\n      Name cloudwatch_logs\n      Match *\n      region us-west-2\n      log_group_name /fargate-serverless/fargate-fluentbit-logs20230509014113352200000006\n      log_stream_prefix fargate-logs-\n      auto_create_group true\n  parsers.conf: |\n    [PARSER]\n      Name regex\n      Format regex\n      Regex ^(?&lt;time&gt;[^ ]+) (?&lt;stream&gt;[^ ]+) (?&lt;logtag&gt;[^ ]+) (?&lt;message&gt;.+)$\n      Time_Key time\n      Time_Format %Y-%m-%dT%H:%M:%S.%L%z\n      Time_Keep On\n      Decode_Field_As json message\nimmutable: false\nkind: ConfigMap\nmetadata:\n  creationTimestamp: \"2023-05-08T21:14:52Z\"\n  name: aws-logging\n  namespace: aws-observability\n  resourceVersion: \"1795\"\n  uid: d822bcf5-a441-4996-857e-7fb1357bc07e\n</code></pre> <p>You can also validate if the CloudWatch LogGroup was created accordingly, and LogStreams were populated:</p> <pre><code>aws logs describe-log-groups \\\n    --log-group-name-prefix \"/fargate-serverless/fargate-fluentbit\"\n</code></pre> <pre><code>{\n    \"logGroups\": [\n        {\n            \"logGroupName\": \"/fargate-serverless/fargate-fluentbit-logs20230509014113352200000006\",\n            \"creationTime\": 1683580491652,\n            \"retentionInDays\": 90,\n            \"metricFilterCount\": 0,\n            \"arn\": \"arn:aws:logs:us-west-2:111222333444:log-group:/fargate-serverless/fargate-fluentbit-logs20230509014113352200000006:*\",\n            \"storedBytes\": 0\n        }\n    ]\n}\n</code></pre> <pre><code>aws logs describe-log-streams \\\n    --log-group-name \"/fargate-serverless/fargate-fluentbit-logs20230509014113352200000006\" \\\n    --log-stream-name-prefix fargate-logs --query 'logStreams[].logStreamName'\n</code></pre> <pre><code>[\n    \"fargate-logs-flblogs.var.log.fluent-bit.log\",\n    \"fargate-logs-kube.var.log.containers.aws-load-balancer-controller-7f989fc6c-gr1sq_kube-system_aws-load-balancer-controller-feaa22b4cdaa71ecfc8355feb81d4b61ea85598a7bb57aef07667c767c6b98e4.log\",\n    \"fargate-logs-kube.var.log.containers.aws-load-balancer-controller-7f989fc6c-wzr46_kube-system_aws-load-balancer-controller-69075ea9ab3c7474eac2a1696d3a84a848a151420cd783d79aeef960b181567f.log\",\n    \"fargate-logs-kube.var.log.containers.coredns-7b7bddbc85-8cx1q_kube-system_coredns-9e4f3ab435269a566bcbaa606c02c146ad58508e67cef09fa87d5c09e4ac0088.log\",\n    \"fargate-logs-kube.var.log.containers.coredns-7b7bddbc85-g1jwp_kube-system_coredns-11016818361cd68c32bf8f0b1328f3d92a6d7b8cf5879bfe8b301f393cb011cc.log\"\n]\n</code></pre> </li> </ol>"},{"location":"patterns/fargate-serverless/#example","title":"Example","text":"<ol> <li> <p>Create an ingress resource using the AWS load balancer controller deployed, pointing to our application service:</p> <pre><code>kubectl get svc -n app-2048\n</code></pre> <pre><code>NAME       TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE\napp-2048   NodePort   172.20.33.217   &lt;none&gt;        80:32568/TCP   2m48s\n</code></pre> <pre><code>kubectl -n app-2048 create ingress app-2048 --class alb --rule=\"/*=app-2048:80\" \\\n--annotation alb.ingress.kubernetes.io/scheme=internet-facing \\\n--annotation alb.ingress.kubernetes.io/target-type=ip\n</code></pre> <pre><code>kubectl -n app-2048 get ingress\n</code></pre> <pre><code>NAME       CLASS   HOSTS   ADDRESS                                                                 PORTS   AGE\napp-2048   alb     *       k8s-app2048-app2048-6d9c5e92d6-1234567890.us-west-2.elb.amazonaws.com   80      4m9s\n</code></pre> </li> <li> <p>Open the browser to access the application via the URL address shown in the last output in the ADDRESS column.</p> <p>In our example: <code>k8s-app2048-app2048-6d9c5e92d6-1234567890.us-west-2.elb.amazonaws.com</code></p> <p>Info</p> <p>You might need to wait a few minutes, and then refresh your browser. If your Ingress isn't created after several minutes, then run this command to view the AWS Load Balancer Controller logs:</p> <pre><code>kubectl logs -n kube-system deployment.apps/aws-load-balancer-controller\n</code></pre> </li> </ol>"},{"location":"patterns/fargate-serverless/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/fully-private-cluster/","title":"Fully Private Cluster","text":""},{"location":"patterns/fully-private-cluster/#fully-private-amazon-eks-cluster","title":"Fully Private Amazon EKS Cluster","text":"<p>This pattern demonstrates an Amazon EKS cluster that does not have internet access. The private cluster must pull images from a container registry that is within in your VPC, and also must have endpoint private access enabled. This is required for nodes to register with the cluster endpoint.</p> <p>Please see this document for more details on configuring fully private EKS Clusters.</p> <p>For fully Private EKS clusters requires the following VPC endpoints to be created to communicate with AWS services. This example solution will provide these endpoints if you choose to create VPC. If you are using an existing VPC then you may need to ensure these endpoints are created.</p> <pre><code>com.amazonaws.region.aps-workspaces       - If using AWS Managed Prometheus Workspace\ncom.amazonaws.region.ssm                  - Secrets Management\ncom.amazonaws.region.ec2\ncom.amazonaws.region.ecr.api\ncom.amazonaws.region.ecr.dkr\ncom.amazonaws.region.logs                 \u2013 For CloudWatch Logs\ncom.amazonaws.region.sts                  \u2013 If using AWS Fargate or IAM roles for service accounts\ncom.amazonaws.region.elasticloadbalancing \u2013 If using Application Load Balancers\ncom.amazonaws.region.autoscaling          \u2013 If using Cluster Autoscaler\ncom.amazonaws.region.s3\n</code></pre>"},{"location":"patterns/fully-private-cluster/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/fully-private-cluster/#validate","title":"Validate","text":"<ol> <li> <p>Test by listing Nodes in in the cluster:</p> <pre><code>kubectl get nodes\n</code></pre> <pre><code>NAME                                        STATUS   ROLES    AGE     VERSION\nip-10-0-19-90.us-west-2.compute.internal    Ready    &lt;none&gt;   8m34s   v1.26.2-eks-a59e1f0\nip-10-0-44-110.us-west-2.compute.internal   Ready    &lt;none&gt;   8m36s   v1.26.2-eks-a59e1f0\nip-10-0-9-147.us-west-2.compute.internal    Ready    &lt;none&gt;   8m35s   v1.26.2-eks-a59e1f0\n</code></pre> </li> <li> <p>Test by listing all the Pods running currently. All the Pods should reach a status of <code>Running</code> after approximately 60 seconds:</p> <pre><code>kubectl get pods -A\n</code></pre> <pre><code>NAMESPACE     NAME                       READY   STATUS    RESTARTS   AGE\nkube-system   aws-node-j2n9x             1/1     Running   0          7m42s\nkube-system   aws-node-m1jlf             1/1     Running   0          7m45s\nkube-system   aws-node-q458h             1/1     Running   0          7m49s\nkube-system   coredns-6c45d94f67-495rr   1/1     Running   0          14m\nkube-system   coredns-6c45d94f67-5c8tc   1/1     Running   0          14m\nkube-system   kube-proxy-47wfh           1/1     Running   0          8m32s\nkube-system   kube-proxy-f6chz           1/1     Running   0          8m30s\nkube-system   kube-proxy-x2fkc           1/1     Running   0          8m31s\n</code></pre> </li> </ol>"},{"location":"patterns/fully-private-cluster/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/gitops-getting-started-argocd/","title":"GitOps Getting Started (ArgoCD)","text":""},{"location":"patterns/gitops-getting-started-argocd/#argocd-on-amazon-eks","title":"ArgoCD on Amazon EKS","text":"<p>This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD, employing the GitOps Bridge Pattern.</p> <p></p> <p>The GitOps Bridge Pattern enables Kubernetes administrators to utilize Infrastructure as Code (IaC) and GitOps tools for deploying Kubernetes Addons and Workloads. Addons often depend on Cloud resources that are external to the cluster. The configuration metadata for these external resources is required by the Addons' Helm charts. While IaC is used to create these cloud resources, it is not used to install the Helm charts. Instead, the IaC tool stores this metadata either within GitOps resources in the cluster or in a Git repository. The GitOps tool then extracts these metadata values and passes them to the Helm chart during the Addon installation process. This mechanism forms the bridge between IaC and GitOps, hence the term \"GitOps Bridge.\"</p> <p>Additional examples available on the GitOps Bridge Pattern:</p> <ul> <li>argocd-ingress</li> <li>aws-secrets-manager</li> <li>crossplane</li> <li>external-secrets</li> <li>multi-cluster/distributed</li> <li>multi-cluster/hub-spoke</li> <li>multi-cluster/hub-spoke-shared</li> <li>private-git</li> </ul>"},{"location":"patterns/gitops-getting-started-argocd/#prerequisites","title":"Prerequisites","text":"<p>Before you begin, make sure you have the following command line tools installed:</p> <ul> <li>git</li> <li>terraform</li> <li>kubectl</li> <li>argocd</li> </ul>"},{"location":"patterns/gitops-getting-started-argocd/#optional-fork-the-gitops-git-repositories","title":"(Optional) Fork the GitOps git repositories","text":"<p>See the appendix section Fork GitOps Repositories for more info on the terraform variables to override.</p>"},{"location":"patterns/gitops-getting-started-argocd/#deploy-the-eks-cluster","title":"Deploy the EKS Cluster","text":"<p>Initialize Terraform and deploy the EKS cluster:</p> <pre><code>terraform init\nterraform apply -target=\"module.vpc\" -auto-approve\nterraform apply -target=\"module.eks\" -auto-approve\nterraform apply -auto-approve\n</code></pre> <p>To retrieve <code>kubectl</code> config, execute the terraform output command:</p> <pre><code>terraform output -raw configure_kubectl\n</code></pre> <p>The expected output will have two lines you run in your terminal</p> <pre><code>export KUBECONFIG=\"/tmp/getting-started-gitops\"\naws eks --region us-west-2 update-kubeconfig --name getting-started-gitops\n</code></pre> <p>The first line sets the <code>KUBECONFIG</code> environment variable to a temporary file that includes the cluster name. The second line uses the <code>aws</code> CLI to populate that temporary file with the <code>kubectl</code> configuration. This approach offers the advantage of not altering your existing <code>kubectl</code> context, allowing you to work in other terminal windows without interference.</p> <p>Terraform will add GitOps Bridge Metadata to the ArgoCD secret. The annotations contain metadata for the addons' Helm charts and ArgoCD ApplicationSets.</p> <pre><code>kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster -o json | jq '.items[0].metadata.annotations'\n</code></pre> <p>The output looks like the following:</p> <pre><code>{\n  \"addons_repo_basepath\": \"argocd/\",\n  \"addons_repo_path\": \"bootstrap/control-plane/addons\",\n  \"addons_repo_revision\": \"main\",\n  \"addons_repo_url\": \"https://github.com/aws-samples/eks-blueprints-add-ons\",\n  \"aws_account_id\": \"0123456789\",\n  \"aws_cluster_name\": \"getting-started-gitops\",\n  \"aws_load_balancer_controller_iam_role_arn\": \"arn:aws:iam::0123456789:role/alb-controller\",\n  \"aws_load_balancer_controller_namespace\": \"kube-system\",\n  \"aws_load_balancer_controller_service_account\": \"aws-load-balancer-controller-sa\",\n  \"aws_region\": \"us-west-2\",\n  \"aws_vpc_id\": \"vpc-001d3f00151bbb731\",\n  \"cluster_name\": \"in-cluster\",\n  \"environment\": \"dev\",\n  \"workload_repo_basepath\": \"patterns/gitops/\",\n  \"workload_repo_path\": \"getting-started-argocd/k8s\",\n  \"workload_repo_revision\": \"main\",\n  \"workload_repo_url\": \"https://github.com/csantanapr/terraform-aws-eks-blueprints\"\n}\n</code></pre> <p>The labels offer a straightforward way to enable or disable an addon in ArgoCD for the cluster.</p> <pre><code>kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster -o json | jq '.items[0].metadata.labels' | grep -v false | jq .\n</code></pre> <p>The output looks like the following:</p> <pre><code>{\n  \"argocd.argoproj.io/secret-type\": \"cluster\",\n  \"aws_cluster_name\": \"getting-started-gitops\",\n  \"cluster_name\": \"in-cluster\",\n  \"enable_argocd\": \"true\",\n  \"enable_aws_load_balancer_controller\": \"true\",\n  \"enable_metrics_server\": \"true\",\n  \"environment\": \"dev\",\n  \"kubernetes_version\": \"1.28\"\n}\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#deploy-the-addons","title":"Deploy the Addons","text":"<p>Bootstrap the addons using ArgoCD:</p> <pre><code>kubectl apply -f bootstrap/addons.yaml\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#monitor-gitops-progress-for-addons","title":"Monitor GitOps Progress for Addons","text":"<p>Wait until all the ArgoCD applications' <code>HEALTH STATUS</code> is <code>Healthy</code>. Use <code>Ctrl+C</code> or <code>Cmd+C</code> to exit the <code>watch</code> command. ArgoCD Applications can take a couple of minutes in order to achieve the Healthy status.</p> <pre><code>kubectl get applications -n argocd -w\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                                            SYNC STATUS   HEALTH STATUS\naddon-in-cluster-argo-cd                        Synced        Healthy\naddon-in-cluster-aws-load-balancer-controller   Synced        Healthy\naddon-in-cluster-metrics-server                 Synced        Healthy\ncluster-addons                                  Synced        Healthy\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#verify-the-addons","title":"Verify the Addons","text":"<p>Verify that the addons are ready:</p> <pre><code>kubectl get deployment -n kube-system \\\n  aws-load-balancer-controller \\\n  metrics-server\nkubectl get deploy -n argocd \\\n  argo-cd-argocd-applicationset-controller \\\n  argo-cd-argocd-repo-server \\\n  argo-cd-argocd-server\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                                       READY   UP-TO-DATE   AVAILABLE   AGE\naws-load-balancer-controller               2/2     2            2           7m21s\nmetrics-server                             1/1     1            1           7m41s\nargo-cd-argocd-applicationset-controller   1/1     1            1           109m\nargo-cd-argocd-repo-server                 1/1     1            1           109m\nargo-cd-argocd-server                      1/1     1            1           109m\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#optional-access-argocd","title":"(Optional) Access ArgoCD","text":"<p>Access to the ArgoCD's UI is completely optional, if you want to do it, run the commands shown in the Terraform output as the example below:</p> <pre><code>terraform output -raw access_argocd\n</code></pre> <p>The expected output should contain the <code>kubectl</code> config followed by <code>kubectl</code> command to retrieve the URL, username, password to login into ArgoCD UI or CLI.</p> <pre><code>echo \"ArgoCD Username: admin\"\necho \"ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template=\"{{index .data.password | base64decode}}\")\"\necho \"ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\"\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#deploy-the-workloads","title":"Deploy the Workloads","text":"<p>Deploy a sample application located in k8s/game-2048.yaml using ArgoCD:</p> <pre><code>kubectl apply -f bootstrap/workloads.yaml\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#monitor-gitops-progress-for-workloads","title":"Monitor GitOps Progress for Workloads","text":"<p>Wait until all the ArgoCD applications' <code>HEALTH STATUS</code> is <code>Healthy</code>. Use <code>Ctrl+C</code> or <code>Cmd+C</code> to exit the <code>watch</code> command. ArgoCD Applications can take a couple of minutes in order to achieve the Healthy status.</p> <pre><code>watch kubectl get -n argocd applications workloads\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME        SYNC STATUS   HEALTH STATUS\nworkloads   Synced        Healthy\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#verify-the-application","title":"Verify the Application","text":"<p>Verify that the application configuration is present and the pod is running:</p> <pre><code>kubectl get -n game-2048 deployments,service,ep,ingress\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                        READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/game-2048   1/1     1            1           7h59m\n\nNAME                TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE\nservice/game-2048   ClusterIP   172.20.155.47   &lt;none&gt;        80/TCP    7h59m\n\nNAME                  ENDPOINTS       AGE\nendpoints/game-2048   10.0.13.64:80   7h59m\n\nNAME                CLASS   HOSTS   ADDRESS                              PORTS   AGE\ningress/game-2048   alb     *       k8s-&lt;&gt;.us-west-2.elb.amazonaws.com   80      7h59m\n</code></pre> <p>AWS Load Balancer can take a couple of minutes in order to be created.</p> <p>Run the following command and wait until and event for ingress <code>game-2048</code> contains <code>Successfully reconciled</code>. Use <code>Ctrl+C</code> or <code>Cmd+C</code>to exit the <code>watch</code> command.</p> <pre><code>kubectl events -n game-2048 --for ingress/game-2048 --watch\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>LAST SEEN   TYPE     REASON                   OBJECT              MESSAGE\n11m         Normal   SuccessfullyReconciled   Ingress/game-2048   Successfully reconciled\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#access-the-application-using-aws-load-balancer","title":"Access the Application using AWS Load Balancer","text":"<p>Verify the application endpoint health using <code>wget</code>:</p> <pre><code>kubectl exec -n game-2048 deploy/game-2048 -- \\\nwget -S --spider $(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>  HTTP/1.1 200 OK\n  Date: Wed, 01 Nov 2023 22:44:57 GMT\n  Content-Type: text/html\n  Content-Length: 3988\n</code></pre> <p>A success response should contain <code>HTTP/1.1 200 OK</code>.</p> <p>Retrieve the ingress URL to access the application in your local web browser.</p> <pre><code>echo \"Application URL: http://$(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\"\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#container-metrics","title":"Container Metrics","text":"<p>Check the application's CPU and memory metrics:</p> <pre><code>kubectl top pods -n game-2048\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                         CPU(cores)   MEMORY(bytes)\ngame-2048-66fb78b995-h1bjv   1m           2Mi\n</code></pre> <p>Check the CPU and memory metrics for all pods for Addons and Workloads:</p> <pre><code>kubectl top pods -A\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAMESPACE     NAME                                                        CPU(cores)   MEMORY(bytes)\nargocd        argo-cd-argocd-application-controller-0                     43m          138Mi\nargocd        argo-cd-argocd-applicationset-controller-5db688844c-79skp   1m           25Mi\nargocd        argo-cd-argocd-dex-server-cd48d7bc-x7flf                    1m           16Mi\nargocd        argo-cd-argocd-notifications-controller-7d7ccc6b9d-dg9r6    1m           17Mi\nargocd        argo-cd-argocd-redis-7f89c69877-6m2cj                       2m           3Mi\nargocd        argo-cd-argocd-repo-server-644b9b5668-m9ddg                 8m           62Mi\nargocd        argo-cd-argocd-server-57cbbd6f94-lp4wx                      2m           26Mi\ngame-2048     game-2048-66fb78b995-h1bjv                                  1m           2Mi\nkube-system   aws-load-balancer-controller-8488df87c-4nxv6                2m           26Mi\nkube-system   aws-load-balancer-controller-8488df87c-zs4p6                1m           19Mi\nkube-system   aws-node-ck6vq                                              3m           57Mi\nkube-system   aws-node-fv2sg                                              3m           56Mi\nkube-system   coredns-59754897cf-5r2xp                                    1m           13Mi\nkube-system   coredns-59754897cf-fn7jb                                    1m           13Mi\nkube-system   kube-proxy-lz2dc                                            1m           11Mi\nkube-system   kube-proxy-pd2lm                                            1m           12Mi\nkube-system   metrics-server-5b76987ff-5g1sv                              4m           17Mi\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#destroy-the-eks-cluster","title":"Destroy the EKS Cluster","text":"<p>To tear down all the resources and the EKS cluster, run the following command:</p> <pre><code>./destroy.sh\n</code></pre>"},{"location":"patterns/gitops-getting-started-argocd/#appendix","title":"Appendix","text":""},{"location":"patterns/gitops-getting-started-argocd/#fork-gitops-repositories","title":"Fork GitOps Repositories","text":"<p>To modify the <code>values.yaml</code> file for addons or the workload manifest files (.ie yaml), you'll need to fork two repositories: aws-samples/eks-blueprints-add-ons for addons and github.com/aws-ia/terraform-aws-eks-blueprints for workloads located in this pattern directory.</p> <p>After forking, update the following environment variables to point to your forks, replacing the default values.</p> <pre><code>export TF_VAR_gitops_addons_org=https://github.com/aws-samples\nexport TF_VAR_gitops_addons_repo=eks-blueprints-add-ons\nexport TF_VAR_gitops_addons_revision=main\n\nexport TF_VAR_gitops_workload_org=https://github.com/aws-ia\nexport TF_VAR_gitops_workload_repo=terraform-aws-eks-blueprints\nexport TF_VAR_gitops_workload_revision=main\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/","title":"GitOps Multi-Cluster Hub-Spoke Topology (ArgoCD)","text":""},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#multi-cluster-centralized-hub-spoke-topology","title":"Multi-Cluster centralized hub-spoke topology","text":"<p>This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD in a Multi-Cluster Hub-Spoke topology, employing the GitOps Bridge Pattern.</p> <p></p> <p>This example deploys ArgoCD on the Hub cluster (i.e. management/control-plane cluster). The spoke clusters are registered as remote clusters in the Hub Cluster's ArgoCD The ArgoCD on the Hub Cluster deploys addons and workloads to the spoke clusters</p> <p>Each spoke cluster gets deployed an app of apps ArgoCD Application with the name <code>workloads-${env}</code></p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#prerequisites","title":"Prerequisites","text":"<p>Before you begin, make sure you have the following command line tools installed:</p> <ul> <li>git</li> <li>terraform</li> <li>kubectl</li> <li>argocd</li> </ul>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#optional-fork-the-gitops-git-repositories","title":"(Optional) Fork the GitOps git repositories","text":"<p>See the appendix section Fork GitOps Repositories for more info on the terraform variables to override.</p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#deploy-the-hub-eks-cluster","title":"Deploy the Hub EKS Cluster","text":"<p>Change directory to <code>hub</code></p> <pre><code>cd hub\n</code></pre> <p>Initialize Terraform and deploy the EKS cluster:</p> <pre><code>terraform init\nterraform apply -target=\"module.vpc\" -auto-approve\nterraform apply -target=\"module.eks\" -auto-approve\nterraform apply -auto-approve\n</code></pre> <p>To retrieve <code>kubectl</code> config, execute the terraform output command:</p> <pre><code>terraform output -raw configure_kubectl\n</code></pre> <p>The expected output will have two lines you run in your terminal</p> <pre><code>export KUBECONFIG=\"/tmp/hub-spoke\"\naws eks --region us-west-2 update-kubeconfig --name getting-started-gitops --alias hub\n</code></pre> <p>The first line sets the <code>KUBECONFIG</code> environment variable to a temporary file that includes the cluster name. The second line uses the <code>aws</code> CLI to populate that temporary file with the <code>kubectl</code> configuration. This approach offers the advantage of not altering your existing <code>kubectl</code> context, allowing you to work in other terminal windows without interference.</p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#deploy-argocd-apps-of-applicationsets-for-addons","title":"Deploy ArgoCD Apps of ApplicationSets for Addons","text":"<p>This command verifies the initial ArgoCD installation, ArgoCD will be re-configured when the addons are deployed and configured from git. <pre><code>kubectl --context hub get all -n argocd\n</code></pre> This command creates the application set manifest to deploy the addons. <pre><code>kubectl --context hub apply -n argocd -f ../hub/bootstrap/addons.yaml\n</code></pre> The application sets defined here will then deploy addons to any spoke clusters provisioned later using Terraform</p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#monitor-gitops-progress-for-addons-on-hub-eks-cluster","title":"Monitor GitOps Progress for Addons on Hub EKS Cluster","text":"<p>Wait until all the ArgoCD applications' <code>HEALTH STATUS</code> is <code>Healthy</code>. Use <code>Ctrl+C</code> or <code>Cmd+C</code> to exit the <code>watch</code> command. ArgoCD Applications can take a couple of minutes in order to achieve the Healthy status.</p> <pre><code>kubectl --context hub get applications -n argocd -w\n</code></pre> <p>The expected output should look like the following:</p> <pre><code>NAME                                            SYNC STATUS   HEALTH STATUS\naddon-in-cluster-argo-cd                        Synced        Healthy\naddon-in-cluster-aws-load-balancer-controller   Synced        Healthy\naddon-in-cluster-metrics-server                 Synced        Healthy\ncluster-addons                                  Synced        Healthy\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#optional-access-argocd","title":"(Optional) Access ArgoCD","text":"<p>Access to the ArgoCD's UI is completely optional, if you want to do it, run the commands shown in the Terraform output as the example below:</p> <pre><code>terraform output -raw access_argocd\n</code></pre> <p>The expected output should contain the <code>kubectl</code> config followed by <code>kubectl</code> command to retrieve the URL, username, password to login into ArgoCD UI or CLI.</p> <pre><code>echo \"ArgoCD Username: admin\"\necho \"ArgoCD Password: $(kubectl --context hub get secrets argocd-initial-admin-secret -n argocd --template=\"{{index .data.password | base64decode}}\")\"\necho \"ArgoCD URL: https://$(kubectl --context hub get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\"\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#deploy-the-spoke-eks-cluster","title":"Deploy the Spoke EKS Cluster","text":"<p>Use the <code>deploy.sh</code> script to create terraform workspace, initialize Terraform, and deploy the EKS clusters:</p> <p>You may want to create few spoke environments to validate multi-cluster hub spoke to avoid quota limits</p> <pre><code>cd ../spokes\n./deploy.sh dev\n./deploy.sh staging\n./deploy.sh prod\n</code></pre> <p>Each environment uses a Terraform workspace</p> <p>To retrieve <code>kubectl</code> config, execute the terraform output command:</p> <pre><code>terraform workspace select dev\nterraform output -raw configure_kubectl\n</code></pre> <pre><code>terraform workspace select staging\nterraform output -raw configure_kubectl\n</code></pre> <pre><code>terraform workspace select prod\nterraform output -raw configure_kubectl\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#verify-argocd-cluster-secret-for-spokes-have-the-correct-iam-role-to-be-assume-by-hub-cluster","title":"Verify ArgoCD Cluster Secret for Spokes have the correct IAM Role to be assume by Hub Cluster","text":"<pre><code>for i in dev staging prod ; do echo $i &amp;&amp; kubectl --context hub get secret -n argocd spoke-$i --template='{{index .data.config | base64decode}}' ; done\n</code></pre> <p>The output have a section <code>awsAuthConfig</code> with the <code>clusterName</code> and the <code>roleARN</code> that has write access to the spoke cluster</p> <pre><code>{\n  \"tlsClientConfig\": {\n    \"insecure\": false,\n    \"caData\" : \"LS0tL....\"\n  },\n  \"awsAuthConfig\" : {\n    \"clusterName\": \"hub-spoke-dev\",\n    \"roleARN\": \"arn:aws:iam::0123456789:role/hub-spoke-dev-argocd-spoke\"\n  }\n}\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#verify-the-addons-on-spoke-clusters","title":"Verify the Addons on Spoke Clusters","text":"<p>The addons on the spoke clusters are deployed using the Application Sets created on the EKS Hub Cluster. Verify that the addons are ready:</p> <pre><code>for i in dev staging prod ; do echo $i &amp;&amp; kubectl --context $i get deployment -n kube-system ; done\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#deploy-the-sample-application-to-eks-spoke-clusters","title":"Deploy the sample application to EKS Spoke Clusters","text":"<p>This command will deploy the application using kubectl to all clusters connected to the hub cluster, using the manifest files in ./hub/bootstrap/workloads.yaml. <pre><code>kubectl --context hub apply -n argocd -f ../hub/bootstrap/workloads.yaml\n</code></pre></p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#monitor-gitops-progress-for-workloads-from-hub-cluster-run-on-hub-cluster-context","title":"Monitor GitOps Progress for Workloads from Hub Cluster (run on Hub Cluster context)","text":"<p>Watch until all the Workloads ArgoCD Applications are <code>Healthy</code></p> <pre><code>kubectl --context hub get -n argocd applications -w\n</code></pre> <p>Wait until the ArgoCD Applications <code>HEALTH STATUS</code> is <code>Healthy</code>. Crl+C to exit the <code>watch</code> command</p>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#verify-the-application","title":"Verify the Application","text":"<p>Verify that the application configuration is present and the pod is running:</p> <pre><code>for i in dev staging prod ; do echo $i &amp;&amp; kubectl --context $i get all -n workload ; done\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#container-metrics","title":"Container Metrics","text":"<p>Check the application's CPU and memory metrics:</p> <pre><code>for i in dev staging prod ; do echo $i &amp;&amp; kubectl --context $i top pods -n workload ; done\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#destroy-the-spoke-eks-clusters","title":"Destroy the Spoke EKS Clusters","text":"<p>To tear down all the resources and the EKS cluster, run the following command:</p> <pre><code>./destroy.sh dev\n./destroy.sh staging\n./destroy.sh prod\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#destroy-the-hub-eks-clusters","title":"Destroy the Hub EKS Clusters","text":"<p>To tear down all the resources and the EKS cluster, run the following command: Destroy Hub Clusters</p> <pre><code>cd ../hub\n./destroy.sh\n</code></pre>"},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#appendix","title":"Appendix","text":""},{"location":"patterns/gitops-multi-cluster-hub-spoke-argocd/#fork-gitops-repositories","title":"Fork GitOps Repositories","text":"<p>To modify the <code>values.yaml</code> file or the helm chart version for addons, you'll need to fork the repository aws-samples/eks-blueprints-add-ons.</p> <p>After forking, update the following environment variables to point to your forks, replacing the default values.</p> <pre><code>export TF_VAR_gitops_addons_org=https://github.com/aws-samples\nexport TF_VAR_gitops_addons_repo=eks-blueprints-add-ons\nexport TF_VAR_gitops_addons_revision=main\n</code></pre>"},{"location":"patterns/ipv6-eks-cluster/","title":"IPv6 Networking","text":""},{"location":"patterns/ipv6-eks-cluster/#amazon-eks-cluster-w-ipv6-networking","title":"Amazon EKS Cluster w/ IPv6 Networking","text":"<p>This pattern demonstrates an EKS cluster that utilizes IPv6 networking.</p>"},{"location":"patterns/ipv6-eks-cluster/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/ipv6-eks-cluster/#validate","title":"Validate","text":"<ol> <li> <p>Test by listing all the pods running currently; the <code>IP</code> should be an IPv6 address.</p> <pre><code>kubectl get pods -A -o wide\n</code></pre> <pre><code># Output should look like below\nNAMESPACE     NAME                       READY   STATUS    RESTARTS   AGE     IP                                       NODE                                        NOMINATED NODE   READINESS GATES\nkube-system   aws-node-1hd2s             1/1     Running   0          3m5s    2600:1f13:6c4:a703:ecf8:3ac1:76b0:9303   ip-10-0-10-183.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   aws-node-2mdg             1/1     Running   0          3m21s   2600:1f13:6c4:a705:a929:f8d4:9350:1b20   ip-10-0-12-188.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   coredns-799c5565b4-6xrc   1/1     Running   0          10m     2600:1f13:6c4:a705:1bda::                ip-10-0-12-188.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   coredns-799c5565b4-fjq4q   1/1     Running   0          10m     2600:1f13:6c4:a705:1bda::1               ip-10-0-12-188.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   kube-proxy-58tp7           1/1     Running   0          4m25s   2600:1f13:6c4:a703:ecf8:3ac1:76b0:9303   ip-10-0-10-183.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\nkube-system   kube-proxy-3kgw           1/1     Running   0          4m25s   2600:1f13:6c4:a705:a929:f8d4:9350:1b20   ip-10-0-12-188.us-west-2.compute.internal   &lt;none&gt;           &lt;none&gt;\n</code></pre> </li> <li> <p>Test by listing all the nodes running currently; the <code>INTERNAL-IP</code> should be an IPv6 address.</p> <pre><code>kubectl nodes -A -o wide\n</code></pre> <pre><code># Output should look like below\nNAME                                        STATUS   ROLES    AGE     VERSION               INTERNAL-IP                              EXTERNAL-IP   OS-IMAGE         KERNEL-VERSION                 CONTAINER-RUNTIME\nip-10-0-10-183.us-west-2.compute.internal   Ready    &lt;none&gt;   4m57s   v1.24.7-eks-fb459a0   2600:1f13:6c4:a703:ecf8:3ac1:76b0:9303   &lt;none&gt;        Amazon Linux 2   5.4.226-129.415.amzn2.x86_64   containerd://1.6.6\nip-10-0-12-188.us-west-2.compute.internal   Ready    &lt;none&gt;   4m57s   v1.24.7-eks-fb459a0   2600:1f13:6c4:a705:a929:f8d4:9350:1b20   &lt;none&gt;        Amazon Linux 2   5.4.226-129.415.amzn2.x86_64   containerd://1.6.6\n</code></pre> </li> </ol>"},{"location":"patterns/ipv6-eks-cluster/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/istio/","title":"Istio","text":""},{"location":"patterns/istio/#amazon-eks-cluster-w-istio","title":"Amazon EKS Cluster w/ Istio","text":"<p>This example shows how to provision an EKS cluster with Istio.</p> <ul> <li>Deploy EKS Cluster with one managed node group in an VPC</li> <li>Add node_security_group rules for port access required for Istio communication</li> <li>Install Istio using Helm resources in Terraform</li> <li>Install Istio Ingress Gateway using Helm resources in Terraform</li> <li>This step deploys a Service of type <code>LoadBalancer</code> that creates an AWS Network Load Balancer.</li> <li>Deploy/Validate Istio communication using sample application</li> </ul> <p>Refer to the documentation on Istio concepts.</p>"},{"location":"patterns/istio/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and run the following command to deploy this pattern.</p> <pre><code>terraform init\nterraform apply --auto-approve\n</code></pre> <p>Once the resources have been provisioned, you will need to replace the <code>istio-ingress</code> pods due to a <code>istiod</code> dependency issue. Use the following command to perform a rolling restart of the <code>istio-ingress</code> pods:</p> <pre><code>kubectl rollout restart deployment istio-ingress -n istio-ingress\n</code></pre>"},{"location":"patterns/istio/#observability-add-ons","title":"Observability Add-ons","text":"<p>Use the following code snippet to add the Istio Observability Add-ons on the EKS cluster with deployed Istio.</p> <pre><code>for ADDON in kiali jaeger prometheus grafana\ndo\n    ADDON_URL=\"https://raw.githubusercontent.com/istio/istio/release-1.20/samples/addons/$ADDON.yaml\"\n    kubectl apply -f $ADDON_URL\ndone\n</code></pre>"},{"location":"patterns/istio/#validate","title":"Validate","text":"<ol> <li> <p>List out all pods and services in the <code>istio-system</code> namespace:</p> <pre><code>kubectl get pods,svc -n istio-system\nkubectl get pods,svc -n istio-ingress\n</code></pre> <pre><code>NAME                             READY   STATUS    RESTARTS   AGE\npod/grafana-7d4f5589fb-4xj9m     1/1     Running   0          4m14s\npod/istiod-ff577f8b8-c8ssk       1/1     Running   0          4m40s\npod/jaeger-58c79c85cd-n7bkx      1/1     Running   0          4m14s\npod/kiali-749d76d7bb-8kjg7       1/1     Running   0          4m14s\npod/prometheus-5d5d6d6fc-s1txl   2/2     Running   0          4m15s\n\nNAME                       TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                 AGE\nservice/grafana            ClusterIP   172.20.141.12    &lt;none&gt;        3000/TCP                                4m14s\nservice/istiod             ClusterIP   172.20.172.70    &lt;none&gt;        15010/TCP,15012/TCP,443/TCP,15014/TCP   4m40s\nservice/jaeger-collector   ClusterIP   172.20.223.28    &lt;none&gt;        14268/TCP,14250/TCP,9411/TCP            4m15s\nservice/kiali              ClusterIP   172.20.182.231   &lt;none&gt;        20001/TCP,9090/TCP                      4m15s\nservice/prometheus         ClusterIP   172.20.89.64     &lt;none&gt;        9090/TCP                                4m14s\nservice/tracing            ClusterIP   172.20.253.201   &lt;none&gt;        80/TCP,16685/TCP                        4m14s\nservice/zipkin             ClusterIP   172.20.221.157   &lt;none&gt;        9411/TCP                                4m15s\n\nNAME                                 READY   STATUS    RESTARTS   AGE\npod/istio-ingress-6f7c5dffd8-g1szr   1/1     Running   0          4m28s\n\nNAME                    TYPE           CLUSTER-IP      EXTERNAL-IP                                                                     PORT(S)                                      AGE\nservice/istio-ingress   LoadBalancer   172.20.104.27   k8s-istioing-istioing-844c89b6c2-875b8c9a4b4e9365.elb.us-west-2.amazonaws.com   15021:32760/TCP,80:31496/TCP,443:32534/TCP   4m28s\n</code></pre> </li> <li> <p>Verify all the Helm releases installed in the <code>istio-system</code> and <code>istio-ingress</code> namespaces:</p> <pre><code>helm list -n istio-system\n</code></pre> <pre><code>NAME           NAMESPACE    REVISION UPDATED                              STATUS   CHART          APP VERSION\nistio-base    istio-system 1        2023-07-19 11:05:41.599921 -0700 PDT deployed base-1.18.1    1.18.1\nistiod        istio-system 1        2023-07-19 11:05:48.087616 -0700 PDT deployed istiod-1.18.1  1.18.1\n</code></pre> <pre><code>helm list -n istio-ingress\n</code></pre> <pre><code>NAME           NAMESPACE    REVISION UPDATED                              STATUS   CHART          APP VERSION\nistio-ingress istio-ingress 1        2023-07-19 11:06:03.41609 -0700 PDT  deployed gateway-1.18.1 1.18.1\n</code></pre> </li> </ol>"},{"location":"patterns/istio/#observability-add-ons_1","title":"Observability Add-ons","text":"<p>Validate the setup of the observability add-ons by running the following commands and accessing each of the service endpoints using this URL of the form http://localhost:\\ where <code>&lt;port&gt;</code> is one of the port number for the corresponding service. <pre><code># Visualize Istio Mesh console using Kiali\nkubectl port-forward svc/kiali 20001:20001 -n istio-system\n\n# Get to the Prometheus UI\nkubectl port-forward svc/prometheus 9090:9090 -n istio-system\n\n# Visualize metrics in using Grafana\nkubectl port-forward svc/grafana 3000:3000 -n istio-system\n\n# Visualize application traces via Jaeger\nkubectl port-forward svc/jaeger 16686:16686 -n istio-system\n</code></pre>"},{"location":"patterns/istio/#example","title":"Example","text":"<ol> <li> <p>Create the <code>sample</code> namespace and enable the sidecar injection on it</p> <pre><code>kubectl create namespace sample\nkubectl label namespace sample istio-injection=enabled\n</code></pre> <pre><code>namespace/sample created\nnamespace/sample labeled\n</code></pre> </li> <li> <p>Deploy <code>helloworld</code> app</p> <pre><code>cat &lt;&lt;EOF &gt; helloworld.yaml\napiVersion: v1\nkind: Service\nmetadata:\n  name: helloworld\n  labels:\n    app: helloworld\n    service: helloworld\nspec:\n  ports:\n  - port: 5000\n    name: http\n  selector:\n    app: helloworld\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: helloworld-v1\n  labels:\n    app: helloworld\n    version: v1\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app: helloworld\n      version: v1\n  template:\n    metadata:\n      labels:\n        app: helloworld\n        version: v1\n    spec:\n      containers:\n      - name: helloworld\n        image: docker.io/istio/examples-helloworld-v1\n        resources:\n          requests:\n            cpu: \"100m\"\n        imagePullPolicy: IfNotPresent #Always\n        ports:\n        - containerPort: 5000\nEOF\n\nkubectl apply -f helloworld.yaml -n sample\n</code></pre> <pre><code>service/helloworld created\ndeployment.apps/helloworld-v1 created\n</code></pre> </li> <li> <p>Deploy <code>sleep</code> app that we will use to connect to <code>helloworld</code> app</p> <pre><code>cat &lt;&lt;EOF &gt; sleep.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: sleep\n---\napiVersion: v1\nkind: Service\nmetadata:\n  name: sleep\n  labels:\n    app: sleep\n    service: sleep\nspec:\n  ports:\n  - port: 80\n    name: http\n  selector:\n    app: sleep\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: sleep\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app: sleep\n  template:\n    metadata:\n      labels:\n        app: sleep\n    spec:\n      terminationGracePeriodSeconds: 0\n      serviceAccountName: sleep\n      containers:\n      - name: sleep\n        image: curlimages/curl\n        command: [\"/bin/sleep\", \"infinity\"]\n        imagePullPolicy: IfNotPresent\n        volumeMounts:\n        - mountPath: /etc/sleep/tls\n          name: secret-volume\n      volumes:\n      - name: secret-volume\n        secret:\n          secretName: sleep-secret\n          optional: true\nEOF\n\nkubectl apply -f sleep.yaml -n sample\n</code></pre> <pre><code>serviceaccount/sleep created\nservice/sleep created\ndeployment.apps/sleep created\n</code></pre> </li> <li> <p>Check all the pods in the <code>sample</code> namespace</p> <pre><code>kubectl get pods -n sample\n</code></pre> <pre><code>NAME                           READY   STATUS    RESTARTS   AGE\nhelloworld-v1-b6c45f55-bx2xk   2/2     Running   0          50s\nsleep-9454cc476-p2zxr          2/2     Running   0          15s\n</code></pre> </li> <li> <p>Connect to <code>helloworld</code> app from <code>sleep</code> app and verify if the connection uses envoy proxy</p> <pre><code>kubectl exec -n sample -c sleep \\\n    \"$(kubectl get pod -n sample -l \\\n    app=sleep -o jsonpath='{.items[0].metadata.name}')\" \\\n    -- curl -v helloworld.sample:5000/hello\n</code></pre> <pre><code>* processing: helloworld.sample:5000/hello\n...\n* Connection #0 to host helloworld.sample left intact\n</code></pre> </li> </ol>"},{"location":"patterns/istio/#destroy","title":"Destroy","text":"<p>The AWS Load Balancer Controller add-on asynchronously reconciles resource deletions. During stack destruction, the istio ingress resource and the load balancer controller add-on are deleted in quick succession, preventing the removal of some of the AWS resources associated with the ingress gateway load balancer like, the frontend and the backend security groups. This causes the final <code>terraform destroy -auto-approve</code> command to timeout and fail with VPC dependency errors like below:</p> <pre><code>\u2502 Error: deleting EC2 VPC (vpc-XXXX): operation error EC2: DeleteVpc, https response error StatusCode: 400, RequestID: XXXXX-XXXX-XXXX-XXXX-XXXXXX, api error DependencyViolation: The vpc 'vpc-XXXX' has dependencies and cannot be deleted.\n</code></pre> <p>A possible workaround is to manually uninstall the <code>istio-ingress</code> helm chart.</p> <pre><code>terraform destroy -target='module.eks_blueprints_addons.helm_release.this[\"istio-ingress\"]' -auto-approve\n</code></pre> <p>Once the chart is uninstalled move on to destroy the stack.</p> <pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/karpenter-mng/","title":"Karpenter on EKS MNG","text":""},{"location":"patterns/karpenter-mng/#karpenter-on-eks-mng","title":"Karpenter on EKS MNG","text":"<p>This pattern demonstrates how to provision Karpenter on an EKS managed node group. Deploying onto standard EC2 instances created by an EKS managed node group will allow for daemonsets to run on the nodes created for the Karpenter controller, and therefore better unification of tooling across your data plane. This solution is comprised of the following components:</p> <ol> <li>An EKS managed node group that applies both a taint as well as a label for the Karpenter controller. We want the Karpenter controller to target these nodes via a <code>nodeSelector</code> in order to avoid the controller pods from running on nodes that Karpenter itself creates and manages. In addition, we are applying a taint to keep other pods off of these nodes as they are primarily intended for the controller pods. We apply a toleration to the CoreDNS addon, to allow those pods to run on the controller nodes as well. This is needed so that when a cluster is created, the CoreDNS pods have a place to run in order for the Karpenter controller to be provisioned and start managing the additional compute requirements for the cluster. Without letting CoreDNS run on these nodes, the controllers would fail to deploy and the data plane would be in a \"deadlock\" waiting for resources to deploy but unable to do so.</li> <li>The <code>eks-pod-identity-agent</code> addon has been provisioned to allow the Karpenter controller to utilize EKS Pod Identity for AWS permissions via an IAM role.</li> <li>The VPC subnets and node security group have been tagged with <code>\"karpenter.sh/discovery\" = local.name</code> for discoverability by the controller. The controller will discover these resources and use them to provision EC2 resources for the cluster.</li> <li>An IAM role for the Karpenter controller has been created with a trust policy that trusts the EKS Pod Identity service principal. This allows the EKS Pod Identity service to provide AWS credentials to the Karpenter controller pods in order to call AWS APIs.</li> <li>An IAM role for the nodes that Karpenter will create has been created along with a cluster access entry which allows the nodes to acquire permissions to join the cluster. Karpenter will create and manage the instance profile that utilizes this IAM role.</li> <li>An SQS queue has been created that is subscribed to certain EC2 CloudWatch events. This queue is used by Karpenter, allowing it to respond to certain EC2 lifecycle events and gracefully migrate pods off the instance before it is terminated.</li> </ol>"},{"location":"patterns/karpenter-mng/#code","title":"Code","text":"<p>The areas of significance related to this pattern are highlighted in the code provided below:</p> <pre><code>################################################################################\n# Cluster\n################################################################################\n\nmodule \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n  version = \"~&gt; 20.9\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.29\"\n\n  # Give the Terraform identity admin access to the cluster\n  # which will allow it to deploy resources into the cluster\n  enable_cluster_creator_admin_permissions = true\n  cluster_endpoint_public_access           = true\n\n  cluster_addons = {\n    coredns = {\n      configuration_values = jsonencode({\n        tolerations = [\n          # Allow CoreDNS to run on the same nodes as the Karpenter controller\n          # for use during cluster creation when Karpenter nodes do not yet exist\n          {\n            key    = \"karpenter.sh/controller\"\n            value  = \"true\"\n            effect = \"NoSchedule\"\n          }\n        ]\n      })\n    }\n    eks-pod-identity-agent = {}\n    kube-proxy             = {}\n    vpc-cni                = {}\n  }\n\n  enable_efa_support = true\n\n  vpc_id     = module.vpc.vpc_id\n  subnet_ids = module.vpc.private_subnets\n\n  eks_managed_node_groups = {\n    karpenter = {\n      instance_types = [\"m5.large\"]\n\n      min_size     = 2\n      max_size     = 3\n      desired_size = 2\n\n      labels = {\n        # Used to ensure Karpenter runs on nodes that it does not manage\n        \"karpenter.sh/controller\" = \"true\"\n      }\n\n      taints = {\n        # The pods that do not tolerate this taint should run on nodes\n        # created by Karpenter\n        karpenter = {\n          key    = \"karpenter.sh/controller\"\n          value  = \"true\"\n          effect = \"NO_SCHEDULE\"\n        }\n      }\n    }\n  }\n\n  tags = merge(local.tags, {\n    # NOTE - if creating multiple security groups with this module, only tag the\n    # security group that Karpenter should utilize with the following tag\n    # (i.e. - at most, only one security group should have this tag in your account)\n    \"karpenter.sh/discovery\" = local.name\n  })\n}\n\noutput \"configure_kubectl\" {\n  description = \"Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig\"\n  value       = \"aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}\"\n}\n\n################################################################################\n# Controller &amp; Node IAM roles, SQS Queue, Eventbridge Rules\n################################################################################\n\nmodule \"karpenter\" {\n  source  = \"terraform-aws-modules/eks/aws//modules/karpenter\"\n  version = \"~&gt; 20.9\"\n\n  cluster_name = module.eks.cluster_name\n\n  # Name needs to match role name passed to the EC2NodeClass\n  node_iam_role_use_name_prefix = false\n  node_iam_role_name            = local.name\n\n  tags = local.tags\n}\n\nresource \"aws_eks_pod_identity_association\" \"karpenter\" {\n  cluster_name    = module.eks.cluster_name\n  namespace       = \"kube-system\"\n  service_account = \"karpenter\"\n  role_arn        = module.karpenter.iam_role_arn\n}\n\n################################################################################\n# Helm charts\n################################################################################\n\nresource \"helm_release\" \"karpenter\" {\n  namespace           = \"kube-system\"\n  name                = \"karpenter\"\n  repository          = \"oci://public.ecr.aws/karpenter\"\n  repository_username = data.aws_ecrpublic_authorization_token.token.user_name\n  repository_password = data.aws_ecrpublic_authorization_token.token.password\n  chart               = \"karpenter\"\n  version             = \"0.36.1\"\n  wait                = false\n\n  values = [\n    &lt;&lt;-EOT\n    nodeSelector:\n      karpenter.sh/controller: 'true'\n    tolerations:\n      - key: CriticalAddonsOnly\n        operator: Exists\n      - key: karpenter.sh/controller\n        operator: Exists\n        effect: NoSchedule\n    settings:\n      clusterName: ${module.eks.cluster_name}\n      clusterEndpoint: ${module.eks.cluster_endpoint}\n      interruptionQueue: ${module.karpenter.queue_name}\n    EOT\n  ]\n\n  lifecycle {\n    ignore_changes = [\n      repository_password\n    ]\n  }\n}\n</code></pre>"},{"location":"patterns/karpenter-mng/#vpc","title":"VPC","text":"<pre><code>module \"vpc\" {\n  source  = \"terraform-aws-modules/vpc/aws\"\n  version = \"~&gt; 5.0\"\n\n  name = local.name\n  cidr = local.vpc_cidr\n\n  azs             = local.azs\n  private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)]\n  public_subnets  = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)]\n\n  enable_nat_gateway = true\n  single_nat_gateway = true\n\n  public_subnet_tags = {\n    \"kubernetes.io/role/elb\" = 1\n  }\n\n  private_subnet_tags = {\n    \"kubernetes.io/role/internal-elb\" = 1\n    # Tags subnets for Karpenter auto-discovery\n    \"karpenter.sh/discovery\" = local.name\n  }\n\n  tags = local.tags\n}\n</code></pre>"},{"location":"patterns/karpenter-mng/#ec2nodeclass-and-nodepool","title":"EC2NodeClass and NodePool","text":"<pre><code>---\napiVersion: karpenter.k8s.aws/v1beta1\nkind: EC2NodeClass\nmetadata:\n  name: default\nspec:\n  amiFamily: AL2\n  role: ex-karpenter-mng\n  subnetSelectorTerms:\n    - tags:\n        karpenter.sh/discovery: ex-karpenter-mng\n  securityGroupSelectorTerms:\n    - tags:\n        karpenter.sh/discovery: ex-karpenter-mng\n  tags:\n    karpenter.sh/discovery: ex-karpenter-mng\n---\napiVersion: karpenter.sh/v1beta1\nkind: NodePool\nmetadata:\n  name: default\nspec:\n  template:\n    spec:\n      nodeClassRef:\n        name: default\n      requirements:\n        - key: \"karpenter.k8s.aws/instance-category\"\n          operator: In\n          values: [\"c\", \"m\", \"r\"]\n        - key: \"karpenter.k8s.aws/instance-cpu\"\n          operator: In\n          values: [\"4\", \"8\", \"16\", \"32\"]\n        - key: \"karpenter.k8s.aws/instance-hypervisor\"\n          operator: In\n          values: [\"nitro\"]\n        - key: \"karpenter.k8s.aws/instance-generation\"\n          operator: Gt\n          values: [\"2\"]\n  limits:\n    cpu: 1000\n  disruption:\n    consolidationPolicy: WhenEmpty\n    consolidateAfter: 30s\n</code></pre>"},{"location":"patterns/karpenter-mng/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/karpenter-mng/#validate","title":"Validate","text":"<ol> <li> <p>Test by listing the nodes in the cluster. You should see four Fargate nodes in the cluster:</p> <pre><code>kubectl get nodes\n\nNAME                                      STATUS   ROLES    AGE     VERSION\nip-10-0-38-5.us-west-2.compute.internal   Ready    &lt;none&gt;   2m40s   v1.29.3-eks-ae9a62a\nip-10-0-9-38.us-west-2.compute.internal   Ready    &lt;none&gt;   2m35s   v1.29.3-eks-ae9a62a\n</code></pre> </li> <li> <p>Provision the Karpenter <code>EC2NodeClass</code> and <code>NodePool</code> resources which provide Karpenter the necessary configurations to provision EC2 resources:</p> <pre><code>kubectl apply -f karpenter.yaml\n</code></pre> </li> <li> <p>Once the Karpenter resources are in place, Karpenter will provision the necessary EC2 resources to satisfy any pending pods in the scheduler's queue. You can demonstrate this with the example deployment provided. First deploy the example deployment which has the initial number replicas set to 0:</p> <pre><code>kubectl apply -f example.yaml\n</code></pre> </li> <li> <p>When you scale the example deployment, you should see Karpenter respond by quickly provisioning EC2 resources to satisfy those pending pod requests:</p> <pre><code>kubectl scale deployment inflate --replicas=3\n</code></pre> </li> <li> <p>Listing the nodes should now show some EC2 compute that Karpenter has created for the example deployment:</p> <pre><code>kubectl get nodes\n\nNAME                                        STATUS   ROLES    AGE     VERSION\nip-10-0-38-109.us-west-2.compute.internal   Ready    &lt;none&gt;   11s     v1.29.3-eks-ae9a62a # &lt;== EC2 created by Karpenter\nip-10-0-38-5.us-west-2.compute.internal     Ready    &lt;none&gt;   3m54s   v1.29.3-eks-ae9a62a\nip-10-0-9-38.us-west-2.compute.internal     Ready    &lt;none&gt;   3m49s   v1.29.3-eks-ae9a62a\n</code></pre> </li> </ol>"},{"location":"patterns/karpenter-mng/#destroy","title":"Destroy","text":"<p>Scale down the deployment to de-provision Karpenter created resources first:</p> <pre><code>kubectl delete -f example.yaml\n</code></pre> <pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/karpenter/","title":"Karpenter on EKS Fargate","text":""},{"location":"patterns/karpenter/#karpenter-on-eks-fargate","title":"Karpenter on EKS Fargate","text":"<p>This pattern demonstrates how to provision Karpenter on a serverless cluster (serverless data plane) using Fargate Profiles.</p>"},{"location":"patterns/karpenter/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/karpenter/#validate","title":"Validate","text":"<ol> <li> <p>Test by listing the nodes in the cluster. You should see four Fargate nodes in the cluster:</p> <pre><code>kubectl get nodes\n\nNAME                                                STATUS   ROLES    AGE     VERSION\nfargate-ip-10-0-11-195.us-west-2.compute.internal   Ready    &lt;none&gt;   5m20s   v1.28.2-eks-f8587cb\nfargate-ip-10-0-27-183.us-west-2.compute.internal   Ready    &lt;none&gt;   5m2s    v1.28.2-eks-f8587cb\nfargate-ip-10-0-4-169.us-west-2.compute.internal    Ready    &lt;none&gt;   5m3s    v1.28.2-eks-f8587cb\nfargate-ip-10-0-44-106.us-west-2.compute.internal   Ready    &lt;none&gt;   5m12s   v1.28.2-eks-f8587cb\n</code></pre> </li> <li> <p>Provision the Karpenter <code>EC2NodeClass</code> and <code>NodePool</code> resources which provide Karpenter the necessary configurations to provision EC2 resources:</p> <pre><code>kubectl apply -f karpenter.yaml\n</code></pre> </li> <li> <p>Once the Karpenter resources are in place, Karpenter will provision the necessary EC2 resources to satisfy any pending pods in the scheduler's queue. You can demonstrate this with the example deployment provided. First deploy the example deployment which has the initial number replicas set to 0:</p> <pre><code>kubectl apply -f example.yaml\n</code></pre> </li> <li> <p>When you scale the example deployment, you should see Karpenter respond by quickly provisioning EC2 resources to satisfy those pending pod requests:</p> <pre><code>kubectl scale deployment inflate --replicas=3\n</code></pre> </li> <li> <p>Listing the nodes should now show some EC2 compute that Karpenter has created for the example deployment:</p> <pre><code>kubectl get nodes\n\nNAME                                                STATUS   ROLES    AGE   VERSION\nfargate-ip-10-0-11-195.us-west-2.compute.internal   Ready    &lt;none&gt;   13m   v1.28.2-eks-f8587cb\nfargate-ip-10-0-27-183.us-west-2.compute.internal   Ready    &lt;none&gt;   12m   v1.28.2-eks-f8587cb\nfargate-ip-10-0-4-169.us-west-2.compute.internal    Ready    &lt;none&gt;   12m   v1.28.2-eks-f8587cb\nfargate-ip-10-0-44-106.us-west-2.compute.internal   Ready    &lt;none&gt;   13m   v1.28.2-eks-f8587cb\nip-10-0-32-199.us-west-2.compute.internal           Ready    &lt;none&gt;   29s   v1.28.2-eks-a5df82a # &lt;== EC2 created by Karpenter\n</code></pre> </li> </ol>"},{"location":"patterns/karpenter/#destroy","title":"Destroy","text":"<p>Scale down the deployment to de-provision Karpenter created resources first:</p> <pre><code>kubectl delete -f example.yaml\n</code></pre> <pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/kubecost/","title":"Kubecost","text":""},{"location":"patterns/kubecost/#kubecost-with-aws-cloud-billing-integration","title":"Kubecost with AWS Cloud Billing Integration","text":"<p>This pattern demonstrates how to install and configure Kubecost with AWS CUR report. The terraform code was created following the official Kubecost documentation for aws cloud billing integration.</p>"},{"location":"patterns/kubecost/#prerequisites","title":"Prerequisites","text":"<p>You need a valid Kubecost token. To generate one, follow the instructions here.</p>"},{"location":"patterns/kubecost/#deploy","title":"Deploy","text":"<pre><code>terraform init\nterraform apply -target=\"module.vpc\" -auto-approve\nterraform apply -target=\"module.eks\" -auto-approve\nterraform apply -auto-approve --var=\"kubecost_token=&lt;your-kubecost-token&gt;\"\n</code></pre> <p>Once all of the resources have successfully been provisioned, the following command can be used to update the <code>kubeconfig</code> on your local machine and allow you to interact with your EKS Cluster using <code>kubectl</code>.</p> <pre><code>aws eks --region &lt;REGION&gt; update-kubeconfig --name &lt;CLUSTER_NAME&gt;\n</code></pre> <p>This command will create a S3 bucket with prefix <code>kubecost-</code> and a Cost and Usage Report (CUR). Within 24h The CUR will generate a CloudFormation template file called <code>crawler-cfn.yml</code> in the S3 bucket. Once that file is generated, navigate to:</p> <pre><code>cd run-me-in-24h/\n</code></pre> <p>To download and apply the CloudFormation template, run:</p> <pre><code>terraform init\nterraform apply --auto-approve\n</code></pre>"},{"location":"patterns/kubecost/#kubecost-ui","title":"Kubecost UI","text":"<p>To access the Kubecost UI run:</p> <pre><code>echo http://$(kubectl -n kubecost get svc cost-analyzer-cost-analyzer \\\n    -o jsonpath='{.status.loadBalancer.ingress[0].hostname}'):9090/\n</code></pre> <p>and then navigate to the output URL.</p> <p>Navigate to Settings -&gt; Diagnostics -&gt; View Full Diagnostics</p> <p>Expected result:</p> <p>Note</p> <p>Spot Data Feed is included in Savings Plan, Reserved Instance, and Out-Of-Cluster.</p>"},{"location":"patterns/kubecost/#destroy","title":"Destroy","text":"<p>First destroy the CloudFormation template:</p> <pre><code>cd run-me-in-24h/\nterraform destroy --auto-approve\ncd ..\nterraform destroy -target=\"module.eks_blueprints_addon\" --var=\"kubecost_token=&lt;your-kubecost-token&gt;\" -auto-approve\nterraform destroy -target=\"module.eks_blueprints_addons\"\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre>"},{"location":"patterns/ml-capacity-block/","title":"ML Capacity Block Reservation (CBR)","text":""},{"location":"patterns/ml-capacity-block/#eks-w-ml-capacity-block-reservation-cbr","title":"EKS w/ ML Capacity Block Reservation (CBR)","text":"<p>This pattern demonstrates how to consume/utilize ML capacity block reservations (CBR) with Amazon EKS. The solution is comprised of primarily 2 components:</p> <p>Warning</p> <p>The use of self-managed node group(s) are required at this time to support capacity block reservations within EKS. This pattern will be updated to demonstrate EKS managed node groups once support has been implemented by the EKS service.</p> <ol> <li> <p>The self-managed node group that will utilize the CBR should have the subnets provided to it restricted to the availability zone where the CBR has been allocated. For example - if the CBR is allocated to <code>us-west-2b</code>, the node group should only have subnet IDs provided to it that reside in <code>us-west-2b</code>. If the subnets that reside in other AZs are provided, its possible to encounter an error such as <code>InvalidParameterException: The following supplied instance types do not exist ...</code>. It is not guaranteed that this error will always be shown, and may appear random since the underlying autoscaling group(s) will provision nodes into different AZs at random. It will only occur when the underlying autoscaling group tries to provision instances into an AZ where capacity is not allocated and there is insufficient on-demand capacity for the desired instance type.</p> </li> <li> <p>The launch template utilized should specify the <code>instance_market_options</code> and <code>capacity_reservation_specification</code> arguments. This is how the CBR is utilized by the node group (i.e. - tells the autoscaling group to launch instances utilizing provided capacity reservation).</p> </li> </ol> <p>Links:</p> <ul> <li>EKS - Capacity Blocks for ML</li> <li>EC2 - Capacity Blocks for ML</li> </ul>"},{"location":"patterns/ml-capacity-block/#code","title":"Code","text":"<pre><code>################################################################################\n# Required Input\n################################################################################\n\n# See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/capacity-blocks-using.html\n# on how to obtain a ML capacity block reservation. Once acquired, you can provide\n# the reservation ID through this input to deploy the pattern\nvariable \"capacity_reservation_id\" {\n  description = \"The ID of the ML capacity block reservation to use for the node group\"\n  type        = string\n}\n\n################################################################################\n# Cluster\n################################################################################\n\nmodule \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n  version = \"~&gt; 20.9\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.29\"\n\n  # Give the Terraform identity admin access to the cluster\n  # which will allow it to deploy resources into the cluster\n  enable_cluster_creator_admin_permissions = true\n  cluster_endpoint_public_access           = true\n\n  cluster_addons = {\n    coredns    = {}\n    kube-proxy = {}\n    vpc-cni    = {}\n  }\n\n  # Add security group rules on the node group security group to\n  # allow EFA traffic\n  enable_efa_support = true\n\n  vpc_id     = module.vpc.vpc_id\n  subnet_ids = module.vpc.private_subnets\n\n  eks_managed_node_groups = {\n    # This node group is for core addons such as CoreDNS\n    default = {\n      instance_types = [\"m5.large\"]\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 2\n    }\n  }\n\n  # Note: ML capacity block reservations are only supported\n  # on self-managed node groups at this time\n  self_managed_node_groups = {\n    odcr = {\n      # The EKS AL2 GPU AMI provides all of the necessary components\n      # for accelerated workloads w/ EFA\n      ami_type      = \"AL2_x86_64_GPU\"\n      instance_type = \"p5.48xlarge\"\n\n      pre_bootstrap_user_data = &lt;&lt;-EOT\n        # Mount instance store volumes in RAID-0 for kubelet and containerd\n        # https://github.com/awslabs/amazon-eks-ami/blob/master/doc/USER_GUIDE.md#raid-0-for-kubelet-and-containerd-raid0\n        /bin/setup-local-disks raid0\n\n        # Ensure only GPU workloads are scheduled on this node group\n        export KUBELET_EXTRA_ARGS='--node-labels=vpc.amazonaws.com/efa.present=true,nvidia.com/gpu.present=true \\\n          --register-with-taints=nvidia.com/gpu=true:NoSchedule'\n\n      EOT\n\n      min_size     = 2\n      max_size     = 2\n      desired_size = 2\n\n      # This will:\n      # 1. Create a placement group to place the instances close to one another\n      # 2. Ignore subnets that reside in AZs that do not support the instance type\n      # 3. Expose all of the available EFA interfaces on the launch template\n      enable_efa_support = true\n\n      # ML capacity block reservation\n      instance_market_options = {\n        market_type = \"capacity-block\"\n      }\n      capacity_reservation_specification = {\n        capacity_reservation_target = {\n          capacity_reservation_id = var.capacity_reservation_id\n        }\n      }\n    }\n  }\n\n  tags = local.tags\n}\n\n################################################################################\n# Helm charts\n################################################################################\n\nresource \"helm_release\" \"nvidia_device_plugin\" {\n  name             = \"nvidia-device-plugin\"\n  repository       = \"https://nvidia.github.io/k8s-device-plugin\"\n  chart            = \"nvidia-device-plugin\"\n  version          = \"0.14.5\"\n  namespace        = \"nvidia-device-plugin\"\n  create_namespace = true\n  wait             = false\n\n  values = [\n    &lt;&lt;-EOT\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n            - matchExpressions:\n              - key: 'nvidia.com/gpu.present'\n                operator: In\n                values:\n                - 'true'\n    EOT\n  ]\n}\n\nresource \"helm_release\" \"aws_efa_device_plugin\" {\n  name       = \"aws-efa-k8s-device-plugin\"\n  repository = \"https://aws.github.io/eks-charts\"\n  chart      = \"aws-efa-k8s-device-plugin\"\n  version    = \"v0.4.4\"\n  namespace  = \"kube-system\"\n  wait       = false\n\n  values = [\n    &lt;&lt;-EOT\n      nodeSelector:\n        vpc.amazonaws.com/efa.present: 'true'\n      tolerations:\n        - key: nvidia.com/gpu\n          operator: Exists\n          effect: NoSchedule\n    EOT\n  ]\n}\n</code></pre>"},{"location":"patterns/ml-capacity-block/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/ml-capacity-block/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/multi-tenancy-with-teams/","title":"Multi-Tenancy w/ Teams","text":""},{"location":"patterns/multi-tenancy-with-teams/#multi-tenancy-w-teams","title":"Multi-Tenancy w/ Teams","text":"<p>This pattern demonstrates how to provision and configure a multi-tenancy Amazon EKS cluster with safeguards for resource consumption and namespace isolation.</p> <p>This example solution provides:</p> <ul> <li>Two development teams - <code>team-red</code> and <code>team-blue</code> - isolated to their respective namespaces</li> <li>An admin team with privileged access to the cluster (<code>team-admin</code>)</li> </ul>"},{"location":"patterns/multi-tenancy-with-teams/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/multi-tenancy-with-teams/#validate","title":"Validate","text":"<p>TODO</p> <p>Add in validation steps</p>"},{"location":"patterns/multi-tenancy-with-teams/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/nvidia-gpu-efa/","title":"NVIDIA GPUs with EFA","text":""},{"location":"patterns/nvidia-gpu-efa/#eks-cluster-w-nvidia-gpus-and-efa-for-machine-learning","title":"EKS Cluster w/ NVIDIA GPUs and EFA for Machine Learning","text":"<p>This pattern demonstrates an Amazon EKS Cluster with an EFA-enabled nodegroup that utilizes <code>p5.48xlarge</code> instances with H100 NVIDIA GPUs used in distributed, multi-node machine learning workloads.</p> <p>The following components are demonstrated in this pattern:</p> <ul> <li>A \"default\" node group that supports addons and components that do not require GPUs nor EFA devices. Any pods that do not tolerate the taints of the GPU node group will be scheduled on instances within this node group.</li> <li>A node group of <code>p5.48xlarge</code> instances with</li> <li>all x32 EFA network interfaces enabled</li> <li>provisioned within a placement group so that the instances are provisioned close to one another in a single availability zone that supports the instance type.</li> <li>a common NVIDIA taint of <code>\"nvidia.com/gpu:NoSchedule\"</code> to ensure only the intended applications are allowed to run on the nodes created</li> <li>two labels to identify that this nodegroup supports NVIDIA GPUs and EFA devices and allow pods to use node selectors with these labels</li> <li>the NVME instance store volumes are mounted in a RAID-0 array to provide a single, large, high-performance storage volume for the GPU workloads</li> <li>kubelet and containerd are configured to utilize the RAID-0 volume, allowing kubelet to discover the additional storage as ephemeral storage that can be utilized by pods</li> <li>A Helm chart deployment for the NVIDIA device plugin to expose and mount the GPUs provided by the instances to the pods that request them</li> <li>A Helm chart deployment for the EFA device plugin to expose and mount the EFA network interfaces provided by the instances to the pods that request them. Since the EFA network interfaces are only found on the instances that provide NVIDIA GPUs in this pattern, we do not apply an additional taint for the EFA network interfaces to avoid over-constraining.</li> </ul>"},{"location":"patterns/nvidia-gpu-efa/#code","title":"Code","text":"<pre><code>################################################################################\n# Cluster\n################################################################################\n\nmodule \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n  version = \"~&gt; 20.9\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.29\"\n\n  # Give the Terraform identity admin access to the cluster\n  # which will allow it to deploy resources into the cluster\n  enable_cluster_creator_admin_permissions = true\n  cluster_endpoint_public_access           = true\n\n  cluster_addons = {\n    coredns    = {}\n    kube-proxy = {}\n    vpc-cni    = {}\n  }\n\n  # Add security group rules on the node group security group to\n  # allow EFA traffic\n  enable_efa_support = true\n\n  vpc_id     = module.vpc.vpc_id\n  subnet_ids = module.vpc.private_subnets\n\n  eks_managed_node_groups = {\n    nvidia-efa = {\n      # The EKS AL2 GPU AMI provides all of the necessary components\n      # for accelerated workloads w/ EFA\n      ami_type       = \"AL2_x86_64_GPU\"\n      instance_types = [\"p5.48xlarge\"]\n\n      pre_bootstrap_user_data = &lt;&lt;-EOT\n        #!/usr/bin/env bash\n\n        # Mount instance store volumes in RAID-0 for kubelet and containerd\n        # https://github.com/awslabs/amazon-eks-ami/blob/master/doc/USER_GUIDE.md#raid-0-for-kubelet-and-containerd-raid0\n        /bin/setup-local-disks raid0\n      EOT\n\n      min_size     = 2\n      max_size     = 2\n      desired_size = 2\n\n      # This will:\n      # 1. Create a placement group to place the instances close to one another\n      # 2. Ignore subnets that reside in AZs that do not support the instance type\n      # 3. Expose all of the available EFA interfaces on the launch template\n      enable_efa_support = true\n\n      labels = {\n        \"vpc.amazonaws.com/efa.present\" = \"true\"\n        \"nvidia.com/gpu.present\"        = \"true\"\n      }\n\n      taints = {\n        # Ensure only GPU workloads are scheduled on this node group\n        gpu = {\n          key    = \"nvidia.com/gpu\"\n          value  = \"true\"\n          effect = \"NO_SCHEDULE\"\n        }\n      }\n    }\n\n    # This node group is for core addons such as CoreDNS\n    default = {\n      instance_types = [\"m5.large\"]\n\n      # Default AMI has only 8GB of storage\n      block_device_mappings = {\n        xvda = {\n          device_name = \"/dev/xvda\"\n          ebs = {\n            volume_size           = 128\n            volume_type           = \"gp3\"\n            delete_on_termination = true\n          }\n        }\n      }\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 2\n    }\n  }\n\n  tags = local.tags\n}\n\n################################################################################\n# Helm charts\n################################################################################\n\nresource \"helm_release\" \"nvidia_device_plugin\" {\n  name             = \"nvidia-device-plugin\"\n  repository       = \"https://nvidia.github.io/k8s-device-plugin\"\n  chart            = \"nvidia-device-plugin\"\n  version          = \"0.14.5\"\n  namespace        = \"nvidia-device-plugin\"\n  create_namespace = true\n  wait             = false\n\n  values = [\n    &lt;&lt;-EOT\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n            - matchExpressions:\n              - key: 'nvidia.com/gpu.present'\n                operator: In\n                values:\n                - 'true'\n    EOT\n  ]\n}\n\nresource \"helm_release\" \"aws_efa_device_plugin\" {\n  name       = \"aws-efa-k8s-device-plugin\"\n  repository = \"https://aws.github.io/eks-charts\"\n  chart      = \"aws-efa-k8s-device-plugin\"\n  version    = \"v0.4.4\"\n  namespace  = \"kube-system\"\n  wait       = false\n\n  values = [\n    &lt;&lt;-EOT\n      nodeSelector:\n        vpc.amazonaws.com/efa.present: 'true'\n      tolerations:\n        - key: nvidia.com/gpu\n          operator: Exists\n          effect: NoSchedule\n    EOT\n  ]\n}\n</code></pre>"},{"location":"patterns/nvidia-gpu-efa/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/nvidia-gpu-efa/#validate","title":"Validate","text":"<p>Note</p> <p>The following steps are shown with <code>g5.8xlarge</code> for frugality. Values shown below will change based on the instance type selected (i.e. - <code>p5.48xlarge</code> has 8 GPUs and 32 EFA interfaces)</p> <ol> <li> <p>List the nodes by instance type:</p> <pre><code>kubectl get nodes -o yaml | grep instance-type | grep node | grep -v f:\n</code></pre> <pre><code>node.kubernetes.io/instance-type: g5.8xlarge\nnode.kubernetes.io/instance-type: m5.large\nnode.kubernetes.io/instance-type: m5.large\nnode.kubernetes.io/instance-type: g5.8xlarge\n</code></pre> <p>You should see two EFA-enabled (in this example <code>g5.8xlarge</code>) nodes in the list.</p> </li> <li> <p>Deploy Kubeflow MPI Operator</p> <p>Kubeflow MPI Operator is required for running MPIJobs on EKS. We will use an MPIJob to test EFA. To deploy the MPI operator execute the following:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/kubeflow/mpi-operator/v0.3.0/deploy/v2beta1/mpi-operator.yaml\n</code></pre> <pre><code>namespace/mpi-operator created\ncustomresourcedefinition.apiextensions.k8s.io/mpijobs.kubeflow.org created\nserviceaccount/mpi-operator created\nclusterrole.rbac.authorization.k8s.io/kubeflow-mpijobs-admin created\nclusterrole.rbac.authorization.k8s.io/kubeflow-mpijobs-edit created\nclusterrole.rbac.authorization.k8s.io/kubeflow-mpijobs-view created\nclusterrole.rbac.authorization.k8s.io/mpi-operator created\nclusterrolebinding.rbac.authorization.k8s.io/mpi-operator created\ndeployment.apps/mpi-operator created\n</code></pre> <p>In addition to deploying the operator, please apply a patch to the mpi-operator clusterrole to allow the mpi-operator service account access to <code>leases</code> resources in the <code>coordination.k8s.io</code> apiGroup.</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/aws-samples/aws-do-eks/main/Container-Root/eks/deployment/kubeflow/mpi-operator/clusterrole-mpi-operator.yaml\n</code></pre> <pre><code>clusterrole.rbac.authorization.k8s.io/mpi-operator configured\n</code></pre> </li> <li> <p>EFA test</p> <p>The results should shown that two EFA adapters are available (one for each worker pod)</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/aws-samples/aws-do-eks/main/Container-Root/eks/deployment/efa-device-plugin/test-efa.yaml\n</code></pre> <pre><code>mpijob.kubeflow.org/efa-info-test created\n</code></pre> <p>Once the test launcher pod enters status <code>Running</code> or <code>Completed</code>, see the test logs using the command below:</p> <pre><code>kubectl logs -f $(kubectl get pods | grep launcher | cut -d ' ' -f 1)\n</code></pre> <pre><code>Warning: Permanently added 'efa-info-test-worker-1.efa-info-test-worker.default.svc,10.11.13.224' (ECDSA) to the list of known hosts.\nWarning: Permanently added 'efa-info-test-worker-0.efa-info-test-worker.default.svc,10.11.4.63' (ECDSA) to the list of known hosts.\n[1,1]&lt;stdout&gt;:provider: efa\n[1,1]&lt;stdout&gt;:    fabric: efa\n[1,1]&lt;stdout&gt;:    domain: rdmap197s0-rdm\n[1,1]&lt;stdout&gt;:    version: 116.10\n[1,1]&lt;stdout&gt;:    type: FI_EP_RDM\n[1,1]&lt;stdout&gt;:    protocol: FI_PROTO_EFA\n[1,0]&lt;stdout&gt;:provider: efa\n[1,0]&lt;stdout&gt;:    fabric: efa\n[1,0]&lt;stdout&gt;:    domain: rdmap197s0-rdm\n[1,0]&lt;stdout&gt;:    version: 116.10\n[1,0]&lt;stdout&gt;:    type: FI_EP_RDM\n[1,0]&lt;stdout&gt;:    protocol: FI_PROTO_EFA\n</code></pre> </li> <li> <p>EFA NCCL test</p> <p>To run the EFA NCCL test please execute the following kubectl command:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/aws-samples/aws-do-eks/main/Container-Root/eks/deployment/efa-device-plugin/test-nccl-efa.yaml\n</code></pre> <pre><code>mpijob.kubeflow.org/test-nccl-efa created\n</code></pre> <p>Once the launcher pod enters <code>Running</code> or <code>Completed</code> state, execute the following to see the test logs:</p> <pre><code>kubectl logs -f $(kubectl get pods | grep launcher | cut -d ' ' -f 1)\n</code></pre> <pre><code>[1,0]&lt;stdout&gt;:test-nccl-efa-worker-0:21:21 [0] NCCL INFO NET/OFI Selected Provider is efa (found 1 nics)\n[1,0]&lt;stdout&gt;:test-nccl-efa-worker-0:21:21 [0] NCCL INFO Using network AWS Libfabric\n[1,0]&lt;stdout&gt;:NCCL version 2.12.7+cuda11.4\n</code></pre> <p>Columns 8 and 12 in the output table show the in-place and out-of-place bus bandwidth calculated for the data size listed in column 1. In this case it is 3.13 and 3.12 GB/s respectively. Your actual results may be slightly different. The calculated average bus bandwidth is displayed at the bottom of the log when the test finishes after it reaches the max data size, specified in the mpijob manifest. In this result the average bus bandwidth is 1.15 GB/s.</p> <pre><code>[1,0]&lt;stdout&gt;:#       size         count      type   redop    root     time   algbw   busbw #wrong     time   algbw   busbw #wrong\n[1,0]&lt;stdout&gt;:#        (B)    (elements)                               (us)  (GB/s)  (GB/s)            (us)  (GB/s)  (GB/s)\n...\n[1,0]&lt;stdout&gt;:      262144         65536     float     sum      -1    195.0    1.34    1.34      0    194.0    1.35    1.35      0\n[1,0]&lt;stdout&gt;:      524288        131072     float     sum      -1    296.9    1.77    1.77      0    291.1    1.80    1.80      0\n[1,0]&lt;stdout&gt;:     1048576        262144     float     sum      -1    583.4    1.80    1.80      0    579.6    1.81    1.81      0\n[1,0]&lt;stdout&gt;:     2097152        524288     float     sum      -1    983.3    2.13    2.13      0    973.9    2.15    2.15      0\n[1,0]&lt;stdout&gt;:     4194304       1048576     float     sum      -1   1745.4    2.40    2.40      0   1673.2    2.51    2.51      0\n...\n[1,0]&lt;stdout&gt;:# Avg bus bandwidth    : 1.15327\n</code></pre> </li> </ol>"},{"location":"patterns/nvidia-gpu-efa/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/private-public-ingress/","title":"Private and Public Ingress","text":""},{"location":"patterns/private-public-ingress/#amazon-eks-private-and-public-ingress-example","title":"Amazon EKS Private and Public Ingress example","text":"<p>This example demonstrates how to provision an Amazon EKS cluster with two  ingress-nginx controllers; one to expose applications publicly and the other to expose applications internally. It also assigns security groups to the Network Load Balancers used to expose the internal and external ingress controllers.</p> <p>This solution:</p> <ul> <li>Installs an ingress-nginx controller for public traffic</li> <li>Installs an ingress-nginx controller for internal traffic</li> </ul> <p>To expose your application services via an <code>Ingress</code> resource with this solution you can set the respective <code>ingressClassName</code> as either <code>ingress-nginx-external</code> or <code>ingress-nginx-internal</code>.</p> <p>Refer to the documentation for <code>AWS Load Balancer controller</code> configuration options.</p>"},{"location":"patterns/private-public-ingress/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/private-public-ingress/#validate","title":"Validate","text":"<p>TODO</p> <p>Add in validation steps</p>"},{"location":"patterns/private-public-ingress/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/privatelink-access/","title":"PrivateLink Access","text":""},{"location":"patterns/privatelink-access/#private-eks-cluster-access-via-aws-privatelink","title":"Private EKS cluster access via AWS PrivateLink","text":"<p>This pattern demonstrates how to access a private EKS cluster using AWS PrivateLink.</p> <p>Refer to the documentation for further details on  <code>AWS PrivateLink</code>.</p>"},{"location":"patterns/privatelink-access/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and follow the steps below to deploy this pattern.</p> <pre><code>terraform init\nterraform apply -target=module.eventbridge -target=module.nlb --auto-approve\nterraform apply --auto-approve\n</code></pre>"},{"location":"patterns/privatelink-access/#test-access-to-eks-kubernetes-api-server-endpoint","title":"Test access to EKS Kubernetes API server endpoint","text":"<p>Of the output values, the value <code>ssm_test</code> is provided to aid in quickly testing the connectivity from the client EC2 instance to the private EKS cluster via AWS PrivateLink. Copy the output value, which looks like the snippet shown below (as an example) and paste it into your terminal to execute and check the connectivity. If configured correctly, the value returned should be <code>ok</code>.</p> <pre><code>COMMAND=\"curl -ks https://9A85B21811733524E3ABCDFEA8714642.gr7.us-west-2.eks.amazonaws.com/readyz\"\n\nCOMMAND_ID=$(aws ssm send-command --region us-west-2 \\\n   --document-name \"AWS-RunShellScript\" \\\n   --parameters \"commands=[$COMMAND]\" \\\n   --targets \"Key=instanceids,Values=i-0a45eff73ba408575\" \\\n   --query 'Command.CommandId' \\\n   --output text)\n\naws ssm get-command-invocation --region us-west-2 \\\n   --command-id $COMMAND_ID \\\n   --instance-id i-0a45eff73ba408575 \\\n   --query 'StandardOutputContent' \\\n   --output text\n</code></pre>"},{"location":"patterns/privatelink-access/#test-access-to-eks-kubernetes-api-with-kubectl","title":"Test access to EKS Kubernetes API with <code>kubectl</code>","text":"<p>Perform the following steps to access the cluster with <code>kubectl</code> from the provided Client EC2 instance.</p>"},{"location":"patterns/privatelink-access/#log-into-the-client-ec2-instance","title":"Log into the Client EC2 instance","text":"<p>Start a new SSM session on the Client EC2 instance using the provided <code>ssm_start_session</code> output value. It should look similar to the snippet shown below. Copy the output value and paste it into your terminal to execute. Your terminal will now be connected to the Client EC2 instance.</p> <pre><code>aws ssm start-session --region us-west-2 --target i-0280cf604085f4a44\n</code></pre>"},{"location":"patterns/privatelink-access/#update-kubeconfig","title":"Update Kubeconfig","text":"<p>On the Client EC2 machine, run the following command to update the local <code>~/.kube/config</code> file to enable access to the cluster:</p> <pre><code>aws eks update-kubeconfig --region us-west-2 --name privatelink-access\n</code></pre>"},{"location":"patterns/privatelink-access/#test-complete-access-with-kubectl","title":"Test complete access with <code>kubectl</code>","text":"<p>Test access by listing the pods running on the cluster:</p> <pre><code>kubectl get pods -A\n</code></pre> <pre><code>NAMESPACE     NAME                       READY   STATUS    RESTARTS   AGE\nkube-system   aws-node-4f8g8             1/1     Running   0          1m\nkube-system   coredns-6ff9c46cd8-59sqp   1/1     Running   0          1m\nkube-system   coredns-6ff9c46cd8-1npb   1/1     Running   0          2m\nkube-system   kube-proxy-mm2zc           1/1     Running   0          1m\n</code></pre>"},{"location":"patterns/privatelink-access/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/sso-iam-identity-center/","title":"SSO - IAM Identity Center","text":""},{"location":"patterns/sso-iam-identity-center/#iam-identity-center-single-sign-on-for-amazon-eks-cluster-with-cluster-access-manager","title":"IAM Identity Center Single Sign-On for Amazon EKS Cluster with Cluster Access Manager","text":"<p>This example demonstrates how to deploy an Amazon EKS cluster that is deployed on the AWS Cloud, integrated with IAM Identity Center (former AWS SSO) as an the Identity Provider (IdP) for Single Sign-On (SSO) authentication having Cluster Access Manager as the entry point API. The configuration to provide authorization for users is simplified using a combination of Access Entries and Kubernetes Role-based access control (RBAC).</p>"},{"location":"patterns/sso-iam-identity-center/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p> <p>This patterns uses aws Identity datasource <code>data \"aws_ssoadmin_instances\" \"this\" {}</code> to retrieve your identity center configuration. You can check that the datasource can works by using the aws cli:</p> <pre><code>aws identitystore list-instances\n</code></pre> <p>You may need to specify the appropriate AWS region in either the previous command and terraform (in main.tf) for the pattern to succeed. If the previous command is empty, you may want to properly configure AWS identity center within your account.</p>"},{"location":"patterns/sso-iam-identity-center/#validate","title":"Validate","text":"<p>After the <code>terraform</code> commands are executed successfully, check if the newly created users are active.</p> <p>To do that use the link provided in the email invite - if you added a valid email address for your users either in your Terraform code or IAM Identity Center Console - or go to the IAM Identity Center Console, in the Users dashboard on the left hand side menu, then select the user, and click on Reset password button on the upper right corner. Choose the option to Generate a one-time password and share the password with the user.</p> <p>With the active users, use one of the <code>terraform output</code> examples to configure your AWS credentials for SSO, as shown in the examples below. After you choose the SSO registration scopes, your browser windows will appear and request to login using your IAM Identity Center username and password.</p>"},{"location":"patterns/sso-iam-identity-center/#admin-user-example","title":"Admin user example","text":"<pre><code>configure_sso_admin = &lt;&lt;EOT\n  # aws configure sso --profile EKSClusterAdmin\n      SSO session name (Recommended): &lt;SESSION_NAME&gt;\n      SSO start URL [None]: https://d-1234567890.awsapps.com/start\n      SSO region [None]: us-west-2\n      SSO registration scopes [sso:account:access]:\n      Attempting to automatically open the SSO authorization page in your default browser.\n      If the browser does not open or you wish to use a different device to authorize this request, open the following URL:\n\n      https://device.sso.us-west-2.amazonaws.com/\n\n      Then enter the code:\n\n      The only AWS account available to you is: 123456789012\n      Using the account ID 123456789012\n      The only role available to you is: EKSClusterAdmin\n      Using the role name EKSClusterAdmin\n      CLI default client Region [us-west-2]: us-west-2\n      CLI default output format [json]: json\n      CLI profile name [EKSClusterAdmin-123456789012]:\n\n      To use this profile, specify the profile name using --profile, as shown:\n\n  # aws eks --region us-west-2 update-kubeconfig --name iam-identity-center --profile EKSClusterAdmin\n\nEOT\n</code></pre>"},{"location":"patterns/sso-iam-identity-center/#read-only-user-example","title":"Read-only user example","text":"<pre><code>configure_sso_user = &lt;&lt;EOT\n  # aws configure sso --profile EKSClusterUser\n      SSO session name (Recommended): &lt;SESSION_NAME&gt;\n      SSO start URL [None]: https://d-1234567890.awsapps.com/start\n      SSO region [None]: us-west-2\n      SSO registration scopes [sso:account:access]:\n      Attempting to automatically open the SSO authorization page in your default browser.\n      If the browser does not open or you wish to use a different device to authorize this request, open the following URL:\n\n      https://device.sso.us-west-2.amazonaws.com/\n\n      Then enter the code:\n\n      The only AWS account available to you is: 123456789012\n      Using the account ID 123456789012\n      The only role available to you is: EKSClusterUser\n      Using the role name EKSClusterUser\n      CLI default client Region [us-west-2]: us-west-2\n      CLI default output format [json]: json\n      CLI profile name [EKSClusterUser-123456789012]:\n\n      To use this profile, specify the profile name using --profile, as shown:\n\n  # aws eks --region us-west-2 update-kubeconfig --name iam-identity-center --profile EKSClusterUser\n\nEOT\n</code></pre> <p>With the <code>kubeconfig</code> configured, you'll be able to run <code>kubectl</code> commands in your Amazon EKS Cluster with the impersonated user. The read-only user has a <code>cluster-viewer</code> Kubernetes role bound to it's group, whereas the admin user, has the <code>admin</code> Kubernetes role bound to it's group.</p> <pre><code>kubectl get pods -A\nNAMESPACE          NAME                        READY   STATUS    RESTARTS   AGE\namazon-guardduty   aws-guardduty-agent-bl2v2   1/1     Running   0          3h54m\namazon-guardduty   aws-guardduty-agent-s2vcx   1/1     Running   0          3h54m\namazon-guardduty   aws-guardduty-agent-w8gfc   1/1     Running   0          3h54m\nkube-system        aws-node-m9hmd              1/1     Running   0          3h53m\nkube-system        aws-node-w42b8              1/1     Running   0          3h53m\nkube-system        aws-node-wm6rm              1/1     Running   0          3h53m\nkube-system        coredns-6ff9c46cd8-94jlr    1/1     Running   0          3h59m\nkube-system        coredns-6ff9c46cd8-n2mrb    1/1     Running   0          3h59m\nkube-system        kube-proxy-7fb86            1/1     Running   0          3h54m\nkube-system        kube-proxy-p4f5g            1/1     Running   0          3h54m\nkube-system        kube-proxy-q1fmc            1/1     Running   0          3h54m\n</code></pre> <p>If not revoked after the cluster creation, it's possible to use the <code>configure_kubectl</code> output to assume the Cluster creator role with <code>cluster-admin</code> access.</p> <pre><code>configure_kubectl = \"aws eks --region us-west-2 update-kubeconfig --name iam-identity-center\"\n</code></pre>"},{"location":"patterns/sso-iam-identity-center/#destroy","title":"Destroy","text":"<p>If you revoked the Cluster creator <code>cluster-admin</code> permission, you may need to re-associate the <code>AmazonEKSClusterAdminPolicy</code> access entry to run <code>terraform destroy</code>.</p> <pre><code>terraform destroy -target module.developers_team -auto-approve\nterraform destroy -target module.eks -auto-approve\nterraform destroy -auto-approve\n</code></pre>"},{"location":"patterns/sso-okta/","title":"SSO - Okta","text":""},{"location":"patterns/sso-okta/#okta-single-sign-on-for-amazon-eks-cluster","title":"Okta Single Sign-On for Amazon EKS Cluster","text":"<p>This example demonstrates how to deploy an Amazon EKS cluster that is deployed on the AWS Cloud, integrated with Okta as an the Identity Provider (IdP) for Single Sign-On (SSO) authentication. The configuration for authorization is done using Kubernetes Role-based access control (RBAC).</p>"},{"location":"patterns/sso-okta/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/sso-okta/#validate","title":"Validate","text":"<p>After the <code>terraform</code> commands are executed successfully, check if the newly created users are active.</p> <p>To do that use the link provided in the email invite if you added a valid email address for your users, or go to the Okta Admin Dashboard, select the user, and click on Set Password and Activate button.</p> <p>With the active users, use the <code>terraform output</code> example to setup your <code>kubeconfig</code> profile to authenticate through Okta.</p> <pre><code>configure_kubeconfig = &lt;&lt;EOT\n    kubectl config set-credentials oidc \\\n      --exec-api-version=client.authentication.k8s.io/v1beta1 \\\n      --exec-command=kubectl \\\n      --exec-arg=oidc-login \\\n      --exec-arg=get-token \\\n      --exec-arg=--oidc-issuer-url=https://dev-ORGID.okta.com/oauth2/1234567890abcdefghij \\\n      --exec-arg=--oidc-client-id=1234567890abcdefghij\n      --exec-arg=--oidc-extra-scope=\"email offline_access profile openid\"\n</code></pre> <p>With the <code>kubeconfig</code> configured, you'll be able to run <code>kubectl</code> commands in your Amazon EKS Cluster using the <code>--user</code> cli option to impersonate the Okta authenticated user. When <code>kubectl</code> command is issued with the <code>--user</code> option for the first time, your browser window will open and require you to authenticate.</p> <p>The read-only user has a <code>cluster-viewer</code> Kubernetes role bound to it's group, whereas the admin user, has the <code>admin</code> Kubernetes role bound to it's group.</p> <pre><code>kubectl get pods -A\nNAMESPACE          NAME                        READY   STATUS    RESTARTS   AGE\namazon-guardduty   aws-guardduty-agent-bl2v2   1/1     Running   0          3h54m\namazon-guardduty   aws-guardduty-agent-s1vcx   1/1     Running   0          3h54m\namazon-guardduty   aws-guardduty-agent-w8gfc   1/1     Running   0          3h54m\nkube-system        aws-node-m9hmd              1/1     Running   0          3h53m\nkube-system        aws-node-w42b8              1/1     Running   0          3h53m\nkube-system        aws-node-wm6rm              1/1     Running   0          3h53m\nkube-system        coredns-6ff9c46cd8-94jlr    1/1     Running   0          3h59m\nkube-system        coredns-6ff9c46cd8-nw2rb    1/1     Running   0          3h59m\nkube-system        kube-proxy-7fb86            1/1     Running   0          3h54m\nkube-system        kube-proxy-p4f5g            1/1     Running   0          3h54m\nkube-system        kube-proxy-qk2mc            1/1     Running   0          3h54m\n</code></pre> <p>You can also use the <code>configure_kubectl</code> output to assume the Cluster creator role with <code>cluster-admin</code> access.</p> <pre><code>configure_kubectl = \"aws eks --region us-west-2 update-kubeconfig --name okta\"\n</code></pre> <p>It's also possible to pre-configure your <code>kubeconfig</code> using the <code>okta_login</code> output. This will also require you to authenticate in a browser window.</p> <pre><code>okta_login = \"kubectl oidc-login setup --oidc-issuer-url=https://dev-ORGID.okta.com/oauth2/1234567890abcdefghij--oidc-client-id=1234567890abcdefghij\"\n</code></pre>"},{"location":"patterns/sso-okta/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/stateful/","title":"Stateful","text":""},{"location":"patterns/stateful/#amazon-eks-cluster-for-stateful-workloads","title":"Amazon EKS Cluster for Stateful Workloads","text":""},{"location":"patterns/stateful/#features","title":"Features","text":"<p>Please note: not all of the features listed below are required for stateful workloads on EKS. We are simply grouping together a set of features that are commonly encountered when managing stateful workloads. Users are encouraged to only enable the features that are required for their workload(s) and use case(s).</p>"},{"location":"patterns/stateful/#velero","title":"velero","text":"<p>(From the project documentation) <code>velero</code> (formerly Heptio Ark) gives you tools to back up and restore your Kubernetes cluster resources and persistent volumes. You can run Velero with a public cloud platform or on-premises. Velero lets you:</p> <ul> <li>Take backups of your cluster and restore in case of loss.</li> <li>Migrate cluster resources to other clusters.</li> <li>Replicate your production cluster to development and testing clusters.</li> </ul>"},{"location":"patterns/stateful/#ebs-efs-csi-drivers","title":"EBS &amp; EFS CSI Drivers","text":"<ul> <li>A second storage class for <code>gp3</code> backed volumes has been added and made the default over the EKS default <code>gp2</code> storage class (<code>gp2</code> storage class remains in the cluster for use, but it is no longer the default storage class)</li> <li>A standard implementation of the EFS CSI driver</li> </ul>"},{"location":"patterns/stateful/#eks-managed-nodegroup-w-multiple-volumes","title":"EKS Managed Nodegroup w/ Multiple Volumes","text":"<p>An EKS managed nodegroup that utilizes multiple EBS volumes. The primary use case demonstrated in this example is a second volume that is dedicated to the <code>containerd</code> runtime to ensure the root volume is not filled up nor has its I/O exhausted to ensure the instance does not reach a degraded state. The <code>containerd</code> directories are mapped to this volume. You can read more about this recommendation in our EKS best practices guide and refer to the <code>containerd</code> documentation for more information. The update for <code>containerd</code> to use the second volume is managed through the provided user data.</p> <p>In addition, the following properties are configured on the nodegroup volumes:</p> <ul> <li>EBS encryption using a customer managed key (CMK)</li> <li>Configuring the volumes to use GP3 storage</li> </ul>"},{"location":"patterns/stateful/#eks-managed-nodegroup-w-instance-store-volumes","title":"EKS Managed Nodegroup w/ Instance Store Volume(s)","text":"<p>An EKS managed nodegroup that utilizes EC2 instances with ephemeral instance store(s). Instance stores are ideal for temporary storage of information that changes frequently, such as buffers, caches, scratch data, and other temporary content, or for data that is replicated across a fleet of instances. You can read more about instance stores in the AWS documentation; and be sure to check out the <code>Block device mapping instance store caveats</code> section as well which covers why the example has provided user data for mounting the instance store(s). The size and number of instance stores will vary based on the EC2 instance type and class.</p> <p>In addition, the following properties are configured on the nodegroup volumes:</p> <ul> <li>EBS encryption using a customer managed key (CMK)</li> <li>Configuring the volumes to use GP3 storage</li> </ul>"},{"location":"patterns/stateful/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/stateful/#validate","title":"Validate","text":"<p>The following command will update the <code>kubeconfig</code> on your local machine and allow you to interact with your EKS Cluster using <code>kubectl</code> to validate the Velero deployment.</p> <ol> <li> <p>Run <code>update-kubeconfig</code> command:</p> <pre><code>aws eks --region &lt;REGION&gt; update-kubeconfig --name &lt;CLUSTER_NAME&gt;\n</code></pre> </li> <li> <p>List the storage classes to view that <code>efs</code>, <code>gp2</code>, and <code>gp3</code> classes are present and <code>gp3</code> is the default storage class</p> <pre><code>kubectl get storageclasses\n</code></pre> <pre><code>NAME            PROVISIONER             RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE\nefs             efs.csi.aws.com         Delete          Immediate              true                   2m19s\ngp2             kubernetes.io/aws-ebs   Delete          WaitForFirstConsumer   false                  15m\ngp3 (default)   ebs.csi.aws.com         Delete          WaitForFirstConsumer   true                   2m19s\n</code></pre> </li> <li> <p>From an instance launched with instance store(s), check that the instance store has been mounted correctly. To verify, first install the <code>nvme-cli</code> tool and then use it to verify. To verify, you can access the instance using SSM Session Manager:</p> <pre><code># Install the nvme-cli tool\nsudo yum install nvme-cli -y\n\n# Show NVMe volumes attached\nsudo nvme list\n</code></pre> <pre><code># Notice the model is `EC2 NVMe Instance Storage` for the instance store\nNode             SN                   Model                                    Namespace Usage                      Format           FW Rev\n---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------\n/dev/nvme0n1     vol0546d3c3b0af0bf6d Amazon Elastic Block Store               1          25.77  GB /  25.77  GB    512   B +  0 B   1.0\n/dev/nvme1n1     AWS24BBF51AF55097008 Amazon EC2 NVMe Instance Storage         1          75.00  GB /  75.00  GB    512   B +  0 B   0\n\n# Show disks, their partitions and mounts\nsudo lsblk\n\n# Output should look like below\nNAME          MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT\nnvme0n1       259:0    0   24G  0 disk\n\u251c\u2500nvme0n1p1   259:2    0   24G  0 part /\n\u2514\u2500nvme0n1p128 259:3    0    1M  0 part\nnvme1n1       259:1    0 69.9G  0 disk /local1 # &lt;--- this is the instance store\n</code></pre> </li> <li> <p>From an instance launched with multiple volume(s), check that the instance store has been mounted correctly. To verify, first install the <code>nvme-cli</code> tool and then use it to verify. To verify, you can access the instance using SSM Session Manager:</p> <pre><code># Install the nvme-cli tool\nsudo yum install nvme-cli -y\n\n# Show NVMe volumes attached\nsudo nvme list\n</code></pre> <pre><code># /dev/nvme0n1 is the root volume and /dev/nvme1n1 is the second, additional volume\nNode             SN                   Model                                    Namespace Usage                      Format           FW Rev\n---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------\n/dev/nvme0n1     vol0cd37dab9e4a5c184 Amazon Elastic Block Store               1          68.72  GB /  68.72  GB    512   B +  0 B   1.0\n/dev/nvme1n1     vol0ad3629c159ee869c Amazon Elastic Block Store               1          25.77  GB /  25.77  GB    512   B +  0 B   1.0\n</code></pre> </li> <li> <p>From the same instance used in step 4, check that the containerd directories are using the second <code>/dev/nvme1n1</code> volume:</p> <pre><code>df /var/lib/containerd/\n</code></pre> <pre><code># Output should look like below, which shows the directory on the\n# /dev/nvme1n1 volume and NOT on /dev/nvme0n1 (root volume)\nFilesystem     1K-blocks    Used Available Use% Mounted on\n/dev/nvme1n1    24594768 2886716  20433380  13% /var/lib/containerd\n</code></pre> <pre><code>df /run/containerd/\n</code></pre> <pre><code># Output should look like below, which shows the directory on the\n# /dev/nvme1n1 volume and NOT on /dev/nvme0n1 (root volume)\nFilesystem     1K-blocks    Used Available Use% Mounted on\n/dev/nvme1n1    24594768 2886716  20433380  13% /run/containerd\n</code></pre> </li> <li> <p>Test by listing velero resources provisioned:</p> <pre><code>kubectl get all -n velero\n\n# Output should look similar to below\nNAME                         READY   STATUS    RESTARTS   AGE\npod/velero-b4d8fd5c7-5smp6   1/1     Running   0          112s\n\nNAME             TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE\nservice/velero   ClusterIP   172.20.217.203   &lt;none&gt;        8085/TCP   114s\n\nNAME                     READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/velero   1/1     1            1           114s\n\nNAME                               DESIRED   CURRENT   READY   AGE\nreplicaset.apps/velero-b4d8fd5c7   1         1         1       114s\n</code></pre> </li> <li> <p>Get backup location using velero CLI</p> <pre><code>velero backup-location get\n\n# Output should look similar to below\nNAME      PROVIDER   BUCKET/PREFIX             PHASE       LAST VALIDATED                  ACCESS MODE   DEFAULT\ndefault   aws        velero-ss1wm44hv1fzb32d   Available   2022-05-22 10:53:26 -0400 EDT   ReadWrite     true\n</code></pre> </li> </ol>"},{"location":"patterns/stateful/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/targeted-odcr/","title":"Targeted On-Demand Capacity Reservation (ODCR)","text":""},{"location":"patterns/targeted-odcr/#eks-cluster-with-on-demand-capacity-reservations-odcr","title":"EKS Cluster with On-demand Capacity Reservations (ODCR)","text":"<p>This pattern demonstrates how to consume/utilize on-demand capacity reservations (ODCRs) with Amazon EKS. The solution is comprised of primarily 3 components:</p> <ol> <li> <p>The node group that will utilize the ODCRs should have the subnets provided to it restricted to the availability zone where the ODCR(s) capacity is allocated. For example - if the ODCR(s) are allocated to <code>us-west-2b</code>, the node group should only have subnet IDs provided to it that reside in <code>us-west-2b</code>. If the subnets that reside in other AZs are provided, its possible to encounter an error such as <code>InvalidParameterException: The following supplied instance types do not exist ...</code>. It is not guaranteed that this error will always be shown, and may appear random since the underlying autoscaling group(s) will provision nodes into different AZs at random. It will only occur when the underlying autoscaling group tries to provision instances into an AZ where capacity is not allocated and there is insufficient on-demand capacity for the desired instance type.</p> </li> <li> <p>A custom launch template is required in order to specify the <code>capacity_reservation_specification</code> arguments. This is how the ODCRs are integrated into the node group (i.e. - tells the autoscaling group to utilize the provided capacity reservation(s)).</p> <p>Info</p> <p>By default, the <code>terraform-aws-eks</code> module creates and utilizes a custom launch template with EKS managed node groups which means users just need to supply the <code>capacity_reservation_specification</code> in their node group definition.</p> </li> <li> <p>A resource group will need to be created for the capacity reservations. The resource group acts like a container, allowing for ODCRs to be added or removed as needed to adjust the available capacity. Utilizing the resource group allows for this additional capacity to be adjusted without any modification or disruption to the existing node group or launch template. As soon as the ODCR has been associated to the resource group, the node group can scale up to start utilizing that capacity.</p> </li> </ol> <p>Links:</p> <ul> <li>Tutorial: Launch On-Demand Instances using targeted Capacity Reservations</li> <li>Target a group of Amazon EC2 On-Demand Capacity Reservations</li> </ul>"},{"location":"patterns/targeted-odcr/#code","title":"Code","text":"<pre><code>################################################################################\n# Cluster\n################################################################################\n\nmodule \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n  version = \"~&gt; 20.9\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.29\"\n\n  # To facilitate easier interaction for demonstration purposes\n  cluster_endpoint_public_access = true\n\n  # Gives Terraform identity admin access to the cluster\n  enable_cluster_creator_admin_permissions = true\n\n  cluster_addons = {\n    coredns    = {}\n    kube-proxy = {}\n    vpc-cni    = {}\n  }\n\n  vpc_id     = module.vpc.vpc_id\n  subnet_ids = module.vpc.private_subnets\n\n  eks_managed_node_group_defaults = {\n    iam_role_additional_policies = {\n      # Not required, but used in the example to access the nodes to inspect drivers and devices\n      AmazonSSMManagedInstanceCore = \"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore\"\n    }\n  }\n\n  eks_managed_node_groups = {\n    odcr = {\n      instance_types = [\"t3.micro\", \"t3.small\"]\n\n      min_size     = 4\n      max_size     = 5\n      desired_size = 2\n\n      # First subnet is in the \"${local.region}a\" availability zone\n      # where the capacity reservation is created\n      subnet_ids = [element(module.vpc.private_subnets, 0)]\n      capacity_reservation_specification = {\n        capacity_reservation_target = {\n          capacity_reservation_resource_group_arn = aws_resourcegroups_group.odcr.arn\n        }\n      }\n    }\n  }\n\n  tags = local.tags\n}\n\n################################################################################\n# Resource Group\n################################################################################\n\nresource \"aws_resourcegroups_group\" \"odcr\" {\n  name        = \"${local.name}-p5-odcr\"\n  description = \"P5 instance on-demand capacity reservations\"\n\n  configuration {\n    type = \"AWS::EC2::CapacityReservationPool\"\n  }\n\n  configuration {\n    type = \"AWS::ResourceGroups::Generic\"\n\n    parameters {\n      name   = \"allowed-resource-types\"\n      values = [\"AWS::EC2::CapacityReservation\"]\n    }\n  }\n}\n\nresource \"aws_resourcegroups_resource\" \"odcr_1\" {\n  group_arn = aws_resourcegroups_group.odcr.arn\n  # Replace the following with the ARN of the capacity reservation\n  # provided by AWS when supplied with a capacity reservation\n  resource_arn = aws_ec2_capacity_reservation.micro.arn\n}\n\nresource \"aws_resourcegroups_resource\" \"odcr_2\" {\n  group_arn = aws_resourcegroups_group.odcr.arn\n  # Replace the following with the ARN of the capacity reservation\n  # provided by AWS when supplied with a capacity reservation\n  resource_arn = aws_ec2_capacity_reservation.small.arn\n}\n\n################################################################################\n# Capacity Reservation\n# These are created for the example, but are not necessary when\n# AWS EC2 provides you with a capacity reservation ID\n################################################################################\n\nresource \"aws_ec2_capacity_reservation\" \"micro\" {\n  instance_type           = \"t3.micro\"\n  instance_platform       = \"Linux/UNIX\"\n  availability_zone       = \"${local.region}a\"\n  instance_count          = 2\n  instance_match_criteria = \"targeted\"\n\n  # Just for example - 30 minutes from time of creation\n  end_date      = timeadd(timestamp(), \"30m\")\n  end_date_type = \"limited\"\n}\n\nresource \"aws_ec2_capacity_reservation\" \"small\" {\n  instance_type           = \"t3.small\"\n  instance_platform       = \"Linux/UNIX\"\n  availability_zone       = \"${local.region}a\"\n  instance_count          = 2\n  instance_match_criteria = \"targeted\"\n\n  # Just for example - 30 minutes from time of creation\n  end_date      = timeadd(timestamp(), \"30m\")\n  end_date_type = \"limited\"\n}\n\n################################################################################\n# Kubectl Output\n################################################################################\n\noutput \"configure_kubectl\" {\n  description = \"Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig\"\n  value       = \"aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}\"\n}\n</code></pre>"},{"location":"patterns/targeted-odcr/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/targeted-odcr/#validate","title":"Validate","text":"<ol> <li> <p>Navigate to the EC2 console page - on the left hand side, click on <code>Capacity Reservations</code>under the <code>Instances</code> section. You should see the capacity reservation(s) that have been created similar to the screenshot below. For this example, you can see that <code>Available capacity</code> column is empty, which means that the capacity reservations have been fully utilized by the example (as expected).</p> <p> </p> </li> <li> <p>Click on one of the capacity reservation IDs to view the details of the capacity reservation. You should see the details of the capacity reservation similar to the screenshot below. For this example, you can see that <code>Available capacity</code> is <code>0 instances</code>, which means that the capacity reservation has been fully utilized by the example (as expected).</p> <p> </p> </li> </ol>"},{"location":"patterns/targeted-odcr/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/tls-with-aws-pca-issuer/","title":"TLS w/ AWS PCA Issuer","text":""},{"location":"patterns/tls-with-aws-pca-issuer/#tls-with-aws-pca-issuer","title":"TLS with AWS PCA Issuer","text":"<p>This pattern demonstrates how to enable TLS with AWS PCA issuer on an Amazon EKS cluster.</p>"},{"location":"patterns/tls-with-aws-pca-issuer/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/tls-with-aws-pca-issuer/#validate","title":"Validate","text":"<ol> <li> <p>List all the pods running in <code>aws-privateca-issuer</code> and <code>cert-manager</code> Namespace.</p> <pre><code>kubectl get pods -n aws-privateca-issuer\nkubectl get pods -n cert-manager\n</code></pre> </li> <li> <p>View the <code>certificate</code> status in the <code>default</code> Namespace. It should be in <code>Ready</code> state, and be pointing to a <code>secret</code> created in the same Namespace.</p> <pre><code>kubectl get certificate -o wide\n</code></pre> <pre><code>NAME      READY   SECRET                  ISSUER                    STATUS                                          AGE\nexample   True    example-clusterissuer   tls-with-aws-pca-issuer   Certificate is up to date and has not expired   41m\n</code></pre> <pre><code>kubectl get secret example-clusterissuer\n</code></pre> <pre><code>NAME                    TYPE                DATA   AGE\nexample-clusterissuer   kubernetes.io/tls   3      43m\n</code></pre> </li> </ol>"},{"location":"patterns/tls-with-aws-pca-issuer/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/vpc-lattice/","title":"Amazon VPC Lattice","text":""},{"location":"patterns/vpc-lattice/#amazon-vpc-lattice","title":"Amazon VPC Lattice","text":"<p>This folder contains use case-driven patterns covering different aspects of the Amazon VPC Lattice service.</p>"},{"location":"patterns/vpc-lattice/#use-cases","title":"Use cases","text":"<ul> <li> <p>Simple Client to Server Communication</p> <p>This pattern describes how to expose a simple API within an Amazon EKS cluster deployed in VPC A to a client application hosted in VPC B through Amazon VPC Lattice.</p> </li> </ul>"},{"location":"patterns/vpc-lattice/#supporting-resources","title":"Supporting resources","text":"<ul> <li>Documentation</li> <li>AWS Gateway API Controller</li> </ul>"},{"location":"patterns/wireguard-with-cilium/","title":"Wireguard /w Cilium","text":""},{"location":"patterns/wireguard-with-cilium/#transparent-encryption-with-cilium-and-wireguard","title":"Transparent Encryption with Cilium and Wireguard","text":"<p>This pattern demonstrates Cilium configured in CNI chaining mode with the VPC CNI and with Wireguard transparent encryption enabled on an Amazon EKS cluster.</p> <ul> <li>Cilium CNI Chaining Documentation</li> <li>Cilium Wireguard Encryption Documentation</li> </ul>"},{"location":"patterns/wireguard-with-cilium/#focal-points","title":"Focal Points","text":"<ul> <li><code>eks.tf</code> contains the cluster configuration and the deployment of Cilium.<ul> <li>There are no specific requirements from an EKS perspective, other than the Linux Kernel version used by the OS must be 5.10+</li> </ul> </li> <li><code>example.yaml</code> provides a sample application used to demonstrate the encrypted connectivity. This is optional and not required for the pattern.</li> </ul>"},{"location":"patterns/wireguard-with-cilium/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/wireguard-with-cilium/#validate","title":"Validate","text":"<ol> <li> <p>Deploy the example pods:</p> <pre><code>kubectl apply -f example.yaml\n</code></pre> <pre><code>pod/server created\nservice/server created\npod/client created\n</code></pre> </li> <li> <p>Get the Cilium status from one of the Cilium pods.</p> <p>Under the <code>Encryption</code> field, it should state <code>Wireguard</code> with a PubKey. <code>NodeEncryption: Disabled</code> is expected since <code>NodeEncryption</code> was not enabled via the Helm values provided.</p> <pre><code>kubectl -n kube-system exec -ti ds/cilium -- cilium status\n</code></pre> <pre><code>Defaulted container \"cilium-agent\" out of: cilium-agent, config (init), mount-cgroup (init), apply-sysctl-overwrites (init), mount-bpf-fs (init), clean-cilium-state (init), install-cni-binaries (init)\nKVStore:                 Ok   Disabled\nKubernetes:              Ok   1.28+ (v1.28.1-eks-43840fb) [linux/amd64]\nKubernetes APIs:         [\"EndpointSliceOrEndpoint\", \"cilium/v2::CiliumClusterwideNetworkPolicy\", \"cilium/v2::CiliumEndpoint\", \"cilium/v2::CiliumNetworkPolicy\", \"cilium/v2::CiliumNode\", \"cilium/v2alpha1::CiliumCIDRGroup\", \"core/v1::Namespace\", \"core/v1::Pods\", \"core/v1::Service\", \"networking.k8s.io/v1::NetworkPolicy\"]\nKubeProxyReplacement:    False   [eth0 10.0.21.109 (Direct Routing), eth1 10.0.27.0]\nHost firewall:           Disabled\nCNI Chaining:            aws-cni\nCilium:                  Ok   1.14.2 (v1.14.2-a6748946)\nNodeMonitor:             Listening for events on 2 CPUs with 64x4096 of shared memory\nCilium health daemon:    Ok\nIPAM:                    IPv4: 1/254 allocated from 10.0.0.0/24,\nIPv4 BIG TCP:            Disabled\nIPv6 BIG TCP:            Disabled\nBandwidthManager:        Disabled\nHost Routing:            Legacy\nMasquerading:            Disabled\nController Status:       24/24 healthy\nProxy Status:            No managed proxy redirect\nGlobal Identity Range:   min 256, max 65535\nHubble:                  Ok          Current/Max Flows: 410/4095 (10.01%), Flows/s: 1.59   Metrics: Disabled\nEncryption:              Wireguard   [NodeEncryption: Disabled, cilium_wg0 (Pubkey: /yuqsZyG91AzVIkZ3AIq8qjQ0gGKQd6GWcRYh4LYpko=, Port: 51871, Peers: 1)]\nCluster health:                      Probe disabled\n</code></pre> </li> <li> <p>Open a shell inside the cilium container</p> <pre><code>kubectl -n kube-system exec -ti ds/cilium -- bash\n</code></pre> </li> <li> <p>Install <code>tcpdump</code></p> <pre><code>apt-get update\napt-get install -y tcpdump\n</code></pre> </li> <li> <p>Start a packet capture on <code>cilium_wg0</code> and verify you see payload in clear text, it means the traffic is encrypted with wireguard</p> <pre><code>tcpdump -A -c 40 -i cilium_wg0 | grep \"Welcome to nginx!\"\n</code></pre> <pre><code>tcpdump: verbose output suppressed, use -v[v]... for full protocol decode\nlistening on cilium_wg0, link-type RAW (Raw IP), snapshot length 262144 bytes\n&lt;title&gt;Welcome to nginx!&lt;/title&gt;\n&lt;h1&gt;Welcome to nginx!&lt;/h1&gt;\n40 packets captured\n40 packets received by filter\n0 packets dropped by kernel\n</code></pre> <p>Exit</p> <p>Exit the container shell by typing <code>exit</code> before continuing to next step</p> </li> <li> <p>Deploy the Cilium connectivity resources to check and evaluate connectivity:</p> <pre><code>kubectl create ns cilium-test\nkubectl apply -n cilium-test -f https://raw.githubusercontent.com/cilium/cilium/v1.14.1/examples/kubernetes/connectivity-check/connectivity-check.yaml\n</code></pre> <pre><code>deployment.apps/echo-a created\ndeployment.apps/echo-b created\ndeployment.apps/echo-b-host created\ndeployment.apps/pod-to-a created\ndeployment.apps/pod-to-external-1111 created\ndeployment.apps/pod-to-a-denied-cnp created\ndeployment.apps/pod-to-a-allowed-cnp created\ndeployment.apps/pod-to-external-fqdn-allow-google-cnp created\ndeployment.apps/pod-to-b-multi-node-clusterip created\ndeployment.apps/pod-to-b-multi-node-headless created\ndeployment.apps/host-to-b-multi-node-clusterip created\ndeployment.apps/host-to-b-multi-node-headless created\ndeployment.apps/pod-to-b-multi-node-nodeport created\ndeployment.apps/pod-to-b-intra-node-nodeport created\nservice/echo-a created\nservice/echo-b created\nservice/echo-b-headless created\nservice/echo-b-host-headless created\nciliumnetworkpolicy.cilium.io/pod-to-a-denied-cnp created\nciliumnetworkpolicy.cilium.io/pod-to-a-allowed-cnp created\nciliumnetworkpolicy.cilium.io/pod-to-external-fqdn-allow-google-cnp created\n</code></pre> </li> <li> <p>View the logs of any of the connectivity tests to view the results:</p> <pre><code>kubectl logs &lt;cilium test pod&gt; -n cilium-test\n</code></pre> <pre><code>\\{^_^}/ hi!\n\nLoading /default.json\nDone\n\nResources\nhttp://:8080/private\nhttp://:8080/public\n\nHome\nhttp://:8080\n\nType s + enter at any time to create a snapshot of the database\nWatching...\n\nGET /public 200 7.063 ms - 57\nGET /public 200 3.126 ms - 57\nGET /public 200 3.039 ms - 57\nGET /public 200 2.776 ms - 57\nGET /public 200 3.087 ms - 57\nGET /public 200 2.781 ms - 57\nGET /public 200 2.977 ms - 57\nGET /public 200 2.596 ms - 57\nGET /public 200 2.991 ms - 57\nGET /public 200 2.708 ms - 57\nGET /public 200 3.066 ms - 57\nGET /public 200 2.616 ms - 57\nGET /public 200 2.875 ms - 57\nGET /public 200 2.689 ms - 57\nGET /public 200 2.800 ms - 57\nGET /public 200 2.556 ms - 57\nGET /public 200 3.238 ms - 57\nGET /public 200 2.538 ms - 57\nGET /public 200 2.890 ms - 57\nGET /public 200 2.666 ms - 57\nGET /public 200 2.729 ms - 57\nGET /public 200 2.580 ms - 57\nGET /public 200 2.919 ms - 57\nGET /public 200 2.630 ms - 57\nGET /public 200 2.857 ms - 57\nGET /public 200 2.716 ms - 57\nGET /public 200 1.693 ms - 57\nGET /public 200 2.715 ms - 57\nGET /public 200 2.729 ms - 57\nGET /public 200 2.655 ms - 57\n</code></pre> </li> </ol>"},{"location":"patterns/wireguard-with-cilium/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"patterns/vpc-lattice/client-server-communication/","title":"Amazon VPC Lattice Client-server Communication","text":""},{"location":"patterns/vpc-lattice/client-server-communication/#amazon-vpc-lattice-simple-client-to-server-communication","title":"Amazon VPC Lattice - Simple Client to Server Communication","text":"<p>This pattern demonstrates how to expose an EKS cluster hosted application to an internal consumer through Amazon VPC Lattice.</p>"},{"location":"patterns/vpc-lattice/client-server-communication/#scenario","title":"Scenario","text":"<p>With this solution we showcase how to configure Amazon VPC Lattice using the AWS Gateway API Controller in order to manage Amazon VPC Lattice resources through native K8S Gateway API objects. This pattern deploys two distinct VPCs with a client application running in one of them and a server application in the other. The server application is deployed inside an EKS cluster and made exposed to the client application through Amazon VPC Lattice which establishes connectivity between the two applications. Further we demonstrate how to configure a custom domain name for the exposed service using Amazon Route53 and the external-dns project.</p> <p></p>"},{"location":"patterns/vpc-lattice/client-server-communication/#deploy","title":"Deploy","text":"<p>See here for the prerequisites and steps to deploy this pattern.</p>"},{"location":"patterns/vpc-lattice/client-server-communication/#validate","title":"Validate","text":"<p>In order to test the connectivity between the client and server, please follow the steps outlined below:</p> <ol> <li>Login to the management console of your AWS account and navigate to the EC2 service</li> <li>Select your the EC2 Instance with the name client, click Connect, choose Session Manager and click Connect</li> <li> <p>Within the console test the connectivity to the server application by entering the following command:</p> <pre><code>$ curl -i http://server.example.com\nHTTP/1.1 200 OK\ndate: Thu, 14 Dec 2023 08:29:39 GMT\ncontent-length: 54\ncontent-type: text/plain; charset=utf-8\n\nRequesting to Pod(server-6f487b9bcd-5qm4v): server pod\n</code></pre> </li> </ol>"},{"location":"patterns/vpc-lattice/client-server-communication/#destroy","title":"Destroy","text":"<pre><code>terraform destroy -target=\"module.eks_blueprints_addons\" -auto-approve\nterraform destroy -target=\"module.eks\" -auto-approve\nterraform destroy -auto-approve\n</code></pre> <p>See here for more details on cleaning up the resources created.</p>"},{"location":"snippets/ipv4-prefix-delegation/","title":"IPv4 Prefix Delegation","text":"<p>The configuration snippet below shows how to enable prefix delegation to increase the number of available IP addresses on the provisioned EC2 nodes.</p> <ul> <li>Documentation</li> <li>Blog post</li> </ul>"},{"location":"snippets/ipv4-prefix-delegation/#vpc-cni-configuration","title":"VPC CNI Configuration","text":"<p>In this example, the <code>vpc-cni</code> addon is configured using <code>before_compute = true</code>. This is done to ensure the <code>vpc-cni</code> is created and updated before any EC2 instances are created so that the desired settings have applied before they will be referenced. With this configuration, you will now see that nodes created will have <code>--max-pods 110</code> configured do to the use of prefix delegation being enabled on the <code>vpc-cni</code>.</p> <p>If you find that your nodes are not being created with the correct number of max pods (i.e. - for <code>m5.large</code>, if you are seeing a max pods of 29 instead of 110), most likely the <code>vpc-cni</code> was not configured before the EC2 instances.</p> <pre><code>module \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n\n  # Truncated for brevity\n  ...\n\n  cluster_addons = {\n    vpc-cni = {\n      before_compute = true\n      most_recent    = true # To ensure access to the latest settings provided\n      configuration_values = jsonencode({\n        env = {\n          ENABLE_PREFIX_DELEGATION = \"true\"\n          WARM_PREFIX_TARGET       = \"1\"\n        }\n      })\n    }\n  }\n\n  ...\n}\n</code></pre> <p>When enabled, inspect one of the <code>aws-node-*</code> (AWS VPC CNI) pods to ensure prefix delegation is enabled and warm prefix target is 1:</p> <pre><code>kubectl describe ds -n kube-system aws-node | grep ENABLE_PREFIX_DELEGATION: -A 3\n</code></pre> <p>Output should look similar to below (truncated for brevity):</p> <pre><code>    ENABLE_PREFIX_DELEGATION:               true # &lt;- this should be set to true\n    WARM_ENI_TARGET:                        1\n    WARM_PREFIX_TARGET:                     1 # &lt;- this should be set to 1\n    ...\n</code></pre>"},{"location":"snippets/vpc-cni-custom-networking/","title":"VPC CNI Custom Networking","text":"<p>Custom networking addresses the IP exhaustion issue by assigning the node and Pod IPs from secondary VPC address spaces (CIDR). Custom networking support supports ENIConfig custom resource. The ENIConfig includes an alternate subnet CIDR range (carved from a secondary VPC CIDR), along with the security group(s) that the Pods will belong to. When custom networking is enabled, the VPC CNI creates secondary ENIs in the subnet defined under ENIConfig. The CNI assigns Pods an IP addresses from a CIDR range defined in a ENIConfig CRD.</p> <p>Since the primary ENI is not used by custom networking, the maximum number of Pods you can run on a node is lower. The host network Pods continue to use IP address assigned to the primary ENI. Additionally, the primary ENI is used to handle source network translation and route Pods traffic outside the node.</p> <ul> <li>Documentation</li> <li>Best Practices Guide</li> </ul>"},{"location":"snippets/vpc-cni-custom-networking/#vpc-cni-configuration","title":"VPC CNI Configuration","text":"<p>In this example, the <code>vpc-cni</code> addon is configured using <code>before_compute = true</code>. This is done to ensure the <code>vpc-cni</code> is created and updated before any EC2 instances are created so that the desired settings have applied before they will be referenced. With this configuration, you will now see that nodes created will have <code>--max-pods 110</code> configured do to the use of prefix delegation being enabled on the <code>vpc-cni</code>.</p> <p>If you find that your nodes are not being created with the correct number of max pods (i.e. - for <code>m5.large</code>, if you are seeing a max pods of 29 instead of 110), most likely the <code>vpc-cni</code> was not configured before the EC2 instances.</p>"},{"location":"snippets/vpc-cni-custom-networking/#components","title":"Components","text":"<p>To enable VPC CNI custom networking, you must configuring the following components:</p> <ol> <li> <p>Create a VPC with additional CIDR block associations. These additional CIDR blocks will be used to create subnets for the VPC CNI custom networking:</p> <pre><code>module \"vpc\" {\nsource  = \"terraform-aws-modules/vpc/aws\"\n\n# Truncated for brevity\n...\n\nsecondary_cidr_blocks = [local.secondary_vpc_cidr] # can add up to 5 total CIDR blocks\n\nazs = local.azs\nprivate_subnets = concat(\n   [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)],\n   [for k, v in local.azs : cidrsubnet(local.secondary_vpc_cidr, 2, k)]\n)\n\n...\n}\n</code></pre> </li> <li> <p>Specify the VPC CNI custom networking configuration in the <code>vpc-cni</code> addon configuration:</p> <pre><code>module \"eks\" {\nsource  = \"terraform-aws-modules/eks/aws\"\n\n# Truncated for brevity\n...\n\ncluster_addons = {\n   vpc-cni = {\n      before_compute = true\n      most_recent    = true # To ensure access to the latest settings provided\n      configuration_values = jsonencode({\n      env = {\n         AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG = \"true\"\n         ENI_CONFIG_LABEL_DEF               = \"topology.kubernetes.io/zone\"\n      })\n   }\n}\n\n...\n}\n</code></pre> </li> <li> <p>Create the <code>ENIConfig</code> custom resource for each subnet that you want to deploy pods into:</p> <pre><code>resource \"kubectl_manifest\" \"eni_config\" {\nfor_each = zipmap(local.azs, slice(module.vpc.private_subnets, 3, 6))\n\nyaml_body = yamlencode({\n   apiVersion = \"crd.k8s.amazonaws.com/v1alpha1\"\n   kind       = \"ENIConfig\"\n   metadata = {\n      name = each.key\n   }\n   spec = {\n      securityGroups = [\n      module.eks.node_security_group_id,\n      ]\n      subnet = each.value\n   }\n})\n}\n</code></pre> </li> </ol> <p>Once those settings have been successfully applied, you can verify if custom networking is enabled correctly by inspecting one of the <code>aws-node-*</code> (AWS VPC CNI) pods:</p> <pre><code>kubectl describe pod aws-node-ttg4h -n kube-system\n\n# Output should look similar below (truncated for brevity)\n  Environment:\n    ADDITIONAL_ENI_TAGS:                    {}\n    AWS_VPC_CNI_NODE_PORT_SUPPORT:          true\n    AWS_VPC_ENI_MTU:                        9001\n    AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:     false\n    AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG:     true # &lt;- this should be set to true\n    AWS_VPC_K8S_CNI_EXTERNALSNAT:           false\n    AWS_VPC_K8S_CNI_LOGLEVEL:               DEBUG\n    ...\n</code></pre>"},{"location":"v4-to-v5/addons/","title":"Migrate to EKS Blueprints Addons Module","text":"<p>Please consult the docs/v4-to-v5/example directory for reference configurations. If you find a bug, please open an issue with supporting configuration to reproduce.</p>"},{"location":"v4-to-v5/addons/#this-guide-is-under-active-development","title":"\u26a0\ufe0f This guide is under active development.","text":""},{"location":"v4-to-v5/addons/#list-of-backwards-incompatible-changes","title":"List of backwards incompatible changes","text":"<p>-</p>"},{"location":"v4-to-v5/addons/#additional-changes","title":"Additional changes","text":""},{"location":"v4-to-v5/addons/#added","title":"Added","text":"<p>-</p>"},{"location":"v4-to-v5/addons/#modified","title":"Modified","text":"<p>-</p>"},{"location":"v4-to-v5/addons/#removed","title":"Removed","text":"<p>-</p>"},{"location":"v4-to-v5/addons/#variable-and-output-changes","title":"Variable and output changes","text":"<ol> <li> <p>Removed variables:</p> <p>-</p> </li> <li> <p>Renamed variables:</p> <p>-</p> </li> <li> <p>Added variables:</p> <p>-</p> </li> <li> <p>Removed outputs:</p> <p>-</p> </li> <li> <p>Renamed outputs:</p> <p>-</p> </li> <li> <p>Added outputs:</p> <p>-</p> </li> </ol>"},{"location":"v4-to-v5/addons/#upgrade-migrations","title":"Upgrade Migrations","text":""},{"location":"v4-to-v5/addons/#before-v4x-example","title":"Before - v4.x Example","text":"<pre><code>module \"eks_blueprints_addons\" {\n  source = \"github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1\"\n\n  eks_cluster_id       = module.eks.cluster_name\n  eks_cluster_endpoint = module.eks.cluster_endpoint\n  eks_oidc_provider    = module.eks.oidc_provider\n  eks_cluster_version  = module.eks.cluster_version\n\n  # TODO\n\n}\n</code></pre>"},{"location":"v4-to-v5/addons/#after-v5x-example","title":"After - v5.x Example","text":"<pre><code>module \"eks_blueprints_addons\" {\n  source  = \"aws-ia/eks-blueprints-addons/aws\"\n  version = \"~&gt; 1.0\"\n\n  cluster_name      = module.eks.cluster_name\n  cluster_endpoint  = module.eks.cluster_endpoint\n  cluster_version   = module.eks.cluster_version\n  oidc_provider_arn = module.eks.oidc_provider_arn\n\n  # TODO\n\n}\n</code></pre>"},{"location":"v4-to-v5/addons/#diff-of-before-vs-after","title":"Diff of Before vs After","text":"<pre><code>module \"eks_blueprints_addons\" {\n-  source = \"github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1\"\n+  source  = \"aws-ia/eks-blueprints-addons/aws\"\n+  version = \"~&gt; 1.0\"\n\n  # TODO\n}\n</code></pre>"},{"location":"v4-to-v5/addons/#state-move-commands","title":"State Move Commands","text":"<p>In conjunction with the changes above, users can elect to move their external capacity provider(s) under this module using the following move command. Command is shown using the values from the example shown above, please update to suit your configuration names:</p> <pre><code>terraform state mv 'xxx' 'yyy'\n</code></pre>"},{"location":"v4-to-v5/cluster/","title":"Migrate to EKS Module v19.x","text":"<p>Please consult the docs/v4-to-v5/example directory for reference configurations. If you find a bug, please open an issue with supporting configuration to reproduce.</p>"},{"location":"v4-to-v5/cluster/#backwards-incompatible-changes","title":"Backwards incompatible changes","text":"<ul> <li>The cluster module provided in EKS Blueprints is being removed entirely from the project. Instead, users are encouraged to use the <code>terraform-aws-eks</code> module for creating and managing their EKS cluster in Terraform.</li> <li>The KMS module provided in EKS Blueprints has been removed. Users can leverage the KMS creation/management functionality provided by the <code>terraform-aws-eks</code> module or utilize the standalone <code>terraform-aws-kms</code> module.</li> <li>The EMR on EKS module provided in EKS Blueprints has been removed. Instead, users are encouraged to use the <code>terraform-aws-emr</code> virtual cluster sub-module for creating and managing their EMR on EKS virtual cluster in Terraform.</li> <li>The teams multi-tenancy module provided in EKS Blueprints has been removed. Instead, users are encouraged to use the <code>terraform-aws-eks-blueprints-teams</code> module for creating and managing their multi-tenancy constructions within their EKS clusters in Terraform.</li> </ul>"},{"location":"v4-to-v5/cluster/#additional-changes","title":"Additional changes","text":""},{"location":"v4-to-v5/cluster/#added","title":"Added","text":"<ul> <li>N/A</li> </ul>"},{"location":"v4-to-v5/cluster/#modified","title":"Modified","text":"<ul> <li>N/A</li> </ul>"},{"location":"v4-to-v5/cluster/#removed","title":"Removed","text":"<ul> <li>All noted above under <code>Backwards incompatible changes</code></li> </ul>"},{"location":"v4-to-v5/cluster/#variable-and-output-changes","title":"Variable and output changes","text":"<p>Since the change is to replace the EKS Blueprints cluster module with the <code>terraform-aws-eks</code> module, there aren't technically any variable or output changes other than their removal. Please consult the <code>terraform-aws-eks</code> module for its respective variables/outputs.</p> <ol> <li> <p>Removed variables:</p> <ul> <li>All</li> </ul> </li> <li> <p>Renamed variables:</p> <ul> <li>None</li> </ul> </li> <li> <p>Added variables:</p> <ul> <li>None</li> </ul> </li> <li> <p>Removed outputs:</p> <ul> <li>All</li> </ul> </li> <li> <p>Renamed outputs:</p> <ul> <li>None</li> </ul> </li> <li> <p>Added outputs:</p> <ul> <li>None</li> </ul> </li> </ol>"},{"location":"v4-to-v5/cluster/#upgrade-migrations","title":"Upgrade Migrations","text":""},{"location":"v4-to-v5/cluster/#before-v432-example","title":"Before - v4.32 Example","text":"<pre><code>module \"eks\" {\n  source = \"github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.32.1\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.26\"\n\n  vpc_id                          = module.vpc.vpc_id\n  private_subnet_ids              = module.vpc.private_subnets\n  cluster_endpoint_private_access = true\n\n  map_roles = [\n    {\n      rolearn  = data.aws_caller_identity.current.arn\n      username = \"me\"\n      groups   = [\"system:masters\"]\n    },\n  ]\n\n  managed_node_groups = {\n    managed = {\n      node_group_name = \"managed\"\n      instance_types  = [\"m5.large\"]\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n      k8s_labels = {\n        Which = \"managed\"\n      }\n    }\n  }\n\n  fargate_profiles = {\n    fargate = {\n      fargate_profile_name = \"fargate\"\n      fargate_profile_namespaces = [{\n        namespace = \"default\"\n\n        k8s_labels = {\n          Which = \"fargate\"\n        }\n      }]\n      subnet_ids = module.vpc.private_subnets\n    }\n  }\n\n  self_managed_node_groups = {\n    self_managed = {\n      node_group_name    = \"self_managed\"\n      instance_type      = \"m5.large\"\n      launch_template_os = \"amazonlinux2eks\"\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n      k8s_labels = {\n        Which = \"self-managed\"\n      }\n    }\n  }\n\n  tags = {\n    Blueprint  = local.name\n    GithubRepo = \"github.com/aws-ia/terraform-aws-eks-blueprints\"\n  }\n}\n</code></pre>"},{"location":"v4-to-v5/cluster/#after-v50-example","title":"After - v5.0 Example","text":"<p>Any of the values that are marked with <code># Backwards compat</code> are provided to demonstrate configuration level changes to reduce the number of Terraform changes when migrating to the EKS module.</p> <pre><code>module \"eks\" {\n  source  = \"terraform-aws-modules/eks/aws\"\n  version = \"~&gt; 19.13\"\n\n  cluster_name                   = local.name\n  cluster_version                = \"1.26\"\n  cluster_endpoint_public_access = true # Backwards compat\n  cluster_enabled_log_types      = [\"api\", \"audit\", \"authenticator\", \"controllerManager\", \"scheduler\"] # Backwards compat\n\n  iam_role_name            = \"${local.name}-cluster-role\" # Backwards compat\n  iam_role_use_name_prefix = false                        # Backwards compat\n\n  kms_key_aliases = [local.name] # Backwards compat\n\n  vpc_id     = module.vpc.vpc_id\n  subnet_ids = module.vpc.private_subnets\n\n  manage_aws_auth_configmap = true\n  aws_auth_roles = [\n    {\n      rolearn  = data.aws_caller_identity.current.arn\n      username = \"me\"\n      groups   = [\"system:masters\"]\n    },\n  ]\n\n  eks_managed_node_groups = {\n    managed = {\n      iam_role_name              = \"${local.name}-managed\" # Backwards compat\n      iam_role_use_name_prefix   = false                   # Backwards compat\n      use_custom_launch_template = false                   # Backwards compat\n\n      instance_types = [\"m5.large\"]\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n      labels = {\n        Which = \"managed\"\n      }\n    }\n  }\n\n  fargate_profiles = {\n    fargate = {\n      iam_role_name            = \"${local.name}-fargate\" # Backwards compat\n      iam_role_use_name_prefix = false                   # Backwards compat\n\n      selectors = [{\n        namespace = \"default\"\n        labels = {\n          Which = \"fargate\"\n        }\n      }]\n    }\n  }\n\n  self_managed_node_groups = {\n    self_managed = {\n      name            = \"${local.name}-self_managed\" # Backwards compat\n      use_name_prefix = false                        # Backwards compat\n\n      iam_role_name            = \"${local.name}-self_managed\" # Backwards compat\n      iam_role_use_name_prefix = false                        # Backwards compat\n\n      launch_template_name            = \"self_managed-${local.name}\" # Backwards compat\n      launch_template_use_name_prefix = false                        # Backwards compat\n\n      instance_type = \"m5.large\"\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n      labels = {\n        Which = \"self-managed\"\n      }\n    }\n  }\n\n  tags = {\n    Blueprint  = local.name\n    GithubRepo = \"github.com/aws-ia/terraform-aws-eks-blueprints\"\n  }\n}\n</code></pre>"},{"location":"v4-to-v5/cluster/#diff-of-before-vs-after","title":"Diff of Before vs After","text":"<pre><code>module \"eks\" {\n-  source = \"github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.32.1\"\n+  source  = \"terraform-aws-modules/eks/aws\"\n+  version = \"~&gt; 19.13\"\n\n  cluster_name    = local.name\n  cluster_version = \"1.26\"\n\n  vpc_id                          = module.vpc.vpc_id\n  private_subnet_ids              = module.vpc.private_subnets\n+  cluster_endpoint_public_access  = true\n-  cluster_endpoint_private_access = true\n\n-  map_roles = [\n+  aws_auth_roles = [\n    {\n      rolearn  = data.aws_caller_identity.current.arn\n      username = \"me\"\n      groups   = [\"system:masters\"]\n    },\n  ]\n\n-  managed_node_groups = {\n+  eks_managed_node_groups = {\n    managed = {\n-      node_group_name = \"managed\"\n      instance_types  = [\"m5.large\"]\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n-      k8s_labels = {\n+      labels = {\n        Which = \"managed\"\n      }\n    }\n  }\n\n  fargate_profiles = {\n    fargate = {\n-      fargate_profile_name = \"fargate\"\n-      fargate_profile_namespaces = [{\n+      selectors = [{\n        namespace = \"default\"\n\n-        k8s_labels = {\n+        labels = {\n          Which = \"fargate\"\n        }\n      }]\n-      subnet_ids = module.vpc.private_subnets\n    }\n  }\n\n  self_managed_node_groups = {\n    self_managed = {\n-      node_group_name    = \"self_managed\"\n      instance_type      = \"m5.large\"\n-      launch_template_os = \"amazonlinux2eks\"\n\n      min_size     = 1\n      max_size     = 2\n      desired_size = 1\n\n-      k8s_labels = {\n+      labels = {\n        Which = \"self-managed\"\n      }\n    }\n  }\n\n  tags = {\n    Blueprint  = local.name\n    GithubRepo = \"github.com/aws-ia/terraform-aws-eks-blueprints\"\n  }\n}\n</code></pre>"},{"location":"v4-to-v5/cluster/#state-move-commands","title":"State Move Commands","text":"<p>The following Terraform state move commands are provided to aid in migrating the control plane and data plane components.</p> <pre><code># This is not removing the configmap from the cluster -\n# it will be adopted by the new module\nterraform state rm 'module.eks.kubernetes_config_map.aws_auth[0]'\n\n# Cluster\nterraform state mv 'module.eks.module.aws_eks.aws_eks_cluster.this[0]' 'module.eks.aws_eks_cluster.this[0]'\n\n# Cluster IAM role\nterraform state mv 'module.eks.module.aws_eks.aws_iam_role.this[0]' 'module.eks.aws_iam_role.this[0]'\nterraform state mv 'module.eks.module.aws_eks.aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSClusterPolicy\"]' 'module.eks.aws_iam_role_policy_attachment.this[\"AmazonEKSClusterPolicy\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSVPCResourceController\"]' 'module.eks.aws_iam_role_policy_attachment.this[\"AmazonEKSVPCResourceController\"]'\n\n# Cluster primary security group tags\n# Note: This will depend on the tags applied to the module - here we\n#       are demonstrating the two tags used in the configuration above\nterraform state mv 'module.eks.module.aws_eks.aws_ec2_tag.cluster_primary_security_group[\"Blueprint\"]' 'module.eks.aws_ec2_tag.cluster_primary_security_group[\"Blueprint\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_ec2_tag.cluster_primary_security_group[\"GithubRepo\"]' 'module.eks.aws_ec2_tag.cluster_primary_security_group[\"GithubRepo\"]'\n\n# Cluster security group\nterraform state mv 'module.eks.module.aws_eks.aws_security_group.cluster[0]' 'module.eks.aws_security_group.cluster[0]'\n\n# Cluster security group rules\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.cluster[\"ingress_nodes_443\"]' 'module.eks.aws_security_group_rule.cluster[\"ingress_nodes_443\"]'\n\n# Node security group\nterraform state mv 'module.eks.module.aws_eks.aws_security_group.node[0]' 'module.eks.aws_security_group.node[0]'\n\n# Node security group rules\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.node[\"ingress_cluster_443\"]' 'module.eks.aws_security_group_rule.node[\"ingress_cluster_443\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.node[\"ingress_cluster_kubelet\"]' 'module.eks.aws_security_group_rule.node[\"ingress_cluster_kubelet\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.node[\"ingress_self_coredns_tcp\"]' 'module.eks.aws_security_group_rule.node[\"ingress_self_coredns_tcp\"]'\nterraform state mv 'module.eks.module.aws_eks.aws_security_group_rule.node[\"ingress_self_coredns_udp\"]' 'module.eks.aws_security_group_rule.node[\"ingress_self_coredns_udp\"]'\n\n# OIDC provider\nterraform state mv 'module.eks.module.aws_eks.aws_iam_openid_connect_provider.oidc_provider[0]' 'module.eks.aws_iam_openid_connect_provider.oidc_provider[0]'\n\n# Fargate profile(s)\n# Note: This demonstrates migrating one profile that is stored under the\n#       key `fargate` in the module definition. The same set of steps would\n#       need to be performed for each profile, changing only the key name\nterraform state mv 'module.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_eks_fargate_profile.eks_fargate' 'module.eks.module.fargate_profile[\"fargate\"].aws_eks_fargate_profile.this[0]'\nterraform state mv 'module.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_iam_role.fargate[0]' 'module.eks.module.fargate_profile[\"fargate\"].aws_iam_role.this[0]'\nterraform state mv 'module.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_iam_role_policy_attachment.fargate_pod_execution_role_policy[\"arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy\"]' 'module.eks.module.fargate_profile[\"fargate\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy\"]'\n\n# Managed nodegroup(s)\n# Note: This demonstrates migrating one nodegroup that is stored under the\n#       key `managed` in the module definition. The same set of steps would\n#       need to be performed for each nodegroup, changing only the key name\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_eks_node_group.managed_ng' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_eks_node_group.this[0]'\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role.managed_ng[0]' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_iam_role.this[0]'\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role_policy_attachment.managed_ng[\"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\"]' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\"]'\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role_policy_attachment.managed_ng[\"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy\"]' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy\"]'\nterraform state mv 'module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role_policy_attachment.managed_ng[\"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy\"]' 'module.eks.module.eks_managed_node_group[\"managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy\"]'\n\n# Self-managed nodegroup(s)\n# Note: This demonstrates migrating one nodegroup that is stored under the\n#       key `self_managed` in the module definition. The same set of steps would\n#       need to be performed for each nodegroup, changing only the key name\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_autoscaling_group.self_managed_ng' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_autoscaling_group.this[0]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_instance_profile.self_managed_ng[0]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_instance_profile.this[0]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role.self_managed_ng[0]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_role.this[0]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role_policy_attachment.self_managed_ng[\"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\"]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\"]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role_policy_attachment.self_managed_ng[\"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy\"]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy\"]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role_policy_attachment.self_managed_ng[\"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy\"]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_iam_role_policy_attachment.this[\"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy\"]'\nterraform state mv 'module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].module.launch_template_self_managed_ng.aws_launch_template.this[\"self-managed-node-group\"]' 'module.eks.module.self_managed_node_group[\"self_managed\"].aws_launch_template.this[0]'\n\n# Secrets KMS key\nterraform state mv ' module.eks.module.kms[0].aws_kms_key.this' 'module.eks.module.kms.aws_kms_key.this[0]'\nterraform state mv 'module.eks.module.kms[0].aws_kms_alias.this' 'module.eks.module.kms.aws_kms_alias.this[\"migration\"]'\n\n# Cloudwatch Log Group\nterraform import 'module.eks.aws_cloudwatch_log_group.this[0]' /aws/eks/migration/cluster\n</code></pre>"},{"location":"v4-to-v5/cluster/#removed-resources","title":"Removed Resources","text":"<p>The following resources will be destroyed when migrating from EKS Blueprints v4.32.1 cluster to the v19.x EKS cluster:</p> <pre><code>module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_instance_profile.managed_ng[0]\n</code></pre> <ul> <li>It is not directly used and was intended to be used by Karpenter. The https://github.com/aws-ia/terraform-aws-eks-blueprints-addons module provides its own resource for creating an IAM instance profile for Karpenter</li> </ul> <pre><code>module.eks.module.aws_eks_managed_node_groups[\"managed\"].aws_iam_role_policy_attachment.managed_ng[\"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore\"]\n</code></pre> <ul> <li>IAM policy is not required by EKS - users can re-add this policy at their discretion</li> </ul> <pre><code>module.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_iam_policy.cwlogs[0]\nmodule.eks.module.aws_eks_fargate_profiles[\"fargate\"].aws_iam_role_policy_attachment.cwlogs[0]\n</code></pre> <ul> <li>Policy is not required by EKS</li> </ul> <pre><code>module.eks.module.aws_eks_self_managed_node_groups[\"self_managed\"].aws_iam_role_policy_attachment.self_managed_ng[\"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore\"]\n</code></pre> <ul> <li>IAM policy is not required by EKS - users can re-add this policy at their discretion</li> </ul>"},{"location":"v4-to-v5/motivation/","title":"Direction for v5 of Terraform EKS Blueprints","text":""},{"location":"v4-to-v5/motivation/#what-has-worked","title":"What Has Worked","text":"<ul> <li> <p>EKS Blueprints was started to make it easier for customers to adopt Amazon Elastic Kubernetes Service (EKS) in a shorter period of time. The project has been quite successful in this regard - hearing from customers stating that EKS Blueprints has helped them get from zero to one or more clusters running with applications in less than 1-2 weeks.</p> </li> <li> <p>EKS Blueprints has also been successful in providing working examples to users that demonstrate common architectural patterns and workload solutions. Some popular examples include:</p> </li> <li>Spark on EKS</li> <li>Karpenter on EKS Fargate</li> <li>Transparent encryption with Wireguard and Cilium</li> <li>Fully serverless cluster with EKS Fargate</li> </ul>"},{"location":"v4-to-v5/motivation/#what-has-not","title":"What Has Not","text":"<ul> <li> <p>Scaling and managing addons that are created through EKS Blueprints. With almost 1,200 projects on the CNCF roadmap, the number of various ways and methods that a project allows for deploying onto a cluster (i.e. - Datadog offers 5 different Helm charts for its service, Prometheus hosts over 30 Helm charts for its services), as well as the number of different tools used to provision addons (i.e. - Terraform, ArgoCD, FluxCD, etc.), supporting both the number of addons and their different forms has been extremely challenging for the team. In addition to managing just the sheer number of addons, supporting the different configurations that users wish to have exposed in conjunction with testing and validating those various configurations is only compounded by the number of addons and their methods of creation.</p> </li> <li> <p>Managing resources provisioned on the cluster using Terraform. Terraform is a fantastic tool for provisioning infrastructure and it is the tool of choice for many customers when it comes to creating resources in AWS. However, there are a number of downsides with Terraform when it comes to provisioning resources on a Kubernetes cluster. These include:</p> </li> <li> <p>Ordering of dependencies when relationships live outside of Terraform's HCL syntax. Terraform wants to evaluate the current state of what it controls and be able to plan a series of actions to align the current state with the desired state in one action. It does this once for each <code>terraform plan</code> or <code>terraform apply</code>, and if any issues are encountered, it simply fails and halts execution. When Terraform cannot infer the ordering of dependencies across resources (i.e. - through passing outputs of parent resources to arguments of child resources using the Terraform <code>&lt;resource&gt;.&lt;name&gt;.&lt;attribute&gt;</code> syntax), it will view this as no relationship between the resources and attempt to execute their provisioning in parallel and asynchronously. Any resources that are left waiting for a dependency will eventually timeout and fail, causing Terraform itself to timeout and fail the apply. This is where the reconciliation loop of a Kubernetes controller or operator on the cluster is better suited - continuously trying to reconcile the state over and over again as dependencies are eventually resolved. (To be clear - the issue of dependency ordering still exists, but the controller/operator will keep retrying and on each retry, some resources will succeed which will move the execution along with each cycle until everything is fully deployed. Terraform could do this if it kept re-trying, but it does not do this today)</p> </li> <li> <p>Publicly exposing access to the EKS endpoints in order to provision resources defined outside of the VPC onto the cluster. When using Terraform, the resource provisioning operation is a \"push\" model where Terraform will send requests to the EKS API Server to create resources. Coupled with the fact that the Terraform operation typically resides outside of the VPC where the cluster is running, this results in users enabling public access to the EKS endpoints to provision resources. However, the more widely accepted approach by the Kubernetes community has been the adoption of GitOps which uses a \"pull\" based model, where an operator or controller running on the cluster will pull the resource definitions from a Git repository and reconcile state from within the cluster itself. This approach is more secure as it does not require public access to the EKS endpoints and instead relies on the cluster's internal network to communicate with the EKS API Server.</p> </li> <li> <p>The nesting of multiple sub-modules in conjunction with the necessity to even require a module to be able to support an addon. When we compare and contrast the Terraform approach to addons versus the GitOps approach, the Terraform approach has a glaring disadvantage - the need to create a module that wraps the addon's Helm chart in order to provision the addon via Terraform. As opposed to the GitOps approach, where users simply consume the charts from where they are stored as needed. This creates a bottleneck on the team to review, test, and validate each new addon as well as the overhead then added for maintaining and updating those addons going forward. This also opens up more areas where breaking changes are encountered which is compounded by the fact that Terraform addons are grouped under an \"umbrella\" module which obfuscates versioning.</p> </li> <li> <p>Being able to support a combination of various tools, modules, frameworks, etc., to meet the needs of customers. The <code>terraform-aws-eks</code> was created long before EKS Blueprints, and many customers had already adopted this module for creating their clusters. In addition, Amazon has since adopted the <code>eksctl</code> as the official CLI for Amazon EKS. When EKS Blueprints was first announced, many customers raised questions asking if they needed to abandon their current clusters created through those other tools in order to adopt EKS Blueprints. The answer is no - users can and should be able to use their existing clusters while EKS Blueprints can help augment that process through its supporting modules (addons, teams, etc.). This left the team with the question - why create a Terraform module for creating an EKS cluster when the <code>terraform-aws-eks</code> already exists and the EKS Blueprints implementation already uses that module for creating the control plane and security groups?</p> </li> </ul>"},{"location":"v4-to-v5/motivation/#what-is-changing","title":"What Is Changing","text":"<p>The direction for EKS Blueprints in v5 will shift from providing an all-encompassing, monolithic \"framework\" and instead focus more on how users can organize a set of modular components to create the desired solution on Amazon EKS. This will allow customers to use the components of their choosing in a way that is more familiar to them and their organization instead of having to adopt and conform to a framework.</p> <p>With this shift in direction, the cluster definition will be removed from the project and instead examples will reference the <code>terraform-aws-eks</code> module for cluster creation. The remaining modules will be moved out to their own respective repositories as standalone projects. This leaves the EKS Blueprint project as the canonical place where users can receive guidance on how to configure their clusters to meet a desired architecture, how best to setup their clusters following well-architected practices, as well as references on the various ways that different workloads can be deployed on Amazon EKS.</p>"},{"location":"v4-to-v5/motivation/#notable-changes","title":"Notable Changes","text":"<ol> <li>EKS Blueprints will remove its Amazon EKS cluster Terraform module components (control plane, EKS managed node group, self-managed node group, and Fargate profile modules) from the project. In its place, users are encouraged to utilize the <code>terraform-aws-eks</code> module which meets or exceeds nearly all of the functionality of the EKS Blueprints v4.x cluster module. This includes the Terraform code contained at the root of the project as well as the <code>aws-eks-fargate-profiles</code>, <code>aws-eks-managed-node-groups</code>, <code>aws-eks-self-managed-node-groups</code>, and <code>launch-templates</code> modules which will all be removed from the project.</li> <li>The <code>aws-kms</code> module will be removed entirely. This was consumed in the root project module for cluster secret encryption. In its place, users can utilize the KMS key creation functionality of the <code>terraform-aws-eks</code> module or the <code>terraform-aws-kms</code> module if they wish to control the key separately from the cluster itself.</li> <li>The <code>emr-on-eks</code> module will be removed entirely; its replacement can be found in the new external module <code>terraform-aws-emr</code>.</li> <li>The <code>irsa</code> and <code>helm-addon</code> modules will be removed entirely; we have released a new external module <code>terraform-aws-eks-blueprints-addon</code> that is available on the Terraform registry that replicates/replaces the functionality of these two modules. This will now allow users, as well as partners, to create their own addons that are not natively supported by EKS Blueprints more easily and following the same process as EKS Blueprints.</li> <li>The <code>aws-eks-teams</code> module will be removed entirely; its replacement will be the new external module <code>terraform-aws-eks-blueprints-teams</code> that incorporates the changes customers have been asking for in https://github.com/aws-ia/terraform-aws-eks-blueprints/issues/842</li> <li>The integration between Terraform and ArgoCD has been removed in the initial release of v5. The team is currently investigating better patterns and solutions in conjunction with the ArgoCD and FluxCD teams that will provide a better, more integrated experience when using a GitOps based approach for cluster management. This will be released in a future version of EKS Blueprints v5 and is tracked here</li> </ol>"},{"location":"v4-to-v5/motivation/#resulting-project-structure","title":"Resulting Project Structure","text":"<p>Previously under the v4.x structure, the EKS Blueprint project was comprised of various repositories across multiple AWS organizations that looked roughly like the following:</p>"},{"location":"v4-to-v5/motivation/#v4x-structure","title":"v4.x Structure","text":"<pre><code>\u251c\u2500\u2500 aws-ia/\n|   \u251c\u2500\u2500 terraform-aws-eks-ack-addons/\n|   \u2514\u2500\u2500 terraform-aws-eks-blueprints/\n|       \u251c\u2500\u2500 aws-auth-configmap.tf\n|       \u251c\u2500\u2500 data.tf\n|       \u251c\u2500\u2500 eks-worker.tf\n|       \u251c\u2500\u2500 locals.tf\n|       \u251c\u2500\u2500 main.tf\n|       \u251c\u2500\u2500 outputs.tf\n|       \u251c\u2500\u2500 variables.tf\n|       \u251c\u2500\u2500 versions.tf\n|       \u251c\u2500\u2500 examples/\n|       \u2514\u2500\u2500 modules\n|           \u251c\u2500\u2500 aws-eks-fargate-profiles/\n|           \u251c\u2500\u2500 aws-eks-managed-node-groups/\n|           \u251c\u2500\u2500 aws-eks-self-managed-node-groups/\n|           \u251c\u2500\u2500 aws-eks-teams/\n|           \u251c\u2500\u2500 aws-kms/\n|           \u251c\u2500\u2500 emr-on-eks/\n|           \u251c\u2500\u2500 irsa/\n|           \u251c\u2500\u2500 kubernetes-addons/\n|           \u2514\u2500\u2500 launch-templates/\n\u251c\u2500\u2500 awslabs/\n|   \u251c\u2500\u2500 crossplane-on-eks/\n|   \u2514\u2500\u2500 data-on-eks/\n\u2514\u2500\u2500 aws-samples/\n    \u251c\u2500\u2500 eks-blueprints-add-ons/   # Previously shared with the CDK based EKS Blueprints project\n    \u2514\u2500\u2500 eks-blueprints-workloads/ # Previously shared with the CDK based EKS Blueprints project\n</code></pre> <p>Under th new v5.x structure, the Terraform based EKS Blueprints project will be comprised of the following repositories:</p>"},{"location":"v4-to-v5/motivation/#v5x-structure","title":"v5.x Structure","text":"<pre><code>\u251c\u2500\u2500 aws-ia/\n|   \u251c\u2500\u2500 terraform-aws-eks-ack-addons/\n|   \u251c\u2500\u2500 terraform-aws-eks-blueprints/       # Will contain only example/blueprint implementations; no modules\n|   \u251c\u2500\u2500 terraform-aws-eks-blueprints-addon  # Module for creating Terraform based addon (IRSA + Helm chart)\n|   \u251c\u2500\u2500 terraform-aws-eks-blueprints-addons # Will contain a select set of addons supported by the EKS Blueprints\n|   \u2514\u2500\u2500 terraform-aws-eks-blueprints-teams  # Was previously `aws-eks-teams/` EKS Blueprint sub-module; updated based on customer feedback\n\u2514\u2500\u2500 awslabs/\n    \u251c\u2500\u2500 crossplane-on-eks/\n    \u2514\u2500\u2500 data-on-eks/        # Data related patterns that used to be located in `terraform-aws-eks-blueprints/` are now located here\n</code></pre>"},{"location":"v4-to-v5/motivation/#what-can-users-expect","title":"What Can Users Expect","text":"<p>With these changes, the team intends to provide a better experience for users of the Terraform EKS Blueprints project as well as new and improved reference architectures. Following the v5 changes, the team intends to:</p> <ol> <li>Improved quality of the examples provided - more information on the intent of the example, why it might be useful for users, what scenarios is the pattern applicable, etc. Where applicable, architectural diagrams and supporting material will be provided to highlight the intent of the example and how its constructed.</li> <li>A more clear distinction between a blueprint and a usage reference. For example - the Karpenter on EKS Fargate blueprint should demonstrate all of the various aspects that users should be aware of and consider in order to take full advantage of this pattern (recommended practices, observability, logging, monitoring, security, day 2 operations, etc.); this is what makes it a blueprint. In contrast, a usage reference would be an example that shows how users can pass configuration values to the Karpenter provisioner. This example is less focused on the holistic architecture and more focused on how one might configure Karpenter using the implementation. The EKS Blueprints repository will focus mostly on holistic architecture and patterns, and any usage references should be saved for the repository that contains that implementation definition (i.e. - the <code>terraform-aws-eks-blueprints-addons</code> repository where the addon implementation is defined).</li> <li>Faster, and more responsive feedback. The first part of this is going to be improved documentation on how to contribute which should help clarify whether a contribution is worthy and willing to be accepted by the team before any effort is spent by the contributor. However, the goal of v5 is to focus more on the value added benefits that EKS Blueprints was created to provide as opposed to simply mass producing Helm chart wrappers (addons) and trying to keep up with that operationally intensive process.</li> <li>Lastly, more examples and blueprints that demonstrate various architectures and workloads that run on top of Amazon EKS as well as integrations into other AWS services.</li> </ol>"},{"location":"v4-to-v5/teams/","title":"Migrate to EKS Blueprints Teams Module","text":"<p>Please consult the docs/v4-to-v5/example directory for reference configurations. If you find a bug, please open an issue with supporting configuration to reproduce.</p>"},{"location":"v4-to-v5/teams/#this-guide-is-under-active-development","title":"\u26a0\ufe0f This guide is under active development.","text":""},{"location":"v4-to-v5/teams/#list-of-backwards-incompatible-changes","title":"List of backwards incompatible changes","text":"<p>-</p>"},{"location":"v4-to-v5/teams/#additional-changes","title":"Additional changes","text":""},{"location":"v4-to-v5/teams/#added","title":"Added","text":"<p>-</p>"},{"location":"v4-to-v5/teams/#modified","title":"Modified","text":"<p>-</p>"},{"location":"v4-to-v5/teams/#removed","title":"Removed","text":"<p>-</p>"},{"location":"v4-to-v5/teams/#variable-and-output-changes","title":"Variable and output changes","text":"<ol> <li> <p>Removed variables:</p> <p>-</p> </li> <li> <p>Renamed variables:</p> <p>-</p> </li> <li> <p>Added variables:</p> <p>-</p> </li> <li> <p>Removed outputs:</p> <p>-</p> </li> <li> <p>Renamed outputs:</p> <p>-</p> </li> <li> <p>Added outputs:</p> <p>-</p> </li> </ol>"},{"location":"v4-to-v5/teams/#upgrade-migrations","title":"Upgrade Migrations","text":""},{"location":"v4-to-v5/teams/#before-v4x-example","title":"Before - v4.x Example","text":"<pre><code>module \"eks_blueprints\" {\n  source = \"github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.32.1\"\n\n  # TODO\n\n}\n</code></pre>"},{"location":"v4-to-v5/teams/#after-v5x-example","title":"After - v5.x Example","text":"<pre><code>module \"eks_blueprints_teams\" {\n  source  = \"aws-ia/eks-blueprints-teams/aws\"\n  version = \"~&gt; 1.0\"\n\n  # TODO\n\n}\n</code></pre>"},{"location":"v4-to-v5/teams/#diff-of-before-vs-after","title":"Diff of Before vs After","text":"<pre><code>module \"eks_blueprints_teams\" {\n-  source = \"github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.32.1\"\n+  source  = \"aws-ia/eks-blueprints-teams/aws\"\n+  version = \"~&gt; 1.0\"\n\n  # TODO\n}\n</code></pre>"},{"location":"v4-to-v5/teams/#state-move-commands","title":"State Move Commands","text":"<p>In conjunction with the changes above, users can elect to move their external capacity provider(s) under this module using the following move command. Command is shown using the values from the example shown above, please update to suit your configuration names:</p> <pre><code>terraform state mv 'xxx' 'yyy'\n</code></pre>"},{"location":"v4-to-v5/example/","title":"Migration - v4 to v5","text":""}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 091ad6e5bd..d5665939f7 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -75,6 +75,11 @@
          <lastmod>2024-05-08</lastmod>
          <changefreq>daily</changefreq>
     </url>
+    <url>
+         <loc>https://aws-ia.github.io/terraform-aws-eks-blueprints/patterns/karpenter-mng/</loc>
+         <lastmod>2024-05-08</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
     <url>
          <loc>https://aws-ia.github.io/terraform-aws-eks-blueprints/patterns/karpenter/</loc>
          <lastmod>2024-05-08</lastmod>
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 895ab6faeda0dee97b9744a617e8b4ae0ed709ac..e213be90c221116c974d2e506fbe206c1f3cfeba 100644
GIT binary patch
delta 646
zcmV;10(t$61(F4j7k`+^ZW}QShVOX_!*_a|ASu$qac+Hr_FT{!S|&zYM2(lXFUK3w
zdr^R$gFTWc;s3yqNOARkjA6E?DxA~X{Bg0I&nUUvbMkNVpFh6C>->HFuo`O+YBM!X
z@;2YKR=#!J;c!?8H;Pzfk*n{-?X)4?wX<6;zRuSVGkvB7E`M+5XBDzrPpgP7g9ib|
zh3B>#Hw(^PqgrvBYlLZt27yf&sZ>r)x>YyL{k#-Pi@9GvEuWrY`2x#VHM#bEGIKjj
zp0<@f);)4Kt-4P=7meLr<lmFK#Sd$eJ+-Fp2WeOv_15J7Dk_^XNY|6JnorgM^oMG~
z&^fjyy<nTPOMizxClc@&30zL<h7hRQ#nG{Sao`d|qISr2fRY2dlUbyx4}2v`Fcx32
zC$lJYylfZ{a+U1Gh&#v@tH(ylQ8gq762mBC2q)0q{RHH~D9!NxSNmM_IQug)g1Rfa
z-a1trL*tu^Kvw`Mc_8a_T=rkigVsMjmZr8=)el$;O@A2;#N_Q$*d-V@<gzrg05O2W
zf{xos$Fm#klAI6RMhTNU!9izh1EbwwpZ44{f-fV`7R_SZvXwG!G_?nf^wn|-cEt&t
z20-U5qljdk7J+*`gr@_***=UXo;)<WTSYVAMw;WB1<(v$vw6+obMNeaM=i)DeoRKi
zA+X43cz<6^HI|?ac6O@~2d?Btto_9^jsuVJzwn3)tjF%PIsq=nI3{+ex07UzPZF0h
zm5TJ1_zb1emXFX=Aa5JzR-|-#<<0R{t7LmOFZD{xuz%^W?{gBf@{JZB?)qbHQx<7(
gPuvkq-mUKH>4>p}e-IAOe?o2j8?QK=yA&P(08jEf*#H0l

delta 640
zcmV-`0)PFI1&jrd7k^mEZW}QSz2_?o-|2CJq(}?Lx%CU$b3tq9F)`XAN?!iH9B)YP
zMFDmWwnb6Id**P{>irn}Y)?fv#kcw6VmY5tbgAb!yv=|9_zti0_w~c76)#k0svPBQ
zzN<BV>$=0?un?{kFv}v9p%b^$g>+Y8w_JRkuODXmOa?B$&VSF!C%2xgh?>EJ0OMld
zx@(&Sr>;^dxJ@O%v_+%9#y84^W0h{zO=mxwLa8D3>!;<@Gb~?V`Kl^c*(Wu(!#L2k
z(8sz*_NP_%Npiv1-9i36xLf?N26>=b)%_q9OQqf#{9jpNGkWQIl3LQq8Ug*F647^#
ztwGONC+yPUkbfcx3>XMpis}aMso15Xd;9FbMTSWAkje--d3GnWlqMMrg(yPHL&2WR
zQlaA|U_i(fvR5PSM7CH4tfV}OmSj(2m`d~h1S;K6gq&|u4adLgr>yDh&&mkuuH<^_
ztU~jZZ!QA00Fu*4*4c61e>rbjfB9Id>RLrVV#yRO|9}5u+mK6A%@PP69A<P}Pr9Gn
zXqU-3RaJ@@-3bnQS{E4XmdUv1o)L!JfVOCs#;wlhw$Y{-v_=-o&C8XJ=Qsj7XPJsf
z(qjR**Cae80B1WHPkho??S_i>j+L}KHcLQzU+sgn`%S&G`yN#<m-;a&Wrx5b&2U^y
zw-a6$?0M|Y@dqw6G%WqaGmbsC@LzbudDgUho$JA+5L#r1+Sf;G{3LSD(<O_}SI;`&
z)0U6Wlc&Kpo!j-E(_t}%+by4VGS(QWS89U&ONV`*qL_tmbZo!l52;RFq`{cDQ!qJO
a-O<w#L-zlm9G<^~>iRd6IT&e99RL8X3raNr

diff --git a/snippets/ipv4-prefix-delegation/index.html b/snippets/ipv4-prefix-delegation/index.html
index 4577f29ec2..a1d56c3e89 100644
--- a/snippets/ipv4-prefix-delegation/index.html
+++ b/snippets/ipv4-prefix-delegation/index.html
@@ -554,12 +554,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -856,10 +877,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -870,8 +891,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/snippets/vpc-cni-custom-networking/index.html b/snippets/vpc-cni-custom-networking/index.html
index 31cc5775d6..3ff47ccfb0 100644
--- a/snippets/vpc-cni-custom-networking/index.html
+++ b/snippets/vpc-cni-custom-networking/index.html
@@ -554,12 +554,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -856,10 +877,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -870,8 +891,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/v4-to-v5/addons/index.html b/v4-to-v5/addons/index.html
index 06413cb91f..504ca7a8c6 100644
--- a/v4-to-v5/addons/index.html
+++ b/v4-to-v5/addons/index.html
@@ -554,12 +554,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -856,10 +877,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -870,8 +891,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/v4-to-v5/cluster/index.html b/v4-to-v5/cluster/index.html
index 853c7432f7..524b431044 100644
--- a/v4-to-v5/cluster/index.html
+++ b/v4-to-v5/cluster/index.html
@@ -554,12 +554,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -856,10 +877,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -870,8 +891,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/v4-to-v5/example/index.html b/v4-to-v5/example/index.html
index a39cf020a9..cc50b8d790 100644
--- a/v4-to-v5/example/index.html
+++ b/v4-to-v5/example/index.html
@@ -552,12 +552,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -854,10 +875,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -868,8 +889,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/v4-to-v5/motivation/index.html b/v4-to-v5/motivation/index.html
index cbe3e87cf3..17866929c6 100644
--- a/v4-to-v5/motivation/index.html
+++ b/v4-to-v5/motivation/index.html
@@ -554,12 +554,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -856,10 +877,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -870,8 +891,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>
diff --git a/v4-to-v5/teams/index.html b/v4-to-v5/teams/index.html
index e0ffe8da46..cbcec2eb26 100644
--- a/v4-to-v5/teams/index.html
+++ b/v4-to-v5/teams/index.html
@@ -554,12 +554,33 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../patterns/karpenter-mng/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Karpenter on EKS MNG
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../patterns/karpenter/" class="md-nav__link">
         
   
   <span class="md-ellipsis">
-    Karpenter
+    Karpenter on EKS Fargate
   </span>
   
 
@@ -856,10 +877,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_25" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_26" >
         
           
-          <label class="md-nav__link" for="__nav_3_25" id="__nav_3_25_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_26" id="__nav_3_26_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -870,8 +891,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_25_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_25">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_26_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_26">
             <span class="md-nav__icon md-icon"></span>
             Vpc lattice
           </label>