feat(#20): dedicated Nodegroups for internal workloads

superluminar-io · Feb 3, 2021 · a4c7baf · a4c7baf
1 parent 20286b3
commit a4c7baf
Show file tree

Hide file tree

Showing 13 changed files with 104 additions and 22 deletions.
diff --git a/integration-tests/jest.config.js b/integration-tests/jest.config.js
@@ -6,9 +6,6 @@ module.exports = {
   transform: {
     "^.+\\.tsx?$": "ts-jest",
   },
-  moduleNameMapper: {
-    "^~/(.*)$": "<rootDir>/src/$1",
-  },
   testTimeout: 1000 * 60, // 60 seconds for now
   resetMocks: true,
   restoreMocks: true,

diff --git a/integration-tests/tsconfig.json b/integration-tests/tsconfig.json
@@ -9,9 +9,6 @@
     "baseUrl": ".",
     "outDir": "dist",
     "paths": {
-      "~/*": [
-        "lib/*"
-      ],
       "@superluminar-io/*": [
         "../packages/@superluminar-io/*/src/"
       ]

diff --git a/packages/@superluminar-io/super-eks/jest.config.js b/packages/@superluminar-io/super-eks/jest.config.js
@@ -6,9 +6,6 @@ module.exports = {
   transform: {
     "^.+\\.(ts|tsx)$": "ts-jest",
   },
-  moduleNameMapper: {
-    "^~/(.*)$": "<rootDir>/src/$1",
-  },
   testTimeout: 1000 * 60, // 60 seconds for now
   resetMocks: true,
   restoreMocks: true,

diff --git a/packages/@superluminar-io/super-eks/src/config/cluster.ts b/packages/@superluminar-io/super-eks/src/config/cluster.ts
@@ -0,0 +1,36 @@
+export enum taintEffect {
+  NoSchedule = "NoSchedule",
+  PreferNoSchedule = "PreferNoSchedule",
+  NoExecute = "NoExecute",
+}
+
+export interface NodeTaint {
+  key: string
+  value: string
+  effect: taintEffect
+}
+
+export const SuperEksNodegroup = {
+  taint: {
+    key: "workload",
+    value: "super-eks",
+    effect: taintEffect.NoSchedule,
+  },
+  labels: { workload: "super-eks" },
+}
+
+export function nodeTaintUserdata(taint: NodeTaint): string {
+  return `MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="
+
+--==MYBOUNDARY==
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+#!/bin/bash
+sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" --register-with-taints=${taint.key}=${taint.value}:${taint.effect}"' /etc/eks/bootstrap.sh
+
+
+--==MYBOUNDARY==--\\
+`
+}
diff --git a/packages/@superluminar-io/super-eks/src/constructs/aws-load-balancer-controller.ts b/packages/@superluminar-io/super-eks/src/constructs/aws-load-balancer-controller.ts
@@ -4,6 +4,8 @@ import * as cdk from "@aws-cdk/core"
 import * as eks from "@aws-cdk/aws-eks"
 import * as iam from "@aws-cdk/aws-iam"
 
+import { SuperEksNodegroup } from "../config/cluster"
+
 export interface AwsLoadBalancerControllerProps {
   readonly cluster: eks.ICluster
   readonly vpcId: string
@@ -65,6 +67,8 @@ export class AwsLoadBalancerController extends cdk.Construct {
           create: false,
           name: serviceAccount.serviceAccountName,
         },
+        tolerations: [SuperEksNodegroup.taint],
+        nodeSelector: SuperEksNodegroup.labels,
       },
     })
   }

diff --git a/packages/@superluminar-io/super-eks/src/constructs/external-dns.ts b/packages/@superluminar-io/super-eks/src/constructs/external-dns.ts
@@ -2,6 +2,8 @@ import * as cdk from "@aws-cdk/core"
 import * as eks from "@aws-cdk/aws-eks"
 import * as iam from "@aws-cdk/aws-iam"
 
+import { SuperEksNodegroup } from "../config/cluster"
+
 export interface ExternalDNSProps {
   /**
    * The EKS cluster to install to
@@ -75,6 +77,8 @@ export class ExternalDNS extends cdk.Construct {
           create: false,
           name: serviceAccount.serviceAccountName,
         },
+        tolerations: [SuperEksNodegroup.taint],
+        nodeSelector: SuperEksNodegroup.labels,
       },
     })
 

diff --git a/packages/@superluminar-io/super-eks/src/constructs/fluent-bit.ts b/packages/@superluminar-io/super-eks/src/constructs/fluent-bit.ts
@@ -2,6 +2,8 @@ import * as cdk from "@aws-cdk/core"
 import * as eks from "@aws-cdk/aws-eks"
 import * as iam from "@aws-cdk/aws-iam"
 
+import { SuperEksNodegroup } from "../config/cluster"
+
 export interface FluentBitProps {
   readonly cluster: eks.ICluster
   readonly region: string
@@ -46,6 +48,8 @@ export class FluentBit extends cdk.Construct {
           create: false,
           name: serviceAccount.serviceAccountName,
         },
+        tolerations: [SuperEksNodegroup.taint],
+        nodeSelector: SuperEksNodegroup.labels,
         firehose: {
           enabled: false,
         },

diff --git a/packages/@superluminar-io/super-eks/src/constructs/super-eks.ts b/packages/@superluminar-io/super-eks/src/constructs/super-eks.ts
@@ -1,12 +1,14 @@
 import * as cdk from "@aws-cdk/core"
 import { IHostedZone } from "@aws-cdk/aws-route53"
-
 import * as eks from "@aws-cdk/aws-eks"
 import * as ec2 from "@aws-cdk/aws-ec2"
+
 import { ExternalDNS } from "./external-dns"
 import { AwsLoadBalancerController } from "./aws-load-balancer-controller"
 import { FluentBit } from "./fluent-bit"
 
+import { SuperEksNodegroup, nodeTaintUserdata } from "../config/cluster"
+
 export interface SuperEksProps {
   hostedZone: IHostedZone
   vpc?: ec2.IVpc
@@ -21,7 +23,10 @@ export class SuperEks extends cdk.Construct {
   constructor(scope: cdk.Construct, id: string, props: SuperEksProps) {
     super(scope, id)
     this.props = props
+
     this.cluster = this.configureCluster()
+    this.addSuperEksNodegroup()
+
     this.configureExternalDNS()
     this.configureAwsLoadBalancerController()
     this.configureFluentBit()
@@ -34,6 +39,22 @@ export class SuperEks extends cdk.Construct {
     })
   }
 
+  private addSuperEksNodegroup() {
+    const lt = new ec2.CfnLaunchTemplate(this, "LaunchTemplate", {
+      launchTemplateData: {
+        userData: cdk.Fn.base64(nodeTaintUserdata(SuperEksNodegroup.taint)),
+      },
+    })
+
+    this.cluster.addNodegroupCapacity("SuperEksNodegroup", {
+      labels: SuperEksNodegroup.labels,
+      instanceTypes: [
+        ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.LARGE),
+      ],
+      launchTemplateSpec: { id: lt.ref, version: lt.attrLatestVersionNumber },
+    })
+  }
+
   private configureExternalDNS(): void {
     new ExternalDNS(this, "ExternalDNS", {
       cluster: this.cluster,

diff --git a/packages/@superluminar-io/super-eks/test/constructs/aws-load-balancer-controller.test.ts b/packages/@superluminar-io/super-eks/test/constructs/aws-load-balancer-controller.test.ts
@@ -2,7 +2,7 @@ import "@aws-cdk/assert/jest"
 import { Stack } from "@aws-cdk/core"
 import { Cluster } from "@aws-cdk/aws-eks"
 import * as eks from "@aws-cdk/aws-eks"
-import { AwsLoadBalancerController } from "~/constructs/aws-load-balancer-controller"
+import { AwsLoadBalancerController } from "../../src/constructs/aws-load-balancer-controller"
 
 describe("aws-load-balancer-controller", () => {
   test("namespace can be configured and created", () => {
@@ -37,4 +37,31 @@ describe("aws-load-balancer-controller", () => {
       Namespace: "kube-system",
     })
   })
+
+  test("Chart is configured correctly", () => {
+    const stack = new Stack()
+    const cluster = new Cluster(stack, "EKS", {
+      version: eks.KubernetesVersion.V1_18,
+    })
+    new AwsLoadBalancerController(stack, "ALB", {
+      cluster,
+      region: "eu-west-1",
+      vpcId: "vpc-21313",
+    })
+
+    expect(stack).toHaveResource("Custom::AWSCDK-EKS-HelmChart", {
+      Values: {
+        "Fn::Join": [
+          "",
+          [
+            '{"clusterName":"',
+            {
+              Ref: "EKSE2753513",
+            },
+            '","region":"eu-west-1","vpcId":"vpc-21313","serviceAccount":{"create":false,"name":"aws-load-balancer-controller"},"tolerations":[{"key":"workload","value":"super-eks","effect":"NoSchedule"}],"nodeSelector":{"workload":"super-eks"}}',
+          ],
+        ],
+      },
+    })
+  })
 })
diff --git a/packages/@superluminar-io/super-eks/test/constructs/external-dns.test.ts b/packages/@superluminar-io/super-eks/test/constructs/external-dns.test.ts
@@ -2,7 +2,7 @@ import "@aws-cdk/assert/jest"
 import { Stack } from "@aws-cdk/core"
 import { Cluster } from "@aws-cdk/aws-eks"
 import * as eks from "@aws-cdk/aws-eks"
-import { ExternalDNS } from "~/constructs/external-dns"
+import { ExternalDNS } from "../../src/constructs/external-dns"
 
 describe("external-dns", () => {
   test('namespace defaults to "external-dns"', () => {
@@ -23,7 +23,7 @@ describe("external-dns", () => {
     })
   })
 
-  test("Hosted zone ID filter is set on Helm chart", () => {
+  test("Chart is configured correctly", () => {
     const stack = new Stack()
     const cluster = new Cluster(stack, "EKS", {
       version: eks.KubernetesVersion.V1_18,
@@ -34,7 +34,7 @@ describe("external-dns", () => {
     })
     expect(stack).toHaveResource("Custom::AWSCDK-EKS-HelmChart", {
       Values:
-        '{"zoneIdFilters":["Z1PA6795UKMFR9"],"serviceAccount":{"create":false,"name":"external-dns"}}',
+        '{"zoneIdFilters":["Z1PA6795UKMFR9"],"serviceAccount":{"create":false,"name":"external-dns"},"tolerations":[{"key":"workload","value":"super-eks","effect":"NoSchedule"}],"nodeSelector":{"workload":"super-eks"}}',
     })
   })
 })
diff --git a/packages/@superluminar-io/super-eks/test/constructs/fluent-bit.test.ts b/packages/@superluminar-io/super-eks/test/constructs/fluent-bit.test.ts
@@ -2,7 +2,7 @@ import "@aws-cdk/assert/jest"
 import { Stack } from "@aws-cdk/core"
 import { Cluster } from "@aws-cdk/aws-eks"
 import * as eks from "@aws-cdk/aws-eks"
-import { FluentBit } from "~/constructs/fluent-bit"
+import { FluentBit } from "../../src/constructs/fluent-bit"
 
 describe("fluent-bit", () => {
   test("namespace can be configured and created", () => {
@@ -47,7 +47,7 @@ describe("fluent-bit", () => {
     })
     expect(stack).toHaveResource("Custom::AWSCDK-EKS-HelmChart", {
       Values:
-        '{"serviceAccount":{"create":false,"name":"fluent-bit"},"firehose":{"enabled":false},"kinesis":{"enabled":false},"elasticsearch":{"enabled":false},"cloudWatch":{"region":"eu-west-1"}}',
+        '{"serviceAccount":{"create":false,"name":"fluent-bit"},"tolerations":[{"key":"workload","value":"super-eks","effect":"NoSchedule"}],"nodeSelector":{"workload":"super-eks"},"firehose":{"enabled":false},"kinesis":{"enabled":false},"elasticsearch":{"enabled":false},"cloudWatch":{"region":"eu-west-1"}}',
     })
   })
 })
diff --git a/packages/@superluminar-io/super-eks/test/constructs/super-eks.test.ts b/packages/@superluminar-io/super-eks/test/constructs/super-eks.test.ts
@@ -2,7 +2,7 @@ import { expect as expectCDK, haveResource } from "@aws-cdk/assert"
 import * as cdk from "@aws-cdk/core"
 import * as route53 from "@aws-cdk/aws-route53"
 
-import { SuperEks } from "~/constructs/super-eks"
+import { SuperEks } from "../../src/constructs/super-eks"
 
 test("Empty Cluster", () => {
   const app = new cdk.App()

diff --git a/packages/@superluminar-io/super-eks/tsconfig.json b/packages/@superluminar-io/super-eks/tsconfig.json
@@ -7,11 +7,6 @@
       "es2019"
     ],
     "baseUrl": ".",
-    "paths": {
-      "~/*": [
-        "src/*"
-      ]
-    },
     "outDir": "dist",
     "declaration": true,
     "strict": true,