Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(#20): dedicated Nodegroups for internal workloads #24

Merged
merged 1 commit into from
Feb 3, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions integration-tests/jest.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ module.exports = {
transform: {
"^.+\\.tsx?$": "ts-jest",
},
moduleNameMapper: {
"^~/(.*)$": "<rootDir>/src/$1",
},
testTimeout: 1000 * 60, // 60 seconds for now
resetMocks: true,
restoreMocks: true,
Expand Down
3 changes: 0 additions & 3 deletions integration-tests/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@
"baseUrl": ".",
"outDir": "dist",
"paths": {
"~/*": [
"lib/*"
],
"@superluminar-io/*": [
"../packages/@superluminar-io/*/src/"
]
Expand Down
3 changes: 0 additions & 3 deletions packages/@superluminar-io/super-eks/jest.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ module.exports = {
transform: {
"^.+\\.(ts|tsx)$": "ts-jest",
},
moduleNameMapper: {
"^~/(.*)$": "<rootDir>/src/$1",
},
testTimeout: 1000 * 60, // 60 seconds for now
resetMocks: true,
restoreMocks: true,
Expand Down
36 changes: 36 additions & 0 deletions packages/@superluminar-io/super-eks/src/config/cluster.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
export enum taintEffect {
NoSchedule = "NoSchedule",
PreferNoSchedule = "PreferNoSchedule",
NoExecute = "NoExecute",
}

export interface NodeTaint {
key: string
value: string
effect: taintEffect
}

export const SuperEksNodegroup = {
taint: {
key: "workload",
value: "super-eks",
effect: taintEffect.NoSchedule,
},
labels: { workload: "super-eks" },
}

export function nodeTaintUserdata(taint: NodeTaint): string {
return `MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="

--==MYBOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"

#!/bin/bash
#!/bin/bash
sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" --register-with-taints=${taint.key}=${taint.value}:${taint.effect}"' /etc/eks/bootstrap.sh


--==MYBOUNDARY==--\\
`
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import * as cdk from "@aws-cdk/core"
import * as eks from "@aws-cdk/aws-eks"
import * as iam from "@aws-cdk/aws-iam"

import { SuperEksNodegroup } from "../config/cluster"

export interface AwsLoadBalancerControllerProps {
readonly cluster: eks.ICluster
readonly vpcId: string
Expand Down Expand Up @@ -65,6 +67,8 @@ export class AwsLoadBalancerController extends cdk.Construct {
create: false,
name: serviceAccount.serviceAccountName,
},
tolerations: [SuperEksNodegroup.taint],
nodeSelector: SuperEksNodegroup.labels,
},
})
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import * as cdk from "@aws-cdk/core"
import * as eks from "@aws-cdk/aws-eks"
import * as iam from "@aws-cdk/aws-iam"

import { SuperEksNodegroup } from "../config/cluster"

export interface ExternalDNSProps {
/**
* The EKS cluster to install to
Expand Down Expand Up @@ -75,6 +77,8 @@ export class ExternalDNS extends cdk.Construct {
create: false,
name: serviceAccount.serviceAccountName,
},
tolerations: [SuperEksNodegroup.taint],
nodeSelector: SuperEksNodegroup.labels,
},
})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import * as cdk from "@aws-cdk/core"
import * as eks from "@aws-cdk/aws-eks"
import * as iam from "@aws-cdk/aws-iam"

import { SuperEksNodegroup } from "../config/cluster"

export interface FluentBitProps {
readonly cluster: eks.ICluster
readonly region: string
Expand Down Expand Up @@ -46,6 +48,8 @@ export class FluentBit extends cdk.Construct {
create: false,
name: serviceAccount.serviceAccountName,
},
tolerations: [SuperEksNodegroup.taint],
nodeSelector: SuperEksNodegroup.labels,
firehose: {
enabled: false,
},
Expand Down
23 changes: 22 additions & 1 deletion packages/@superluminar-io/super-eks/src/constructs/super-eks.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import * as cdk from "@aws-cdk/core"
import { IHostedZone } from "@aws-cdk/aws-route53"

import * as eks from "@aws-cdk/aws-eks"
import * as ec2 from "@aws-cdk/aws-ec2"

import { ExternalDNS } from "./external-dns"
import { AwsLoadBalancerController } from "./aws-load-balancer-controller"
import { FluentBit } from "./fluent-bit"

import { SuperEksNodegroup, nodeTaintUserdata } from "../config/cluster"

export interface SuperEksProps {
hostedZone: IHostedZone
vpc?: ec2.IVpc
Expand All @@ -21,7 +23,10 @@ export class SuperEks extends cdk.Construct {
constructor(scope: cdk.Construct, id: string, props: SuperEksProps) {
super(scope, id)
this.props = props

this.cluster = this.configureCluster()
this.addSuperEksNodegroup()

this.configureExternalDNS()
this.configureAwsLoadBalancerController()
this.configureFluentBit()
Expand All @@ -34,6 +39,22 @@ export class SuperEks extends cdk.Construct {
})
}

private addSuperEksNodegroup() {
const lt = new ec2.CfnLaunchTemplate(this, "LaunchTemplate", {
launchTemplateData: {
userData: cdk.Fn.base64(nodeTaintUserdata(SuperEksNodegroup.taint)),
},
})

this.cluster.addNodegroupCapacity("SuperEksNodegroup", {
labels: SuperEksNodegroup.labels,
instanceTypes: [
ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.LARGE),
],
launchTemplateSpec: { id: lt.ref, version: lt.attrLatestVersionNumber },
})
}

private configureExternalDNS(): void {
new ExternalDNS(this, "ExternalDNS", {
cluster: this.cluster,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import "@aws-cdk/assert/jest"
import { Stack } from "@aws-cdk/core"
import { Cluster } from "@aws-cdk/aws-eks"
import * as eks from "@aws-cdk/aws-eks"
import { AwsLoadBalancerController } from "~/constructs/aws-load-balancer-controller"
import { AwsLoadBalancerController } from "../../src/constructs/aws-load-balancer-controller"

describe("aws-load-balancer-controller", () => {
test("namespace can be configured and created", () => {
Expand Down Expand Up @@ -37,4 +37,31 @@ describe("aws-load-balancer-controller", () => {
Namespace: "kube-system",
})
})

test("Chart is configured correctly", () => {
const stack = new Stack()
const cluster = new Cluster(stack, "EKS", {
version: eks.KubernetesVersion.V1_18,
})
new AwsLoadBalancerController(stack, "ALB", {
cluster,
region: "eu-west-1",
vpcId: "vpc-21313",
})

expect(stack).toHaveResource("Custom::AWSCDK-EKS-HelmChart", {
Values: {
"Fn::Join": [
"",
[
'{"clusterName":"',
{
Ref: "EKSE2753513",
},
'","region":"eu-west-1","vpcId":"vpc-21313","serviceAccount":{"create":false,"name":"aws-load-balancer-controller"},"tolerations":[{"key":"workload","value":"super-eks","effect":"NoSchedule"}],"nodeSelector":{"workload":"super-eks"}}',
],
],
},
})
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import "@aws-cdk/assert/jest"
import { Stack } from "@aws-cdk/core"
import { Cluster } from "@aws-cdk/aws-eks"
import * as eks from "@aws-cdk/aws-eks"
import { ExternalDNS } from "~/constructs/external-dns"
import { ExternalDNS } from "../../src/constructs/external-dns"

describe("external-dns", () => {
test('namespace defaults to "external-dns"', () => {
Expand All @@ -23,7 +23,7 @@ describe("external-dns", () => {
})
})

test("Hosted zone ID filter is set on Helm chart", () => {
test("Chart is configured correctly", () => {
const stack = new Stack()
const cluster = new Cluster(stack, "EKS", {
version: eks.KubernetesVersion.V1_18,
Expand All @@ -34,7 +34,7 @@ describe("external-dns", () => {
})
expect(stack).toHaveResource("Custom::AWSCDK-EKS-HelmChart", {
Values:
'{"zoneIdFilters":["Z1PA6795UKMFR9"],"serviceAccount":{"create":false,"name":"external-dns"}}',
'{"zoneIdFilters":["Z1PA6795UKMFR9"],"serviceAccount":{"create":false,"name":"external-dns"},"tolerations":[{"key":"workload","value":"super-eks","effect":"NoSchedule"}],"nodeSelector":{"workload":"super-eks"}}',
})
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import "@aws-cdk/assert/jest"
import { Stack } from "@aws-cdk/core"
import { Cluster } from "@aws-cdk/aws-eks"
import * as eks from "@aws-cdk/aws-eks"
import { FluentBit } from "~/constructs/fluent-bit"
import { FluentBit } from "../../src/constructs/fluent-bit"

describe("fluent-bit", () => {
test("namespace can be configured and created", () => {
Expand Down Expand Up @@ -47,7 +47,7 @@ describe("fluent-bit", () => {
})
expect(stack).toHaveResource("Custom::AWSCDK-EKS-HelmChart", {
Values:
'{"serviceAccount":{"create":false,"name":"fluent-bit"},"firehose":{"enabled":false},"kinesis":{"enabled":false},"elasticsearch":{"enabled":false},"cloudWatch":{"region":"eu-west-1"}}',
'{"serviceAccount":{"create":false,"name":"fluent-bit"},"tolerations":[{"key":"workload","value":"super-eks","effect":"NoSchedule"}],"nodeSelector":{"workload":"super-eks"},"firehose":{"enabled":false},"kinesis":{"enabled":false},"elasticsearch":{"enabled":false},"cloudWatch":{"region":"eu-west-1"}}',
})
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { expect as expectCDK, haveResource } from "@aws-cdk/assert"
import * as cdk from "@aws-cdk/core"
import * as route53 from "@aws-cdk/aws-route53"

import { SuperEks } from "~/constructs/super-eks"
import { SuperEks } from "../../src/constructs/super-eks"

test("Empty Cluster", () => {
const app = new cdk.App()
Expand Down
5 changes: 0 additions & 5 deletions packages/@superluminar-io/super-eks/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,6 @@
"es2019"
],
"baseUrl": ".",
"paths": {
"~/*": [
"src/*"
]
},
"outDir": "dist",
"declaration": true,
"strict": true,
Expand Down