From 5b29b87ad7f7e5d79ebca5278abe5f94a0f3f3e7 Mon Sep 17 00:00:00 2001 From: Prasunna Soppa Date: Mon, 13 May 2024 18:48:57 +0530 Subject: [PATCH 1/5] add support to push logs to s3 bucket --- lib/aws/eks.ts | 2 ++ lib/aws/log_bucket.ts | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 lib/aws/log_bucket.ts diff --git a/lib/aws/eks.ts b/lib/aws/eks.ts index e6af4ba..4da0c1f 100644 --- a/lib/aws/eks.ts +++ b/lib/aws/eks.ts @@ -7,6 +7,7 @@ import { Construct } from "constructs"; import { Config } from "./config"; import { ElasticacheStack } from "./elasticache"; import { DataBaseConstruct } from "./rds"; +import { LogsBucket } from "./log_bucket"; import * as kms from "aws-cdk-lib/aws-kms"; import { readFileSync } from "fs"; import { Secret } from "aws-cdk-lib/aws-secretsmanager"; @@ -56,6 +57,7 @@ export class EksStack { clusterName: "hs-eks-cluster", }); + const logsBucket = new LogsBucket(scope, cluster, "app-logs-s3-service-account"); cluster.node.addDependency(ecrTransfer.codebuildTrigger); cdk.Tags.of(cluster).add("SubStack", "HyperswitchEKS"); diff --git a/lib/aws/log_bucket.ts b/lib/aws/log_bucket.ts new file mode 100644 index 0000000..b6b5e05 --- /dev/null +++ b/lib/aws/log_bucket.ts @@ -0,0 +1,32 @@ +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as cdk from "aws-cdk-lib"; +import { Construct } from 'constructs'; +import * as s3 from "aws-cdk-lib/aws-s3"; +import * as iam from "aws-cdk-lib/aws-iam"; +import * as eks from "aws-cdk-lib/aws-eks"; + + +export class LogsBucket { + bucket: s3.Bucket; + constructor(scope: Construct, cluster: eks.Cluster, serviceAccountName?: string) { + this.bucket = new s3.Bucket(scope, "LogsBucket", { + removalPolicy: cdk.RemovalPolicy.DESTROY, + bucketName: "logs-bucket-eks-hs-2023-05-12", + }); + cluster.node.addDependency(this.bucket); + const ns = cluster.addManifest("logging-ns", { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": "logging" + } + }) + const sa = cluster.addServiceAccount("app-logs-s3-service-account", { + name: serviceAccountName, + namespace: "logging" + }); + sa.node.addDependency(ns); + this.bucket.grantReadWrite(sa); + new cdk.CfnOutput(scope, 'LogsS3Bucket', { value: this.bucket.bucketName }); + } +} From 6dd7916e5fd3af3162f3edfbb23aff172cd1570b Mon Sep 17 00:00:00 2001 From: Prasunna Soppa Date: Tue, 14 May 2024 12:36:18 +0530 Subject: [PATCH 2/5] update bucket name --- lib/aws/eks.ts | 1 + lib/aws/log_bucket.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/aws/eks.ts b/lib/aws/eks.ts index 4da0c1f..fc3fdd2 100644 --- a/lib/aws/eks.ts +++ b/lib/aws/eks.ts @@ -521,6 +521,7 @@ export class EksStack { application: { server: { secrets_manager: "aws_kms", + bucket_name: `logs-bucket-${process.env.CDK_DEFAULT_ACCOUNT}-${process.env.CDK_DEFAULT_REGION}`, serviceAccountAnnotations: { "eks.amazonaws.com/role-arn": hyperswitchServiceAccountRole.roleArn, }, diff --git a/lib/aws/log_bucket.ts b/lib/aws/log_bucket.ts index b6b5e05..252fb19 100644 --- a/lib/aws/log_bucket.ts +++ b/lib/aws/log_bucket.ts @@ -11,7 +11,7 @@ export class LogsBucket { constructor(scope: Construct, cluster: eks.Cluster, serviceAccountName?: string) { this.bucket = new s3.Bucket(scope, "LogsBucket", { removalPolicy: cdk.RemovalPolicy.DESTROY, - bucketName: "logs-bucket-eks-hs-2023-05-12", + bucketName: `logs-bucket-${process.env.CDK_DEFAULT_ACCOUNT}-${process.env.CDK_DEFAULT_REGION}`, }); cluster.node.addDependency(this.bucket); const ns = cluster.addManifest("logging-ns", { From fa5ee71d9c8ce5c01b2455ef533716a640ef7781 Mon Sep 17 00:00:00 2001 From: Prasunna Soppa Date: Wed, 15 May 2024 12:42:07 +0530 Subject: [PATCH 3/5] update session manager policy name --- lib/aws/stack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/aws/stack.ts b/lib/aws/stack.ts index 5a54ea2..12b65c2 100644 --- a/lib/aws/stack.ts +++ b/lib/aws/stack.ts @@ -226,7 +226,7 @@ export class AWSStack extends cdk.Stack { ] }); const ext_jump_policy = new iam.ManagedPolicy(this, 'SessionManagerPolicies', { - managedPolicyName: "SessionManagerPolicies", + managedPolicyName: `SessionManagerPolicies-${process.env.CDK_DEFAULT_ACCOUNT}-${process.env.CDK_DEFAULT_REGION}`, description: "SessionManagerPolicies", document: external_jump_policy }); From f5d1a635cf0b805dea3aadf6f925f9285b1ee222 Mon Sep 17 00:00:00 2001 From: Prasunna Soppa Date: Sun, 19 May 2024 16:19:17 +0530 Subject: [PATCH 4/5] add support to push logs to s3 --- lib/aws/log_bucket.ts | 105 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/lib/aws/log_bucket.ts b/lib/aws/log_bucket.ts index 252fb19..e5f32d1 100644 --- a/lib/aws/log_bucket.ts +++ b/lib/aws/log_bucket.ts @@ -14,7 +14,7 @@ export class LogsBucket { bucketName: `logs-bucket-${process.env.CDK_DEFAULT_ACCOUNT}-${process.env.CDK_DEFAULT_REGION}`, }); cluster.node.addDependency(this.bucket); - const ns = cluster.addManifest("logging-ns", { + const ns = cluster.addManifest("logging-ns", { "apiVersion": "v1", "kind": "Namespace", "metadata": { @@ -27,6 +27,109 @@ export class LogsBucket { }); sa.node.addDependency(ns); this.bucket.grantReadWrite(sa); + + const fluentdChart = cluster.addHelmChart("fluentd", { + chart: "fluentd", + repository: "https://fluent.github.io/helm-charts", + namespace: "logging", + wait: false, + values: { + kind: "DaemonSet", + serviceAccount: { + create: false, + name: sa.serviceAccountName + }, + fullnameOverride: "fluentd-s3", + variant: "s3", + labels: { + app: "fluentd-s3" + }, + resources: { + limits: { + cpu: "1", + memory: "1200Mi" + }, + requests: { + cpu: "200m", + memory: "150Mi" + } + }, + rbac: { + create: false + }, + livenessProbe: null, + readinessProbe: null, + service: { + enabled: false, + type: "ClusterIP", + }, + image: { + repository: "fluent/fluentd-kubernetes-daemonset", + pullPolicy: "IfNotPresent", + tag: "v1.16-debian-s3-1" + }, + env: [ + { + name: "S3_BUCKET", + value: this.bucket.bucketName, + }, + { + name: "S3_REGION", + value: process.env.CDK_DEFAULT_REGION, + } + + ], + terminationGracePeriodSeconds: 30, + dnsPolicy: "ClusterFirst", + restartPolicy: "Always", + schedulerName: "default-scheduler", + securityContext: {}, + fileConfigs:{ + "01_sources.conf":` + @type tail + @id in_tail_hyperswitch-server-router_logs + + path /var/log/containers/hyperswitch-*.log + pos_file /var/log/fluentd-hyperswitch-server-router-containers.log.pos + tag "hyperswitch.*" + read_from_head true + + @type regexp + expression /^(? + `, + "02_filters.conf":"", + "03_dispatch.conf":"", + "04_outputs.conf": ` + + @type json + + @type copy + + @type stdout + + + @type s3 + s3_bucket "#{ENV['S3_BUCKET']}" + s3_region "#{ENV['S3_REGION']}" + path "hyperswitch-logs/%Y/%m/%d/$\{tag\}/" + + @type file + path /var/log/fluent/s3 + timekey 3600 # 1 hour partition + timekey_wait 10m + timekey_zone +0530 + chunk_limit_size 256m + flush_at_shutdown + + + ` + + }, + } + + }); + new cdk.CfnOutput(scope, 'LogsS3Bucket', { value: this.bucket.bucketName }); } } From 37b077ca02380aa048efbb0c479baa494e45c706 Mon Sep 17 00:00:00 2001 From: Prasunna Soppa Date: Thu, 23 May 2024 21:00:08 +0530 Subject: [PATCH 5/5] code refactoring --- lib/aws/log_bucket.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/aws/log_bucket.ts b/lib/aws/log_bucket.ts index e5f32d1..d35af20 100644 --- a/lib/aws/log_bucket.ts +++ b/lib/aws/log_bucket.ts @@ -11,19 +11,19 @@ export class LogsBucket { constructor(scope: Construct, cluster: eks.Cluster, serviceAccountName?: string) { this.bucket = new s3.Bucket(scope, "LogsBucket", { removalPolicy: cdk.RemovalPolicy.DESTROY, - bucketName: `logs-bucket-${process.env.CDK_DEFAULT_ACCOUNT}-${process.env.CDK_DEFAULT_REGION}`, + bucketName: `application-logs-bucket-${process.env.CDK_DEFAULT_ACCOUNT}-${process.env.CDK_DEFAULT_REGION}`, }); cluster.node.addDependency(this.bucket); - const ns = cluster.addManifest("logging-ns", { + const ns = cluster.addManifest("kube-analytics-ns", { "apiVersion": "v1", "kind": "Namespace", "metadata": { - "name": "logging" + "name": "kube-analytics" } }) const sa = cluster.addServiceAccount("app-logs-s3-service-account", { name: serviceAccountName, - namespace: "logging" + namespace: "kube-analytics" }); sa.node.addDependency(ns); this.bucket.grantReadWrite(sa); @@ -31,7 +31,7 @@ export class LogsBucket { const fluentdChart = cluster.addHelmChart("fluentd", { chart: "fluentd", repository: "https://fluent.github.io/helm-charts", - namespace: "logging", + namespace: "kube-analytics", wait: false, values: { kind: "DaemonSet", @@ -129,7 +129,5 @@ export class LogsBucket { } }); - - new cdk.CfnOutput(scope, 'LogsS3Bucket', { value: this.bucket.bucketName }); } }