prom: cloudwatch_exporter

ruby-no-kai · Aug 31, 2022 · a5268cf · a5268cf
1 parent c724eeb
commit a5268cf
Show file tree

Hide file tree

Showing 6 changed files with 367 additions and 0 deletions.
diff --git a/tf/k8s-prom/cloudwatch-exporter.config/apne1.jsonnet b/tf/k8s-prom/cloudwatch-exporter.config/apne1.jsonnet
@@ -0,0 +1,172 @@
+local u = import './util.libsonnet';
+
+{
+  region: 'ap-northeast-1',
+  period_seconds: 300,
+  delay_seconds: 360,
+  metrics: (
+    u.product([
+      [
+        {
+          aws_namespace: 'AWS/NetworkELB',
+          aws_dimensions: ['LoadBalancer', 'AvailabilityZone'],
+        },
+      ],
+      u.product([
+        [
+          { aws_statistics: ['Average', 'Minimum', 'Maximum'] },
+        ],
+        std.map(
+          function(metric) { aws_metric_name: metric },
+          [
+            'ActiveFlowCount_TCP',
+            'ActiveFlowCount_TLS',
+            'ActiveFlowCount_UDP',
+          ]
+        ),
+      ]) +
+      u.product([
+        [
+          { aws_statistics: ['Sum'] },
+        ],
+        std.map(
+          function(metric) { aws_metric_name: metric },
+          [
+            'NewFlowCount_TCP',
+            'NewFlowCount_TLS',
+            'NewFlowCount_UDP',
+            'ProcessedBytes_TCP',
+            'ProcessedBytes_TLS',
+            'ProcessedBytes_UDP',
+            'PortAllocationErrorCount',
+          ],
+        ),
+      ]),
+    ]) +
+
+    u.product([
+      [
+        {
+          aws_namespace: 'AWS/EBS',
+          aws_dimensions: ['VolumeId'],
+        },
+      ],
+      [
+        { aws_statistics: ['Average', 'Minimum', 'Maximum'] },
+      ],
+      std.map(
+        function(metric) { aws_metric_name: metric },
+        [
+          'VolumeReadBytes',
+          'VolumeIdleTime',
+          'VolumeReadOps',
+          'BurstBalance',
+          'VolumeQueueLength',
+          'VolumeWriteBytes',
+          'VolumeWriteOps',
+          'VolumeTotalReadTime',
+          'VolumeTotalWriteTime',
+        ],
+      ),
+    ]) +
+
+    u.product([
+      [
+        {
+          aws_namespace: 'AWS/EC2',
+          aws_dimensions: ['InstanceId'],
+        },
+      ],
+      [
+        { aws_statistics: ['Average', 'Minimum', 'Maximum'] },
+      ],
+      std.map(
+        function(metric) { aws_metric_name: metric },
+        [
+          'CPUCreditBalance',
+          'CPUCreditUsage',
+          'CPUSurplusCreditBalance',
+          'CPUSurplusCreditsCharged',
+          'CPUUtilization',
+          'DiskReadOps',
+          'DiskWriteOps',
+          'NetworkIn',
+          'NetworkOut',
+          'NetworkPacketsIn',
+          'NetworkPacketsOut',
+          'StatusCheckFailed',
+          'StatusCheckFailed_Instance',
+        ],
+      ),
+    ]) +
+
+    u.product([
+      [
+        {
+          aws_namespace: 'AWS/NATGateway',
+          aws_dimensions: ['NatGatewayId'],
+          aws_statistics: ['Sum', 'Average', 'Minimum', 'Maximum'],
+        },
+      ],
+      std.map(
+        function(metric) { aws_metric_name: metric },
+        [
+          'ConnectionAttemptCount',
+          'IdleTimeoutCount',
+          'BytesInFromDestination',
+          'PacketsDropCount',
+          'ConnectionEstablishedCount',
+          'ErrorPortAllocation',
+          'BytesOutToSource',
+          'BytesInFromSource',
+          'BytesOutToDestination',
+          'ActiveConnectionCount',
+        ]
+      ),
+    ]) +
+
+    u.product([
+      [
+        {
+          aws_namespace: 'AWS/RDS',
+          aws_dimensions: ['DBInstanceIdentifier'],
+          aws_statistics: ['Sum', 'Average', 'Minimum', 'Maximum'],
+        },
+      ],
+      std.map(
+        function(metric) { aws_metric_name: metric },
+        [
+          'CPUCreditBalance',
+          'CPUCreditUsage',
+          'CPUUtilization',
+          'DatabaseConnections',
+          'DBLoad',
+          'DBLoadCPU',
+          'DBLoadNonCPU',
+          'Deadlocks',
+          'DiskQueueDepth',
+          'EngineUptime',
+          'FreeableMemory',
+          'FreeLocalStorage',
+          'FreeStorageSpace',
+          'NetworkReceiveThroughput',
+          'NetworkThroughput',
+          'NetworkTransmitThroughput',
+          'OldestReplicationSlotLag',
+          'RDSToAuroraPostgreSQLReplicaLag',
+          'ReadIOPS',
+          'ReadLatency',
+          'ReadThroughput',
+          'ReplicationSlotDiskUsage',
+          'SwapUsage',
+          'TransactionLogsDiskUsage',
+          'TransactionLogsGeneration',
+          'WriteIOPS',
+          'WriteLatency',
+          'WriteThroughput',
+        ]
+      ),
+    ])
+
+  ),
+}
diff --git a/tf/k8s-prom/cloudwatch-exporter.config/apne1hi.jsonnet b/tf/k8s-prom/cloudwatch-exporter.config/apne1hi.jsonnet
@@ -0,0 +1,48 @@
+local u = import './util.libsonnet';
+
+{
+  region: 'ap-northeast-1',
+  period_seconds: 60,
+  delay_seconds: 120,
+  metrics: (
+    u.product([
+      [
+        {
+          aws_namespace: 'AWS/ApplicationELB',
+          aws_dimensions: ['LoadBalancer', 'AvailabilityZone', 'TargetGroup'],
+        },
+      ],
+      u.product([
+        [
+          { aws_statistics: ['Average', 'Minimum', 'Maximum'] },
+          { aws_extended_statistics: ['p50', 'p95', 'p99'] },
+        ],
+        std.map(
+          function(metric) { aws_metric_name: metric },
+          [
+            'TargetResponseTime',
+          ],
+        ),
+      ]) +
+      u.product([
+        [
+          { aws_statistics: ['Sum'] },
+        ],
+        std.map(
+          function(metric) { aws_metric_name: metric },
+          [
+            'RequestCount',
+            'NewConnectionCount',
+            'HTTPCode_Target_5XX_Count',
+            'HTTPCode_Target_4XX_Count',
+            'HTTPCode_Target_3XX_Count',
+            'HTTPCode_Target_2XX_Count',
+            'HTTPCode_ELB_5XX_Count',
+            'HTTPCode_ELB_4XX_Count',
+          ],
+        ),
+
+      ]),
+    ])
+  ),
+}
diff --git a/tf/k8s-prom/cloudwatch-exporter.config/util.libsonnet b/tf/k8s-prom/cloudwatch-exporter.config/util.libsonnet
@@ -0,0 +1,10 @@
+{
+  map(a, f):: std.map(f, a),
+  flatMap(a, f):: std.map(f, a),
+
+  product2(xy, ys)::
+    std.flatMap(function(x) std.map(function(y) x + y, ys), xy),
+
+  product(xss)::
+    std.foldl($.product2, xss, [{}]),
+}
diff --git a/tf/k8s-prom/cloudwatch-exporter.tf b/tf/k8s-prom/cloudwatch-exporter.tf
@@ -0,0 +1,130 @@
+resource "helm_release" "cloudwatch-exporter-apne1" {
+  repository = "https://prometheus-community.github.io/helm-charts"
+  chart      = "prometheus-cloudwatch-exporter"
+  version    = "0.19.2"
+
+  name             = "cloudwatch-exporter-apne1"
+  namespace        = "monitoring"
+  create_namespace = true
+
+  values = [
+    data.external.cloudwatch-exporter-values.result.json,
+    jsonencode({
+      config = data.external.cloudwatch-exporter-config-apne1.result.json
+      serviceMonitor = {
+        enabled = true
+        interval = "120s"
+        timeout = "120s"
+        labels = {
+          release = helm_release.kube-prometheus-stack.name
+        }
+      }
+    })
+  ]
+}
+
+data "external" "cloudwatch-exporter-config-apne1" {
+  program = ["../jsonnet.rb"]
+
+  query = {
+    path = "./cloudwatch-exporter.config/apne1.jsonnet"
+  }
+}
+
+resource "helm_release" "cloudwatch-exporter-apne1hi" {
+  repository = "https://prometheus-community.github.io/helm-charts"
+  chart      = "prometheus-cloudwatch-exporter"
+  version    = "0.19.2"
+
+  name             = "cloudwatch-exporter-apne1hi"
+  namespace        = "monitoring"
+  create_namespace = true
+
+  values = [
+    data.external.cloudwatch-exporter-values.result.json,
+    jsonencode({
+      config = data.external.cloudwatch-exporter-config-apne1hi.result.json
+      serviceMonitor = {
+        enabled = true
+        interval = "60s"
+        timeout = "60s"
+        labels = {
+          release = helm_release.kube-prometheus-stack.name
+        }
+      }
+    })
+  ]
+}
+
+data "external" "cloudwatch-exporter-config-apne1hi" {
+  program = ["../jsonnet.rb"]
+
+  query = {
+    path = "./cloudwatch-exporter.config/apne1hi.jsonnet"
+  }
+}
+
+data "external" "cloudwatch-exporter-values" {
+  program = ["../jsonnet.rb"]
+
+  query = {
+    path = "./cloudwatch-exporter.values.jsonnet"
+  }
+}
+
+data "aws_iam_policy" "nocadmin-base" {
+  name = "NocAdminBase"
+}
+
+resource "aws_iam_role" "cloudwatch-exporter" {
+  name                 = "NwCloudWatchExporter"
+  description          = "k8s cloudwatch_exporter"
+  assume_role_policy   = data.aws_iam_policy_document.cloudwatch-exporter-trust.json
+  permissions_boundary = data.aws_iam_policy.nocadmin-base.arn
+}
+
+data "aws_iam_policy_document" "cloudwatch-exporter-trust" {
+  statement {
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+    effect  = "Allow"
+    principals {
+      type        = "Federated"
+      identifiers = [local.cluster_oidc_config.arn]
+    }
+    condition {
+      test     = "StringEquals"
+      variable = local.cluster_oidc_config.condition
+      values   = ["system:serviceaccount:monitoring:cloudwatch-exporter"]
+    }
+  }
+}
+
+resource "aws_iam_role_policy" "cloudwatch-exporter" {
+  role   = aws_iam_role.cloudwatch-exporter.name
+  name   = "cloudwatch-exporter"
+  policy = data.aws_iam_policy_document.cloudwatch-exporter-policy.json
+}
+
+data "aws_iam_policy_document" "cloudwatch-exporter-policy" {
+  statement {
+    actions = [
+      "cloudwatch:ListMetrics",
+      "cloudwatch:GetMetricStatistics",
+      "cloudwatch:GetMetricData",
+    ]
+    resources = ["*"]
+    effect  = "Allow"
+  }
+}
+
+resource "kubernetes_service_account" "cloudwatch-exporter" {
+  metadata {
+    name      = "cloudwatch-exporter"
+    namespace = "monitoring"
+    annotations = {
+      "eks.amazonaws.com/role-arn"               = aws_iam_role.cloudwatch-exporter.arn
+      "eks.amazonaws.com/sts-regional-endpoints" = true
+    }
+  }
+  automount_service_account_token = true
+}
diff --git a/tf/k8s-prom/cloudwatch-exporter.values.jsonnet b/tf/k8s-prom/cloudwatch-exporter.values.jsonnet
@@ -0,0 +1,6 @@
+{
+  serviceAccount: {
+    create: false,
+    name: 'cloudwatch-exporter',
+  },
+}
diff --git a/tf/k8s-prom/kubernetes.tf b/tf/k8s-prom/kubernetes.tf
@@ -1,5 +1,6 @@
 locals {
   cluster_config = data.terraform_remote_state.k8s.outputs.cluster_config
+  cluster_oidc_config = data.terraform_remote_state.k8s.outputs.cluster_oidc_config
 }
 
 provider "kubernetes" {