From b4549b700f0c355cd0906f6e1e24792800f381a2 Mon Sep 17 00:00:00 2001
From: heydbut <hey.danielbutler@gmail.com>
Date: Mon, 2 Oct 2023 13:48:50 -0400
Subject: [PATCH] Add tool to prometheus endpoint scrape to csv tool

---
 collector/tidb/dashboards/overview/main.tf | 314 +++++++++++++++++++++
 tools/python/prometheus-scrape2csv.py      |  60 ++++
 tools/python/requirements.txt              |   2 +
 3 files changed, 376 insertions(+)
 create mode 100644 collector/tidb/dashboards/overview/main.tf
 create mode 100644 tools/python/prometheus-scrape2csv.py

diff --git a/collector/tidb/dashboards/overview/main.tf b/collector/tidb/dashboards/overview/main.tf
new file mode 100644
index 0000000..be3ecc2
--- /dev/null
+++ b/collector/tidb/dashboards/overview/main.tf
@@ -0,0 +1,314 @@
+terraform {
+  required_providers {
+    lightstep = {
+      source  = "lightstep/lightstep"
+      version = "~> 1.70.10"
+    }
+  }
+  required_version = ">= v1.0.11"
+}
+
+variable "lightstep_project" {
+  description = "Cloud Observability Project Name"
+  type        = string
+}
+
+output "dashboard_url" {
+  value       = "https://app.lightstep.com/${var.lightstep_project}/dashboard/${lightstep_dashboard.collector_pulsar_overview.id}"
+  description = "OpenTelemetry Collector Pulsar Dashboard URL"
+}
+
+resource "lightstep_dashboard" "collector_pulsar_overview" {
+  project_name          = var.lightstep_project
+  dashboard_name        = "OpenTelemetry Pulsar Dashboard"
+  dashboard_description = "Monitor Pulsar and metrics with this summary dashboard."
+
+  chart {
+    name = "CPU Time Spent"
+    rank = "0"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric process_cpu_seconds_total | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Resident Memory Size"
+    rank = "1"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric process_resident_memory_bytes | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Virtual Memory Size"
+    rank = "2"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric process_virtual_memory_bytes | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Process Max FDS"
+    rank = "2"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric process_max_fds | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Process Open FDS"
+    rank = "2"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric process_open_fds | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "UP"
+    rank = "2"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric up | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Active Connections"
+    rank = "10"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_active_connections | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Broker Message Backlog"
+    rank = "10"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_broker_msg_backlog | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Broker Incoming Message Rate"
+    rank = "11"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_broker_rate_in | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Broker Outgoing Message Rate"
+    rank = "12"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_broker_rate_out | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Broker Storage Size"
+    rank = "13"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_broker_storage_size | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Broker Topics Count"
+    rank = "14"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_broker_topics_count | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Message Backlog Across Topics"
+    rank = "15"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_msg_backlog | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Producers Count"
+    rank = "16"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_producers_count | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Consumers Count"
+    rank = "17"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_consumers_count | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Total Incoming Bytes"
+    rank = "18"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_in_bytes_total | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Total Outgoing Bytes"
+    rank = "19"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_out_bytes_total | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Storage Read Rate"
+    rank = "20"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_storage_read_rate | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Storage Write Rate"
+    rank = "21"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_storage_write_rate | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Total Storage Size"
+    rank = "22"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_storage_size | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Total Storage Size"
+    rank = "22"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_storage_logical_size | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Subscription Backlog"
+    rank = "30"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_subscription_back_log | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Subscription Consumers Count"
+    rank = "31"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_subscription_consumers_count | rate | group_by [], sum"
+    }
+  }
+
+  chart {
+    name = "Subscription Message Rate Out"
+    rank = "32"
+    type = "timeseries"
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = "metric pulsar_subscription_msg_rate_out | rate | group_by [], sum"
+    }
+  }
+
+}
diff --git a/tools/python/prometheus-scrape2csv.py b/tools/python/prometheus-scrape2csv.py
new file mode 100644
index 0000000..070adff
--- /dev/null
+++ b/tools/python/prometheus-scrape2csv.py
@@ -0,0 +1,60 @@
+"""
+Generate CSV file by scraping Prometheus metrics endpoint
+"""
+import requests
+import csv
+from prometheus_client import parser
+
+
+def parse_prometheus_metrics(metrics_data: str):
+    metrics = parser.text_string_to_metric_families(metrics_data)
+    parsed_metrics = {}
+    for metric in metrics:
+        metric_name = metric.name
+        metric_description = metric.documentation
+        metric_type = metric.type
+        for sample in metric.samples:
+            metric_labels = sample.labels
+            if metric_name not in parsed_metrics:
+                parsed_metrics[metric_name] = {
+                    "Name": metric_name,
+                    "Description": metric_description,
+                    "Unit": "",
+                    "DataType": metric_type,
+                    "Attributes": ",".join(metric_labels.keys())
+                }
+    return parsed_metrics.values()
+
+
+def fetch_prometheus_metrics(url):
+    response = requests.get(url)
+    if response.status_code == 200:
+        return response.text
+    else:
+        print(f"Failed to fetch data from {url}")
+        return None
+
+
+def write_metrics_to_csv(metrics_list, csv_filename):
+    with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
+        csv_writer = csv.writer(csvfile, quoting=csv.QUOTE_MINIMAL)
+        csv_writer.writerow(["Name", "Description", "Unit", "DataType", "Attributes"])
+        for metric in metrics_list:
+            csv_writer.writerow(
+                [metric["Name"], metric["Description"], metric["Unit"], metric["DataType"], metric["Attributes"]])
+
+
+# Example usage
+if __name__ == "__main__":
+    try:
+        default_prometheus_url = "http://localhost:9091/metrics"
+        prometheus_url = input("Enter Prometheus URL (default: %s): " % default_prometheus_url) or default_prometheus_url
+        md = fetch_prometheus_metrics(prometheus_url)
+        if md:
+            metrics_list = parse_prometheus_metrics(md)
+            write_metrics_to_csv(metrics_list, "metrics.csv")
+            print("Metrics data written to metrics.csv")
+        else:
+            print("Failed to fetch Prometheus metrics.")
+    except KeyboardInterrupt:
+        print("\nExiting...")
diff --git a/tools/python/requirements.txt b/tools/python/requirements.txt
index c94dc6c..2b32f35 100644
--- a/tools/python/requirements.txt
+++ b/tools/python/requirements.txt
@@ -1 +1,3 @@
 docker==6.*
+prometheus-client==0.17.1
+requests==2.31.0