OM60 - POC to sending server metrics partner solution (#54)

* OM60 - POC to sending server metrics partner solution created support with OTel Collector to send server metrics to 1. NewRelic, 2. DataDog, 3. CloudWatch, and 4. Multiple partner solutions for each partner solution a docker-compose and corresponding otel-configuration file are provided we need to create required API keys in the partner-solution before running these files. * OM60 - incorporated feedback 1. added a README.md 2. cleaned up the docker-compose files 3. removed Aerospike CE container from compose file 4. updated the AS_HOST to docker internal host to access existing Aerospike cluster. 5. added longitude and latitude sample in the compose file [ "California" ] 6. removed ap-south-1 AWS region and marked for to-be-changed by user * OM60 - README corrections checking the indents * OM60 - updates to README indentation checks * OM60 - updated README indentation changes * OM60 - README updates removed unnecessary comments and standardized all sections
aerospike · Jun 19, 2023 · bf66aa4 · bf66aa4
1 parent 44dcecd
commit bf66aa4
Show file tree

Hide file tree

Showing 9 changed files with 411 additions and 0 deletions.
diff --git a/examples/otel/README.md b/examples/otel/README.md
@@ -0,0 +1,68 @@
+# OTel Aerospike Monitoring Stack
+
+The otel deploy Aerospike Monitoring Stack is a docker compose configuration
+which creates the following containers running on the same host as a single-node
+Aerospike cluster:
+1. aerospike-prometheus-exporter latest version
+1. otel collector contrib latest version
+
+A prerequisite is that you have a single-node cluster, with the service port
+accessible from port 3000 of the host where the monitoring stack is deployed.  If you
+do not already have such an Aerospike cluster, you can deploy a single-node cluster
+in a container using:
+
+```
+$ docker run -tid --name aerospike -p 3000:3000 -p 3001:3001 -p 3002:3002 aerospike/aerospike-server:latest
+```
+
+These examples cover integration of aerospike monitoring with 
+
+1. NewRelic
+1. Datadog
+1. AWS Cloudwatch
+
+> **Note**: You need to update the API Key or Access Key in the respective configuration files.
+
+## NewRelic 
+### modify newrelic-otel-collector-config.yml and update below key with respective values 
+###### <NEWRELIC-API-KEY> 
+### To start the stack 
+```
+$ docker-compose -f newrelic-docker-compose.yml up
+```
+## To stop the stack
+```
+$ docker-compose -f newrelic-docker-compose.yml down
+```
+Now simply point your browser at NewRelic cloud https://one.newrelic.com/data-explorer to see Aerospike metrics.
+
+## Datadog
+### modify datadog-otel-collector-config.yml and update below key with respective values 
+###### <DATADOG-APP-KEY> 
+### To start the stack 
+```
+$ docker-compose -f datadog-docker-compose.yml up
+```
+
+## To stop the stack
+```
+$ docker-compose -f datadog-docker-compose.yml down
+```
+Now simply point your browser at Datadog cloud https://app.datadoghq.com/metric/explorer to see Aerospike metrics.
+
+## Cloudwatch
+### modify cloudwatch-docker-compose.yml and update below keys with respective values 
+###### AWS_REGION=<AWS_REGION_LOCATION>
+###### AWS_ACCESS_KEY_ID=<MENTION-YOUR-AWS-CLOUD-WATCH-KEY>
+###### AWS_SECRET_ACCESS_KEY=<MENTION-YOUR-AWS-CW-SECRET-ACCESS-KEY>
+### To start the stack 
+```
+$ docker-compose -f cloudwatch-docker-compose.yml up
+```
+
+Now simply point your browser at AWS Cloudwatch to see Aerospike metrics.
+
+## To stop the stack
+```
+$ docker-compose -f cloudwatch-docker-compose.yml down
+```
diff --git a/examples/otel/cloudwatch-docker-compose.yml b/examples/otel/cloudwatch-docker-compose.yml
@@ -0,0 +1,39 @@
+version: "3.7"
+services:
+
+  exporter:
+    image: aerospike/aerospike-prometheus-exporter:latest
+    networks:
+      - aerospike_otel_nw
+    container_name: as_prom_exporter
+    environment:
+    - AS_HOST=host.docker.internal
+    - AS_PORT=3000
+    - "METRIC_LABELS=type='development',source='aerospike', latitude='36.778259', longitude='-119.417931' "
+    extra_hosts:
+    - "host.docker.internal:host-gateway"
+
+  # Collector
+  otel-collector:
+    networks:
+      - aerospike_otel_nw
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: as_otel_contrib
+    privileged: true
+    restart: always
+    environment:
+    - AWS_REGION=<AWS_REGION_LOCATION>
+    - AWS_ACCESS_KEY_ID=<MENTION-YOUR-AWS-CLOUD-WATCH-KEY>
+    - AWS_SECRET_ACCESS_KEY=<MENTION-YOUR-AWS-CW-SECRET-ACCESS-KEY>
+    volumes:
+    - source: ./cloudwatch-otel-collector-config.yml 
+      target: /etc/otel-collector-config.yml 
+      type: bind
+    command: ["--config=/etc/otel-collector-config.yml"]
+
+volumes:
+  otel_prometheus_data: {}
+  otel_data: {}
+networks:
+  aerospike_otel_nw:
+    driver: bridge
diff --git a/examples/otel/cloudwatch-otel-collector-config.yml b/examples/otel/cloudwatch-otel-collector-config.yml
@@ -0,0 +1,50 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "as_otel_contrib:4317"
+
+  prometheus:
+    config:
+      scrape_configs:
+      - job_name: 'otel_as_demo_server'
+        scrape_interval: 10s
+        static_configs:
+          - targets: ['as_prom_exporter:9145']
+
+exporters:
+  prometheus:
+    endpoint: "as_otel_contrib:30145"
+
+  awsemf:
+    namespace: "otel_dc_as_test"
+    region: "ap-south-1"
+    output_destination: cloudwatch
+    dimension_rollup_option: ZeroAndSingleDimensionRollup
+    detailed_metrics: true
+    retain_initial_value_of_delta_metric: true
+    metric_declarations:
+      - dimensions: [[ns,cluster_name,service,file, file_index,set,module,job_type,trid,sindex_name,sindex,user,dc], []]
+        metric_name_selectors:
+         - "^aerospike"        
+
+  logging:
+
+processors:
+  batch:
+
+extensions:
+  health_check:
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+
+service:
+  extensions: [pprof, zpages, health_check]
+  pipelines:
+    metrics:
+      receivers: [prometheus ]
+      processors: [batch]
+      exporters: [logging , prometheus, awsemf]
+
diff --git a/examples/otel/datadog-docker-compose.yml b/examples/otel/datadog-docker-compose.yml
@@ -0,0 +1,35 @@
+version: "3.7"
+services:
+
+  exporter:
+    image: aerospike/aerospike-prometheus-exporter:latest
+    networks:
+      - aerospike_otel_nw
+    container_name: as_prom_exporter
+    environment:
+    - AS_HOST=host.docker.internal
+    - AS_PORT=3000
+    - "METRIC_LABELS=type='development',source='aerospike', latitude='36.778259', longitude='-119.417931' "
+    extra_hosts:
+    - "host.docker.internal:host-gateway"
+
+  # Collector
+  otel-collector:
+    networks:
+      - aerospike_otel_nw
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: as_otel_contrib
+    privileged: true
+    restart: always
+    volumes:
+    - source: ./datadog-otel-collector-config.yml 
+      target: /etc/otel-collector-config.yml 
+      type: bind
+    command: ["--config=/etc/otel-collector-config.yml"]
+
+volumes:
+  otel_prometheus_data: {}
+  otel_data: {}
+networks:
+  aerospike_otel_nw:
+    driver: bridge
diff --git a/examples/otel/datadog-otel-collector-config.yml b/examples/otel/datadog-otel-collector-config.yml
@@ -0,0 +1,50 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "as_otel_contrib:4317"
+
+  prometheus:
+    config:
+      scrape_configs:
+      - job_name: 'otel_as_demos'
+        scrape_interval: 10s
+        static_configs:
+          - targets: ['as_prom_exporter:9145']
+
+exporters:
+  prometheus:
+    endpoint: "0.0.0.0:30145"
+
+  datadog:
+    api: 
+      site: datadoghq.com
+      key: <DATADOG-APP-KEY>
+
+  logging:
+
+processors:
+  batch:
+        send_batch_max_size: 100
+        send_batch_size: 10
+        timeout: 10s
+
+extensions:
+  health_check:
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+
+service:
+  extensions: [pprof, zpages, health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging, datadog ]
+    metrics:
+      receivers: [prometheus ]
+      processors: [batch]
+      exporters: [logging , prometheus, datadog]
+
diff --git a/examples/otel/multi-docker-compose.yml b/examples/otel/multi-docker-compose.yml
@@ -0,0 +1,35 @@
+version: "3.7"
+services:
+
+  exporter:
+    image: aerospike/aerospike-prometheus-exporter:latest
+    networks:
+      - aerospike_otel_nw
+    container_name: as_prom_exporter
+    environment:
+    - AS_HOST=host.docker.internal
+    - AS_PORT=3000
+    - "METRIC_LABELS=type='development',source='aerospike', latitude='36.778259', longitude='-119.417931' "
+    extra_hosts:
+    - "host.docker.internal:host-gateway"
+
+  # Collector
+  otel-collector:
+    networks:
+      - aerospike_otel_nw
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: as_otel_contrib
+    privileged: true
+    restart: always
+    volumes:
+    - source: ./multi-otel-collector-config.yml 
+      target: /etc/otel-collector-config.yml 
+      type: bind
+    command: ["--config=/etc/otel-collector-config.yml"]
+
+volumes:
+  otel_prometheus_data: {}
+  otel_data: {}
+networks:
+  aerospike_otel_nw:
+    driver: bridge
diff --git a/examples/otel/multi-otel-collector-config.yml b/examples/otel/multi-otel-collector-config.yml
@@ -0,0 +1,52 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "as_otel_contrib:4317"
+
+  prometheus:
+    config:
+      scrape_configs:
+      - job_name: 'otel_as_demos'
+        scrape_interval: 10s
+        static_configs:
+          - targets: ['as_prom_exporter:9145']
+
+exporters:
+  prometheus:
+    endpoint: "0.0.0.0:30145"
+
+  datadog:
+    api: 
+      site: datadoghq.com
+      key: <DATADOG-APP-KEY>
+
+  otlp:
+    endpoint: https://otlp.nr-data.net:4317 
+    headers:
+      api-key: <NEWRELIC-API-KEY>
+
+  logging:
+
+processors:
+  batch:
+
+extensions:
+  health_check:
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+
+service:
+  extensions: [pprof, zpages, health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp, logging ]
+    metrics:
+      receivers: [prometheus ]
+      processors: [batch]
+      exporters: [otlp,logging , prometheus, datadog]
+
diff --git a/examples/otel/newrelic-docker-compose.yml b/examples/otel/newrelic-docker-compose.yml
@@ -0,0 +1,35 @@
+version: "3.7"
+services:
+
+  exporter:
+    image: aerospike/aerospike-prometheus-exporter:latest
+    networks:
+      - aerospike_otel_nw
+    container_name: as_prom_exporter
+    environment:
+    - AS_HOST=host.docker.internal
+    - AS_PORT=3000
+    - "METRIC_LABELS=type='development',source='aerospike', latitude='36.778259', longitude='-119.417931' "
+    extra_hosts:
+    - "host.docker.internal:host-gateway"
+
+  # Collector
+  otel-collector:
+    networks:
+      - aerospike_otel_nw
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: as_otel_contrib
+    privileged: true
+    restart: always
+    volumes:
+    - source: ./newrelic-otel-collector-config.yml 
+      target: /etc/otel-collector-config.yml 
+      type: bind
+    command: ["--config=/etc/otel-collector-config.yml"]
+
+volumes:
+  otel_prometheus_data: {}
+  otel_data: {}
+networks:
+  aerospike_otel_nw:
+    driver: bridge