diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 4aa115ad0e..70dbf9a608 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -14,9 +14,10 @@ RUN apk add --no-cache --update --virtual .build-deps sudo build-base ruby-dev \
&& gem install snappy
RUN gem install fluent-plugin-sumologic_output \
- && gem install fluent-plugin-rewrite-tag-filter \
&& gem install fluent-plugin-carbon-v2 \
+ && gem install fluent-plugin-prometheus-format \
&& gem install fluent-plugin-datapoint \
+ && gem install fluent-plugin-rewrite-tag-filter \
&& gem install fluent-plugin-protobuf
RUN gem sources --clear-all \
diff --git a/deploy/docker/Dockerfile-debian b/deploy/docker/Dockerfile-debian
index ba52951592..55aa1bfbf1 100644
--- a/deploy/docker/Dockerfile-debian
+++ b/deploy/docker/Dockerfile-debian
@@ -19,6 +19,8 @@ RUN sudo gem install snappy
RUN sudo gem install fluent-plugin-sumologic_output \
&& sudo gem install fluent-plugin-carbon-v2 \
&& sudo gem install fluent-plugin-datapoint \
+ && sudo gem install fluent-plugin-rewrite-tag-filter \
+ && sudo gem install fluent-plugin-prometheus-format \
&& sudo gem install fluent-plugin-probuf
RUN sudo gem sources --clear-all \
diff --git a/deploy/docker/fluent.conf b/deploy/docker/fluent.conf
index cc70744c0e..54f759f1ff 100644
--- a/deploy/docker/fluent.conf
+++ b/deploy/docker/fluent.conf
@@ -4,6 +4,22 @@
tag "dummy"
dummy {"hello":"world"}
+
+ @type http
+ port 9888
+
+ @type protobuf
+
+
+
+ @type datapoint
+
+
+ @type carbon_v2
+
+
+ @type prometheus_format
+
@type stdout
\ No newline at end of file
diff --git a/deploy/test/test_docker.rb b/deploy/test/test_docker.rb
index e31f1a8478..e6b7dfeaff 100644
--- a/deploy/test/test_docker.rb
+++ b/deploy/test/test_docker.rb
@@ -1,9 +1,8 @@
require 'test/unit'
class TestDocker < Test::Unit::TestCase
- DOCKER_TAG = 'sumologic/kubernetes-fluentd'.freeze
CONTAINER_NAME = 'test-container'.freeze
- DUMMY_OUTPUT = '"hello":"world"'.freeze
+ DUMMY_OUTPUT = 'dummy: {"hello":"world"}'.freeze
def setup
system("docker rm -f #{CONTAINER_NAME}", out: File::NULL, err: File::NULL)
@@ -13,23 +12,27 @@ def teardown
system("docker rm -f #{CONTAINER_NAME}", out: File::NULL, err: File::NULL)
end
+ def docker_tag
+ ENV['DOCKER_TAG'].nil? ? 'sumologic/kubernetes-fluentd' : ENV['DOCKER_TAG']
+ end
+
def test_docker_image_exist
result = `docker images`
- assert result.include?(DOCKER_TAG)
+ assert result.include?(docker_tag)
end
def test_docker_image_runnable
- id = `docker run -d --rm --name #{CONTAINER_NAME} #{DOCKER_TAG}:latest`
+ id = `docker run -d --rm --name #{CONTAINER_NAME} #{docker_tag}:latest`
assert !id.nil? && !id.empty?
- [1..10].each do |i|
- sleep 1
- result = `docker ps --filter "name=#{CONTAINER_NAME}"`
- assert(
- result.include?(CONTAINER_NAME),
- "container stopped after #{i} seconds"
- )
- end
+ sleep_time = 15
+ sleep sleep_time
+ result = `docker ps --filter "name=#{CONTAINER_NAME}"`
+ assert(
+ result.include?(CONTAINER_NAME),
+ "container stopped after #{sleep_time} seconds"
+ )
logs = `docker logs #{CONTAINER_NAME}`
+ puts logs
assert logs.include?(DUMMY_OUTPUT)
end
end
diff --git a/fluent-plugin-prometheus-format/Gemfile b/fluent-plugin-prometheus-format/Gemfile
new file mode 100644
index 0000000000..b4e2a20bb6
--- /dev/null
+++ b/fluent-plugin-prometheus-format/Gemfile
@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec
diff --git a/fluent-plugin-prometheus-format/README.md b/fluent-plugin-prometheus-format/README.md
new file mode 100644
index 0000000000..f2b31a9a67
--- /dev/null
+++ b/fluent-plugin-prometheus-format/README.md
@@ -0,0 +1,103 @@
+# fluent-plugin-prometheus-format
+
+[Fluentd](https://fluentd.org/) filter plugin to transform data points to prometheus format.
+
+- All fields of the metric will be serialized as intrinsic tags (dimensions) in the prometheus format.
+- Sample of Input
+
+```json
+{
+ "endpoint": "http-metrics",
+ "handler": "prometheus",
+ "instance": "172.20.36.191:10251",
+ "job": "kube-scheduler",
+ "namespace": "kube-system",
+ "kubernetes": {
+ "pod": {
+ "name": "kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal"
+ },
+ "service": {
+ "name": "kube-scheduler"
+ }
+ },
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-scheduler",
+ "@metric": "http_request_size_bytes_sum",
+ "@timestamp": 1550862304339,
+ "@value": 1619905.0
+}
+```
+
+- Sample of Output
+
+```json
+{
+ "message": "http_request_size_bytes_sum{endpoint=\"http-metrics\",handler=\"prometheus\",instance=\"172.20.36.191:10251\",job=\"kube-scheduler\",kubernetes.pod.name=\"kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal\",kubernetes.service.name=\"kube-scheduler\",namespace=\"kube-system\",prometheus=\"monitoring/prometheus-operator-prometheus\",prometheus_replica=\"prometheus-prometheus-operator-prometheus-0\",service=\"prometheus-operator-kube-scheduler\",_origin=\"kubernetes\"} 1619905.0 1550862304339"
+}
+```
+
+## Installation
+
+### RubyGems
+
+```sh
+gem install fluent-plugin-prometheus-format
+```
+
+### Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "fluent-plugin-prometheus-format"
+```
+
+And then execute:
+
+```sh
+bundle
+```
+
+## Configuration
+
+### relabel (hash) (optional)
+
+Relabel the field name in the input record.
+For every (`key`, `value`) pair in the hash, the field named with `key` in the input record will be relabeled to `value`.
+If `key` is not a field in the input record, it will be ignored.
+If `value` is an empty string, the field will be removed from the record.
+
+Default value: `{}`.
+
+### inclusions (hash) (optional)
+
+Whitelist of the records with regular expression matching on the field(s).
+For __all__ (`key`, `value`) pairs in the hash, only following records will be included in the output:
+
+- the value of field named with `key` matches the `value` (as regular expression).
+
+Default value: `{}`.
+
+### strict_inclusions (bool) (optional)
+
+If `true`, records missing any field in keys of `inclusions` will be dropped.
+
+Default value: `false`
+
+### exclusions (hash) (optional)
+
+Blacklist of the records with regular expression matching on the field(s).
+For __any__ (`key`, `value`) pair in the hash, following records will be excluded in the output:
+
+- the value of field named with `key` matches the `value` (as regular expression).
+
+Default value: `{}`.
+
+### strict_exclusions (bool) (optional)
+
+If `true`, records missing any field in keys of `exclusions` will be dropped.
+
+Default value: `false`
+
+__NOTE__ inclusions/exclusions rules are applied after relabeling and flatten.
diff --git a/fluent-plugin-prometheus-format/Rakefile b/fluent-plugin-prometheus-format/Rakefile
new file mode 100644
index 0000000000..9756c6eeb6
--- /dev/null
+++ b/fluent-plugin-prometheus-format/Rakefile
@@ -0,0 +1,13 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+ t.libs.push("lib", "test")
+ t.test_files = FileList["test/**/test_*.rb"]
+ t.verbose = true
+ t.warning = true
+end
+
+task default: [:test]
diff --git a/fluent-plugin-prometheus-format/fluent-plugin-prometheus-format.gemspec b/fluent-plugin-prometheus-format/fluent-plugin-prometheus-format.gemspec
new file mode 100644
index 0000000000..6f41456932
--- /dev/null
+++ b/fluent-plugin-prometheus-format/fluent-plugin-prometheus-format.gemspec
@@ -0,0 +1,26 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+ spec.name = "fluent-plugin-prometheus-format"
+ spec.version = "0.0.0"
+ spec.authors = ["Sumo Logic"]
+ spec.email = ["collection@sumologic.com"]
+
+ spec.summary = %q{Fluentd plugin for transfer data points to prometheus metrics format.}
+ spec.homepage = "https://github.com/SumoLogic/sumologic-kubernetes-collection"
+ spec.license = "Apache-2.0"
+
+ test_files, files = `git ls-files -z`.split("\x0").partition do |f|
+ f.match(%r{^(test|spec|features)/})
+ end
+ spec.files = files
+ spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
+ spec.test_files = test_files
+ spec.require_paths = ["lib"]
+
+ spec.add_development_dependency "bundler", "~> 2.0"
+ spec.add_development_dependency "rake", "~> 12.0"
+ spec.add_development_dependency "test-unit", "~> 3.0"
+ spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+end
diff --git a/fluent-plugin-prometheus-format/lib/fluent/plugin/filter_prometheus_format.rb b/fluent-plugin-prometheus-format/lib/fluent/plugin/filter_prometheus_format.rb
new file mode 100644
index 0000000000..1fc250f8d8
--- /dev/null
+++ b/fluent-plugin-prometheus-format/lib/fluent/plugin/filter_prometheus_format.rb
@@ -0,0 +1,161 @@
+require 'fluent/plugin/filter'
+
+module Fluent
+ module Plugin
+ # fluentd plugin for convert data point json to Prometheus format
+ class PrometheusFormatFilter < Fluent::Plugin::Filter
+ Fluent::Plugin.register_filter('prometheus_format', self)
+
+ helpers :record_accessor
+
+ config_param(
+ :relabel,
+ :hash,
+ default: {},
+ symbolize_keys: false,
+ value_type: :string
+ )
+ config_param(
+ :inclusions,
+ :hash,
+ default: {},
+ symbolize_keys: false,
+ value_type: :string
+ )
+ config_param(
+ :exclusions,
+ :hash,
+ default: {},
+ symbolize_keys: false,
+ value_type: :string
+ )
+ config_param :strict_inclusions, :bool, default: false
+ config_param :strict_exclusions, :bool, default: false
+ config_param :sort_labels, :bool, default: true
+
+ def configure(conf)
+ super
+ @inclusions.each do |key, value|
+ @inclusions[key] = Regexp.new(value)
+ end
+ @exclusions.each do |key, value|
+ @exclusions[key] = Regexp.new(value)
+ end
+ @metric_accessor = record_accessor_create("$.#{KEY_METRIC}")
+ @timestamp_accessor = record_accessor_create("$.#{KEY_TIMESTAMP}")
+ @value_accessor = record_accessor_create("$.#{KEY_VALUE}")
+ end
+
+ def filter(_tag, _time, record)
+ dotified = dotify_record(record)
+ relabeled = relabel_record(dotified)
+ wrap(to_prometheus_line(relabeled)) if valid?(relabeled)
+ end
+
+ private
+
+ KEY_METRIC = '@metric'.freeze
+ KEY_TIMESTAMP = '@timestamp'.freeze
+ KEY_VALUE = '@value'.freeze
+ KEY_MESSAGE = 'message'.freeze
+ SPLITOR = '.'.freeze
+
+ ORIGIN_KEY = '_origin'.freeze
+ ORIGIN_VALUE = 'kubernetes'.freeze
+
+ def to_prometheus_line(record)
+ metric = @metric_accessor.call(record)
+ timestamp = @timestamp_accessor.call(record)
+ value = @value_accessor.call(record)
+ "#{metric}\{#{to_tags(record)},#{ORIGIN_KEY}=\"#{ORIGIN_VALUE}\"\} #{value} #{timestamp}"
+ end
+
+ def valid?(record)
+ validate_inclusions(record) && validate_exclusions(record)
+ end
+
+ def validate_inclusions(record)
+ @inclusions.each do |key, regex|
+ return false unless validate_inclusion(key, regex, record)
+ end
+ true
+ end
+
+ def validate_inclusion(key, regex, record)
+ if record.key?(key)
+ !regex.match(record[key]).nil?
+ else
+ !@strict_inclusions
+ end
+ end
+
+ def validate_exclusions(record)
+ @exclusions.each do |key, regex|
+ return false unless validate_exclusion(key, regex, record)
+ end
+ true
+ end
+
+ def validate_exclusion(key, regex, record)
+ if record.key?(key)
+ regex.match(record[key]).nil?
+ else
+ !@strict_exclusions
+ end
+ end
+
+ def relabel_record(record)
+ @relabel.flat_map do |org_key, new_key|
+ if record.key?(org_key)
+ value = record[org_key]
+ record.delete(org_key)
+ record[new_key] = value unless new_key.nil? || new_key.empty?
+ end
+ end
+ record
+ end
+
+ def to_tags(hash)
+ array = @sort_labels ? hash.sort : hash.to_a
+ array.map do |key, value|
+ "#{key}=\"#{escape(value)}\""\
+ unless [KEY_METRIC, KEY_TIMESTAMP, KEY_VALUE].include?(key)
+ end.compact.join(',')
+ end
+
+ def escape(value)
+ value.gsub(/\\/) { '\\\\' }.gsub(/\n/) { '\\n' }.gsub(/\"/) { '\\"' }
+ end
+
+ def dotify_record(record)
+ dotified = {}
+ record.keys.each do |field|
+ value = record[field]
+ dotify(dotified, field, value, nil)
+ end
+ dotified
+ end
+
+ def dotify(hash, key, value, prefix)
+ pk = prefix ? "#{prefix}#{SPLITOR}#{key}" : key.to_s
+ if value.is_a?(Hash)
+ value.each do |k, v|
+ dotify(hash, k, v, pk)
+ end
+ elsif value.is_a?(Array)
+ value.each_with_index.each do |v, i|
+ dotify(hash, i.to_s, v, pk)
+ end
+ else # all non-container types
+ hash[pk] = value
+ end
+ end
+
+ def wrap(message)
+ record = {}
+ record[KEY_MESSAGE] = message
+ record
+ end
+ end
+ end
+end
diff --git a/fluent-plugin-prometheus-format/test/helper.rb b/fluent-plugin-prometheus-format/test/helper.rb
new file mode 100644
index 0000000000..46cfa10c95
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/helper.rb
@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/filter"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)
diff --git a/fluent-plugin-prometheus-format/test/plugin/test_filter_prometheus_format.rb b/fluent-plugin-prometheus-format/test/plugin/test_filter_prometheus_format.rb
new file mode 100644
index 0000000000..d466ae4d2b
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/plugin/test_filter_prometheus_format.rb
@@ -0,0 +1,544 @@
+require 'helper'
+require 'fluent/plugin/filter_prometheus_format.rb'
+
+class PrometheusFormatFilterTest < Test::Unit::TestCase
+ setup do
+ Fluent::Test.setup
+ end
+
+ sub_test_case 'trasform datapoint to prometheus format' do
+ test 'transform single data point' do
+ config = %([])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'relabel keys' do
+ config = %([
+ relabel {
+ "service" : "prometheus.service",
+ "prometheus_replica" : "prometheus.replica"
+ }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.relabel'
+ end
+
+ test 'ignore keys in relabel but not in record' do
+ config = %([
+ relabel {
+ "service" : "prometheus.service",
+ "prometheus_replica" : "prometheus.replica",
+ "some_key" : "some_value",
+ "foo" : ""
+ }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.relabel'
+ end
+
+ test 'remove keys if relabel to empty string' do
+ config = %([
+ relabel { "service" : "", "prometheus_replica" : ""}
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.relabel.remove'
+ end
+ end
+
+ sub_test_case 'trasform datapoint with nested json' do
+ test 'transform single data point' do
+ config = %([])
+ outputs = filter_datapoints(config, 'datapoint.nested')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.nested'
+ end
+
+ test 'relabel keys after flatten' do
+ config = %([
+ relabel {
+ "service" : "",
+ "kubernetes.service.name" : "service_name",
+ "kubernetes.pod.name" : "pod_name"
+ }
+ ])
+ outputs = filter_datapoints(config, 'datapoint.nested')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.nested.relabel'
+ end
+
+ test 'relabel keys with space' do
+ config = %([
+ relabel {
+ "service" : "",
+ "kubernetes.service.na e" : "service_na e",
+ "kubernetes.pod.na e" : "pod_na e"
+ }
+ ])
+ outputs = filter_datapoints(config, 'datapoint.nested.spaces')
+ puts outputs
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.nested.spaces.relabel'
+ end
+
+ test 'transform data point with escaped sequences' do
+ config = %([])
+ outputs = filter_datapoints(config, 'datapoint.nested.escape')
+ puts outputs
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.nested.escape'
+ end
+ end
+
+ sub_test_case 'inclusions, non-strict mode' do
+ test 'do include point with tag value matching regex' do
+ config = %([
+ inclusions { "namespace" : "kube-system" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do not include point with tag value not matching regex' do
+ config = %([
+ inclusions { "namespace" : "cube-system" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do include point without key in inclusions' do
+ config = %([
+ inclusions { "namespaceX" : "XXX" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do include point with all tags value matching regexs' do
+ config = %([
+ inclusions { "namespace" : "kube-system", "instance" : "^172.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do not include point with any tag value not matching regex' do
+ config = %([
+ inclusions { "namespace" : "kube-system", "instance" : "^196.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do include point with tag key not in the record' do
+ config = %([
+ inclusions { "namespaceX" : "XXXX", "instance" : "^172.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do not include point with existing tag value not matching regex' do
+ config = %([
+ inclusions { "namespaceX" : "XXXX", "instance" : "^196.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+ end
+
+ sub_test_case 'inclusions, strict mode' do
+ test 'do include point with tag value matching regex' do
+ config = %([
+ inclusions { "namespace" : "kube-system" }
+ strict_inclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do not include point with tag value not matching regex' do
+ config = %([
+ inclusions { "namespace" : "cube-system" }
+ strict_inclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do not include point without key in inclusions' do
+ config = %([
+ inclusions { "namespaceX" : "XXX" }
+ strict_inclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do include point with all tags value matching regexs' do
+ config = %([
+ inclusions { "namespace" : "kube-system", "instance" : "^172.*" }
+ strict_inclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do not include point with any tag value not matching regex' do
+ config = %([
+ inclusions { "namespace" : "kube-system", "instance" : "^196.*" }
+ strict_inclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do not include point with tag key not in the record' do
+ config = %([
+ inclusions { "namespaceX" : "XXXX", "instance" : "^172.*" }
+ strict_inclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do not include point with existing tag value not matching regex' do
+ config = %([
+ inclusions { "namespaceX" : "XXXX", "instance" : "^196.*" }
+ strict_inclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+ end
+
+ sub_test_case 'exclusions, non-strict mode' do
+ test 'do not include point with tag value matching regex' do
+ config = %([
+ exclusions { "namespace" : "kube-system" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do include point with tag value not matching regex' do
+ config = %([
+ exclusions { "namespace" : "cube-system" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do include point without key in exclusions' do
+ config = %([
+ exclusions { "namespaceX" : "XXX" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do include point with all tags value not matching regexs' do
+ config = %([
+ exclusions { "namespace" : "cube-system", "instance" : "^196.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do not include point with any tag value matching regex' do
+ config = %([
+ exclusions { "namespace" : "kube-system", "instance" : "^196.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do include point with tag key not in the record' do
+ config = %([
+ exclusions { "namespaceX" : "XXXX", "instance" : "^196.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do not include point with existing tag value matching regex' do
+ config = %([
+ exclusions { "namespaceX" : "XXXX", "instance" : "^172.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+ end
+
+ sub_test_case 'exclusions, strict mode' do
+ test 'do not include point with tag value matching regex' do
+ config = %([
+ exclusions { "namespace" : "kube-system" }
+ strict_exclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do include point with tag value not matching regex' do
+ config = %([
+ exclusions { "namespace" : "cube-system" }
+ strict_exclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do not include point without key in exclusions' do
+ config = %([
+ exclusions { "namespaceX" : "XXX" }
+ strict_exclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do include point with all tags value not matching regexs' do
+ config = %([
+ exclusions { "namespace" : "cube-system", "instance" : "^196.*" }
+ strict_exclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint'
+ end
+
+ test 'do not include point with any tag value matching regex' do
+ config = %([
+ exclusions { "namespace" : "kube-system", "instance" : "^196.*" }
+ strict_exclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do not include point with tag key not in the record' do
+ config = %([
+ exclusions { "namespaceX" : "XXXX", "instance" : "^196.*" }
+ strict_exclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+
+ test 'do not include point with existing tag value matching regex' do
+ config = %([
+ exclusions { "namespaceX" : "XXXX", "instance" : "^172.*" }
+ strict_exclusions true
+ ])
+ outputs = filter_datapoints(config, 'datapoint')
+ assert_equal 0, outputs.length
+ end
+ end
+
+ sub_test_case 'both inclusions and exclusions are validated' do
+ test 'both positive' do
+ config = %([
+ inclusions { "namespace" : "kube-system" }
+ exclusions { "instance" : "^196.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint.nested')
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.nested'
+ end
+
+ test 'negative on inclusions' do
+ config = %([
+ inclusions { "namespace" : "cube-system" }
+ exclusions { "instance" : "^196.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint.nested')
+ assert_equal 0, outputs.length
+ end
+
+ test 'negative on exclusions' do
+ config = %([
+ inclusions { "namespace" : "kube-system" }
+ exclusions { "instance" : "^172.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint.nested')
+ assert_equal 0, outputs.length
+ end
+
+ test 'both negative' do
+ config = %([
+ inclusions { "namespace" : "cube-system" }
+ exclusions { "instance" : "^172.*" }
+ ])
+ outputs = filter_datapoints(config, 'datapoint.nested')
+ assert_equal 0, outputs.length
+ end
+ end
+
+ sub_test_case 'validate inclusions after flatten and relabel' do
+ test 'positive' do
+ config = %([
+ relabel {
+ "service" : "",
+ "kubernetes.service.name" : "service_name",
+ "kubernetes.pod.name" : "pod_name"
+ }
+ inclusions { "pod_name" : "^kube-scheduler-.*" }
+ ])
+ outputs = filter_datapoints(
+ config,
+ 'datapoint.nested'
+ )
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.nested.relabel'
+ end
+
+ test 'negative' do
+ config = %([
+ relabel {
+ "service" : "",
+ "kubernetes.service.name" : "service_name",
+ "kubernetes.pod.name" : "pod_name"
+ }
+ inclusions { "pod_name" : "^cube-scheduler-.*" }
+ ])
+ outputs = filter_datapoints(
+ config,
+ 'datapoint.nested'
+ )
+ assert_equal 0, outputs.length
+ end
+ end
+
+ sub_test_case 'validate exclusions after flatten and relabel' do
+ test 'positive' do
+ config = %([
+ relabel {
+ "service" : "",
+ "kubernetes.service.name" : "service_name",
+ "kubernetes.pod.name" : "pod_name"
+ }
+ exclusions { "pod_name" : "^cube-scheduler.*" }
+ ])
+ outputs = filter_datapoints(
+ config,
+ 'datapoint.nested'
+ )
+ assert_equal 1, outputs.length
+ verify_with_expected outputs, 'output.datapoint.nested.relabel'
+ end
+
+ test 'negative' do
+ config = %([
+ relabel {
+ "service" : "",
+ "kubernetes.service.name" : "service_name",
+ "kubernetes.pod.name" : "pod_name"
+ }
+ exclusions { "pod_name" : "^kube-scheduler.*" }
+ ])
+ outputs = filter_datapoints(
+ config,
+ 'datapoint.nested'
+ )
+ assert_equal 0, outputs.length
+ end
+ end
+
+ sub_test_case 'combinations' do
+ test 'case 1' do
+ config = %([
+ relabel {
+ "service" : "",
+ "prometheus_replica" : "",
+ "prometheus" : "prometheus.name",
+ "pod_name" : "pod.name",
+ "pod" : "pod.name"
+ }
+ inclusions {
+ "instance" : "^172\.20\.36.*"
+ }
+ strict_inclusions true
+ exclusions {
+ "pod.name" : "^kube.*"
+ }
+ strict_exclusions true
+ ])
+ outputs = filter_datapoints(
+ config,
+ 'batch'
+ )
+ assert_equal 270, outputs.length
+ verify_with_expected outputs, 'output.batch.case1'
+ end
+
+ test 'case 2' do
+ config = %([
+ relabel {
+ "service" : "",
+ "prometheus_replica" : "",
+ "prometheus" : "prometheus.name",
+ "pod_name" : "pod.name",
+ "pod" : "pod.name",
+ "le" : ""
+ }
+ inclusions {
+ "job" : "apiserver",
+ "verb" : "GET"
+ }
+ strict_inclusions true
+ ])
+ outputs = filter_datapoints(
+ config,
+ 'batch'
+ )
+ assert_equal 52, outputs.length
+ verify_with_expected outputs, 'output.batch.case2'
+ end
+ end
+
+ private
+
+ def create_driver(conf)
+ Fluent::Test::Driver::Filter
+ .new(Fluent::Plugin::PrometheusFormatFilter)
+ .configure(conf)
+ end
+
+ def filter_datapoints(config, json_name)
+ input = JSON.parse!(
+ File.read("test/resources/#{json_name}.json")
+ )
+
+ d = create_driver(config)
+ d.run(default_tag: 'datapoint.input') do
+ input['datapoints'].flat_map do |datapoint|
+ d.feed(datapoint)
+ d.filtered_records
+ end
+ end
+ end
+
+ def verify_with_expected(outputs, expected_json_name)
+ expected = JSON.parse!(
+ File.read("test/resources/#{expected_json_name}.json")
+ )
+ assert_equal expected['outputs'], outputs
+ end
+end
diff --git a/fluent-plugin-prometheus-format/test/resources/batch.json b/fluent-plugin-prometheus-format/test/resources/batch.json
new file mode 100644
index 0000000000..decf48956a
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/batch.json
@@ -0,0 +1,1715 @@
+{
+ "datapoints": [
+ {
+ "@metric": "http_request_size_bytes_sum",
+ "endpoint": "http-metrics",
+ "handler": "prometheus",
+ "instance": "172.20.36.191:10251",
+ "job": "kube-scheduler",
+ "namespace": "kube-system",
+ "pod": "kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-scheduler",
+ "@timestamp": 1550862304339,
+ "@value": 1619905.0
+ },
+ {
+ "@metric": "kube_service_info",
+ "cluster_ip": "100.65.130.202",
+ "endpoint": "http",
+ "instance": "100.124.149.15:8080",
+ "job": "kube-state-metrics",
+ "namespace": "sumologic",
+ "pod": "prometheus-operator-kube-state-metrics-5667d896b7-9zd99",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "fluentd-deployment-service",
+ "@timestamp": 1550862305141,
+ "@value": 1.0
+ },
+ {
+ "@metric": "container_memory_failures_total",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable",
+ "instance": "172.20.77.239:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-77-239.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "scope": "container",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "type": "pgmajfault",
+ "@timestamp": 1550862309107,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_receive_errors_total",
+ "container_name": "POD",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/besteffort/pod893cea12-3601-11e9-9dd9-06b661a2d196/3562d96acd91b96cc0cb18802897a34c016ae8e7d1b46e3b97304c8dbdf9d623",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.77.239:10255",
+ "interface": "calic786dc1bfbb",
+ "job": "kubelet",
+ "name": "k8s_POD_prometheus-operator-prometheus-node-exporter-2qbnr_monitoring_893cea12-3601-11e9-9dd9-06b661a2d196_0",
+ "namespace": "monitoring",
+ "node": "ip-172-20-77-239.us-west-1.compute.internal",
+ "pod_name": "prometheus-operator-prometheus-node-exporter-2qbnr",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862309107,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/besteffort",
+ "instance": "172.20.77.239:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-77-239.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "tcp_state": "established",
+ "@timestamp": 1550862309107,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/besteffort",
+ "instance": "172.20.77.239:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-77-239.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "tcp_state": "synrecv",
+ "@timestamp": 1550862309107,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_spec_cpu_shares",
+ "container_name": "POD",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/podb75a7c5c-33cc-11e9-9dd9-06b661a2d196/79c1fc5f9bebbf43c5fe22b94282986b1085cb8c0a230d77c7196835b2a64c43",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.77.239:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_calico-node-95n5k_kube-system_b75a7c5c-33cc-11e9-9dd9-06b661a2d196_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-77-239.us-west-1.compute.internal",
+ "pod_name": "calico-node-95n5k",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862309107,
+ "@value": 2.0
+ },
+ {
+ "@metric": "container_spec_cpu_period",
+ "endpoint": "http-metrics",
+ "id": "/init.scope",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862309198,
+ "@value": 100000.0
+ },
+ {
+ "@metric": "container_spec_cpu_shares",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/ntp.service",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862309198,
+ "@value": 1024.0
+ },
+ {
+ "@metric": "go_memstats_alloc_bytes_total",
+ "endpoint": "http-metrics",
+ "instance": "172.20.60.195:10252",
+ "job": "kube-controller-manager",
+ "namespace": "kube-system",
+ "pod": "kube-controller-manager-ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-controller-manager",
+ "@timestamp": 1550862309893,
+ "@value": 8783201120.0
+ },
+ {
+ "@metric": "node_load15",
+ "endpoint": "metrics",
+ "instance": "172.20.87.79:9100",
+ "job": "node-exporter",
+ "namespace": "monitoring",
+ "pod": "prometheus-operator-prometheus-node-exporter-q8bm5",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-prometheus-node-exporter",
+ "@timestamp": 1550862310363,
+ "@value": 0.45
+ },
+ {
+ "@metric": "node_netstat_Udp_OutDatagrams",
+ "endpoint": "metrics",
+ "instance": "172.20.87.79:9100",
+ "job": "node-exporter",
+ "namespace": "monitoring",
+ "pod": "prometheus-operator-prometheus-node-exporter-q8bm5",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-prometheus-node-exporter",
+ "@timestamp": 1550862310363,
+ "@value": 6207.0
+ },
+ {
+ "@metric": "node_network_iface_link_mode",
+ "endpoint": "metrics",
+ "instance": "172.20.87.79:9100",
+ "interface": "lo",
+ "job": "node-exporter",
+ "namespace": "monitoring",
+ "pod": "prometheus-operator-prometheus-node-exporter-q8bm5",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-prometheus-node-exporter",
+ "@timestamp": 1550862310363,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_fs_io_current",
+ "container_name": "POD",
+ "device": "/dev/xvda2",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod9a0ffa93-33ec-11e9-9dd9-06b661a2d196/0ff5aa0036a1e06f112a2f97bd2c1e420593f8bcc51701608a4967121128ca64",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.87.79:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_fluentd-sumologic-nc59z_default_9a0ffa93-33ec-11e9-9dd9-06b661a2d196_0",
+ "namespace": "default",
+ "node": "ip-172-20-87-79.us-west-1.compute.internal",
+ "pod_name": "fluentd-sumologic-nc59z",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862311400,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/rpcbind.service",
+ "instance": "172.20.87.79:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-87-79.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "tcp_state": "lastack",
+ "@timestamp": 1550862311400,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/systemd-remount-fs.service",
+ "instance": "172.20.87.79:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-87-79.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "tcp_state": "lastack",
+ "@timestamp": 1550862311400,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "container_name": "POD",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod90904def-3601-11e9-9dd9-06b661a2d196/d71a379eb5a8ae034dd9d4e6433142725c13a4c8545b247c56d8f214852f110f",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.87.79:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_prometheus-prometheus-operator-prometheus-0_monitoring_90904def-3601-11e9-9dd9-06b661a2d196_0",
+ "namespace": "monitoring",
+ "node": "ip-172-20-87-79.us-west-1.compute.internal",
+ "pod_name": "prometheus-prometheus-operator-prometheus-0",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "tcp_state": "lastack",
+ "@timestamp": 1550862311400,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_spec_memory_limit_bytes",
+ "container_name": "POD",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/podbb6a8793-33cc-11e9-9dd9-06b661a2d196/83348d0d96d72b140489dc13316fbe072eaea2b3ae30a03d7fddf53963a89221",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.87.79:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_calico-node-rdmpd_kube-system_bb6a8793-33cc-11e9-9dd9-06b661a2d196_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-87-79.us-west-1.compute.internal",
+ "pod_name": "calico-node-rdmpd",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862311400,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_spec_memory_reservation_limit_bytes",
+ "container_name": "grafana-sc-dashboard",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/besteffort/pod8948798c-3601-11e9-9dd9-06b661a2d196/a6b493f56c9d4311bd9c4d98bab536030ebfffbc6baf6986f13630d9890f2ff6",
+ "image": "sha256:beb080b7344a4741a02a8c2136ba618efe232f257b4a10affbd8899333edf990",
+ "instance": "172.20.87.79:10255",
+ "job": "kubelet",
+ "name": "k8s_grafana-sc-dashboard_prometheus-operator-grafana-7d7784645b-sn6cm_monitoring_8948798c-3601-11e9-9dd9-06b661a2d196_0",
+ "namespace": "monitoring",
+ "node": "ip-172-20-87-79.us-west-1.compute.internal",
+ "pod_name": "prometheus-operator-grafana-7d7784645b-sn6cm",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862311400,
+ "@value": 0.0
+ },
+ {
+ "@metric": "go_memstats_sys_bytes",
+ "endpoint": "metrics",
+ "instance": "172.20.36.191:9100",
+ "job": "node-exporter",
+ "namespace": "monitoring",
+ "pod": "prometheus-operator-prometheus-node-exporter-52rrm",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-prometheus-node-exporter",
+ "@timestamp": 1550862311846,
+ "@value": 71760120.0
+ },
+ {
+ "@metric": "node_network_receive_compressed_total",
+ "device": "eth0",
+ "endpoint": "metrics",
+ "instance": "172.20.36.191:9100",
+ "job": "node-exporter",
+ "namespace": "monitoring",
+ "pod": "prometheus-operator-prometheus-node-exporter-52rrm",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-prometheus-node-exporter",
+ "@timestamp": 1550862311846,
+ "@value": 0.0
+ },
+ {
+ "@metric": "etcd_disk_backend_snapshot_duration_seconds_bucket",
+ "endpoint": "http-metrics",
+ "instance": "172.20.87.192:4001",
+ "job": "kube-etcd",
+ "le": "+Inf",
+ "namespace": "kube-system",
+ "pod": "etcd-server-ip-172-20-87-192.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-etcd",
+ "@timestamp": 1550862313310,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_cpu_schedstat_run_periods_total",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/besteffort",
+ "instance": "172.20.87.192:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-87-192.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862314020,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_fs_inodes_total",
+ "container_name": "POD",
+ "device": "/dev/xvda2",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod3b33bfc69d10e92ba4c7c6c5f3dfed31/b109dddfdcbccf19e294f91e8763b87a16effa9876c95273d0b68b4060624a48",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.87.192:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_kube-proxy-ip-172-20-87-192.us-west-1.compute.internal_kube-system_3b33bfc69d10e92ba4c7c6c5f3dfed31_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-87-192.us-west-1.compute.internal",
+ "pod_name": "kube-proxy-ip-172-20-87-192.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862314020,
+ "@value": 1.0
+ },
+ {
+ "@metric": "container_fs_reads_bytes_total",
+ "device": "/dev/xvdc",
+ "endpoint": "http-metrics",
+ "id": "/",
+ "instance": "172.20.87.192:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-87-192.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862314020,
+ "@value": 2138112.0
+ },
+ {
+ "@metric": "container_fs_writes_total",
+ "device": "/dev/xvdu",
+ "endpoint": "http-metrics",
+ "id": "/kubepods",
+ "instance": "172.20.87.192:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-87-192.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862314020,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_memory_failcnt",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod96e621a4-33cc-11e9-9dd9-06b661a2d196",
+ "instance": "172.20.87.192:10255",
+ "job": "kubelet",
+ "namespace": "kube-system",
+ "node": "ip-172-20-87-192.us-west-1.compute.internal",
+ "pod_name": "calico-node-xh2p5",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862314020,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/systemd-sysctl.service",
+ "instance": "172.20.87.192:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-87-192.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "tcp_state": "listen",
+ "@timestamp": 1550862314020,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_spec_cpu_shares",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/systemd-sysctl.service",
+ "instance": "172.20.87.192:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-87-192.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862314020,
+ "@value": 1024.0
+ },
+ {
+ "@metric": "container_start_time_seconds",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/rsyslog.service",
+ "instance": "172.20.87.192:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-87-192.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862314020,
+ "@value": 1550528813.0
+ },
+ {
+ "@metric": "node_network_address_assign_type",
+ "endpoint": "metrics",
+ "instance": "172.20.87.192:9100",
+ "interface": "docker0",
+ "job": "node-exporter",
+ "namespace": "monitoring",
+ "pod": "prometheus-operator-prometheus-node-exporter-6rgqv",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-prometheus-node-exporter",
+ "@timestamp": 1550862315622,
+ "@value": 3.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "le": "+Inf",
+ "name": "LimitRanger",
+ "namespace": "default",
+ "operation": "CREATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "services",
+ "service": "kubernetes",
+ "type": "admit",
+ "version": "v1",
+ "@timestamp": 1550862315684,
+ "@value": 15.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "le": "156250",
+ "name": "LimitRanger",
+ "namespace": "default",
+ "operation": "UPDATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "configmaps",
+ "service": "kubernetes",
+ "type": "validate",
+ "version": "v1",
+ "@timestamp": 1550862315684,
+ "@value": 1.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_count",
+ "endpoint": "https",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "name": "NodeRestriction",
+ "namespace": "default",
+ "operation": "CREATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "namespaces",
+ "service": "kubernetes",
+ "type": "admit",
+ "version": "v1",
+ "@timestamp": 1550862315684,
+ "@value": 1.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "group": "apiextensions.k8s.io",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "le": "25000",
+ "name": "Priority",
+ "namespace": "default",
+ "operation": "UPDATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "customresourcedefinitions",
+ "service": "kubernetes",
+ "subresource": "status",
+ "type": "validate",
+ "version": "v1beta1",
+ "@timestamp": 1550862315684,
+ "@value": 75.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "group": "apiextensions.k8s.io",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "le": "390625",
+ "name": "ResourceQuota",
+ "namespace": "default",
+ "operation": "UPDATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "customresourcedefinitions",
+ "service": "kubernetes",
+ "subresource": "status",
+ "type": "validate",
+ "version": "v1beta1",
+ "@timestamp": 1550862315684,
+ "@value": 75.0
+ },
+ {
+ "@metric": "apiserver_admission_step_admission_latencies_seconds_sum",
+ "endpoint": "https",
+ "group": "apps",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "namespace": "default",
+ "operation": "CREATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "statefulsets",
+ "service": "kubernetes",
+ "type": "validate",
+ "version": "v1beta2",
+ "@timestamp": 1550862315684,
+ "@value": 16929.0
+ },
+ {
+ "@metric": "apiserver_request_latencies_bucket",
+ "endpoint": "https",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "le": "8e+06",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "resource": "statefulsets",
+ "scope": "cluster",
+ "service": "kubernetes",
+ "verb": "LIST",
+ "@timestamp": 1550862315684,
+ "@value": 30.0
+ },
+ {
+ "@metric": "apiserver_request_latencies_summary",
+ "endpoint": "https",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "quantile": "0.5",
+ "resource": "prometheusrules",
+ "scope": "cluster",
+ "service": "kubernetes",
+ "verb": "WATCH",
+ "@timestamp": 1550862315684,
+ "@value": 430000576.0
+ },
+ {
+ "@metric": "apiserver_response_sizes_bucket",
+ "endpoint": "https",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "le": "100000",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "resource": "limitranges",
+ "scope": "cluster",
+ "service": "kubernetes",
+ "verb": "WATCH",
+ "@timestamp": 1550862315684,
+ "@value": 806.0
+ },
+ {
+ "@metric": "reflector_lists_total",
+ "endpoint": "https",
+ "instance": "172.20.36.191:443",
+ "job": "apiserver",
+ "name": "reflector_storage_cacher_go__apiregistration_k8s_io_apiservices_7989",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "kubernetes",
+ "@timestamp": 1550862315684,
+ "@value": 1.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "976562.5",
+ "name": "DefaultStorageClass",
+ "namespace": "default",
+ "operation": "CREATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "serviceaccounts",
+ "service": "kubernetes",
+ "type": "admit",
+ "version": "v1",
+ "@timestamp": 1550862316737,
+ "@value": 11150.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "+Inf",
+ "name": "ServiceAccount",
+ "namespace": "default",
+ "operation": "CREATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "configmaps",
+ "service": "kubernetes",
+ "type": "admit",
+ "version": "v1",
+ "@timestamp": 1550862316737,
+ "@value": 417.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "group": "apiextensions.k8s.io",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "976562.5",
+ "name": "MutatingAdmissionWebhook",
+ "namespace": "default",
+ "operation": "DELETE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "customresourcedefinitions",
+ "service": "kubernetes",
+ "type": "admit",
+ "version": "v1beta1",
+ "@timestamp": 1550862316737,
+ "@value": 22.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "group": "apiregistration.k8s.io",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "25000",
+ "name": "Initializers",
+ "namespace": "default",
+ "operation": "UPDATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "apiservices",
+ "service": "kubernetes",
+ "subresource": "status",
+ "type": "admit",
+ "version": "v1",
+ "@timestamp": 1550862316737,
+ "@value": 272452.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "group": "apps",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "25000",
+ "name": "DefaultStorageClass",
+ "namespace": "default",
+ "operation": "CREATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "statefulsets",
+ "service": "kubernetes",
+ "type": "admit",
+ "version": "v1beta2",
+ "@timestamp": 1550862316737,
+ "@value": 29.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "group": "monitoring.coreos.com",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "25000",
+ "name": "NodeRestriction",
+ "namespace": "default",
+ "operation": "DELETE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "prometheuses",
+ "service": "kubernetes",
+ "type": "admit",
+ "version": "v1",
+ "@timestamp": 1550862316737,
+ "@value": 26.0
+ },
+ {
+ "@metric": "apiserver_admission_controller_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "group": "monitoring.coreos.com",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "62500",
+ "name": "Priority",
+ "namespace": "default",
+ "operation": "DELETE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "servicemonitors",
+ "service": "kubernetes",
+ "type": "admit",
+ "version": "v1",
+ "@timestamp": 1550862316737,
+ "@value": 312.0
+ },
+ {
+ "@metric": "apiserver_admission_step_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "group": "extensions",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "976562.5",
+ "namespace": "default",
+ "operation": "UPDATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "deployments",
+ "service": "kubernetes",
+ "subresource": "scale",
+ "type": "validate",
+ "version": "v1beta1",
+ "@timestamp": 1550862316737,
+ "@value": 1.0
+ },
+ {
+ "@metric": "apiserver_admission_step_admission_latencies_seconds_bucket",
+ "endpoint": "https",
+ "group": "monitoring.coreos.com",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "62500",
+ "namespace": "default",
+ "operation": "DELETE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "alertmanagers",
+ "service": "kubernetes",
+ "type": "validate",
+ "version": "v1",
+ "@timestamp": 1550862316737,
+ "@value": 26.0
+ },
+ {
+ "@metric": "apiserver_admission_step_admission_latencies_seconds_summary_count",
+ "endpoint": "https",
+ "group": "monitoring.coreos.com",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "namespace": "default",
+ "operation": "CREATE",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "rejected": "false",
+ "resource": "prometheuses",
+ "service": "kubernetes",
+ "type": "admit",
+ "version": "v1",
+ "@timestamp": 1550862316737,
+ "@value": 26.0
+ },
+ {
+ "@metric": "apiserver_request_count",
+ "client": "kubectl/v1.13.3 (darwin/amd64) kubernetes/721bfa7",
+ "code": "200",
+ "contentType": "application/json",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "resource": "pods",
+ "scope": "namespace",
+ "service": "kubernetes",
+ "verb": "DELETE",
+ "@timestamp": 1550862316737,
+ "@value": 4.0
+ },
+ {
+ "@metric": "apiserver_request_latencies_summary",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "quantile": "0.99",
+ "resource": "deployments",
+ "scope": "namespace",
+ "service": "kubernetes",
+ "verb": "DELETE",
+ "@timestamp": 1550862316737,
+ "@value": 0.0
+ },
+ {
+ "@metric": "apiserver_request_latencies_summary_sum",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "resource": "pods",
+ "scope": "namespace",
+ "service": "kubernetes",
+ "verb": "POST",
+ "@timestamp": 1550862316737,
+ "@value": 97061.0
+ },
+ {
+ "@metric": "apiserver_request_latencies_summary",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "quantile": "0.99",
+ "resource": "secrets",
+ "scope": "namespace",
+ "service": "kubernetes",
+ "verb": "POST",
+ "@timestamp": 1550862316737,
+ "@value": 0.0
+ },
+ {
+ "@metric": "apiserver_response_sizes_bucket",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "1e+07",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "resource": "customresourcedefinitions",
+ "scope": "cluster",
+ "service": "kubernetes",
+ "verb": "WATCH",
+ "@timestamp": 1550862316737,
+ "@value": 731.0
+ },
+ {
+ "@metric": "apiserver_response_sizes_bucket",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "le": "1e+06",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "resource": "endpoints",
+ "scope": "namespace",
+ "service": "kubernetes",
+ "verb": "GET",
+ "@timestamp": 1550862316737,
+ "@value": 207875.0
+ },
+ {
+ "@metric": "apiserver_response_sizes_sum",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "resource": "nodes",
+ "scope": "cluster",
+ "service": "kubernetes",
+ "verb": "WATCH",
+ "@timestamp": 1550862316737,
+ "@value": 1059643074.0
+ },
+ {
+ "@metric": "etcd_request_cache_add_latencies_summary_sum",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "kubernetes",
+ "@timestamp": 1550862316737,
+ "@value": 0.0
+ },
+ {
+ "@metric": "reflector_list_duration_seconds_count",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "name": "reflector_storage_cacher_go__clusterroles_11301",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "kubernetes",
+ "@timestamp": 1550862316737,
+ "@value": 1.0
+ },
+ {
+ "@metric": "reflector_watch_duration_seconds_count",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "name": "reflector_storage_cacher_go__controllerrevisions_11318",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "kubernetes",
+ "@timestamp": 1550862316737,
+ "@value": 0.0
+ },
+ {
+ "@metric": "reflector_watch_duration_seconds",
+ "endpoint": "https",
+ "instance": "172.20.60.195:443",
+ "job": "apiserver",
+ "name": "reflector_storage_cacher_go__networkpolicies_11295",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "quantile": "0.5",
+ "service": "kubernetes",
+ "@timestamp": 1550862316737,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_cpu_schedstat_run_periods_total",
+ "endpoint": "http-metrics",
+ "id": "/docker",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862317137,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_fs_reads_total",
+ "container_name": "POD",
+ "device": "/dev/xvda2",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "pod_name": "etcd-server-ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862317137,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_last_seen",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/sysstat.service",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862317137,
+ "@value": 1550862317.0
+ },
+ {
+ "@metric": "container_memory_failures_total",
+ "container_name": "POD",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "pod_name": "etcd-server-ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "scope": "hierarchy",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "type": "pgfault",
+ "@timestamp": 1550862317137,
+ "@value": 373.0
+ },
+ {
+ "@metric": "cluster_quantile:apiserver_request_latencies:histogram_quantile",
+ "endpoint": "https",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "quantile": "0.9",
+ "resource": "customresourcedefinitions",
+ "scope": "cluster",
+ "service": "kubernetes",
+ "verb": "POST",
+ "@timestamp": 1550862317539,
+ "@value": 0.0
+ },
+ {
+ "@metric": "cluster_quantile:apiserver_request_latencies:histogram_quantile",
+ "endpoint": "https",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "quantile": "0.9",
+ "resource": "customresourcedefinitions",
+ "scope": "cluster",
+ "service": "kubernetes",
+ "verb": "POST",
+ "@timestamp": 1550862318539,
+ "@value": 0.0
+ },
+ {
+ "@metric": "cluster_quantile:apiserver_request_latencies:histogram_quantile",
+ "endpoint": "https",
+ "job": "apiserver",
+ "namespace": "default",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "quantile": "0.9",
+ "resource": "customresourcedefinitions",
+ "scope": "cluster",
+ "service": "kubernetes",
+ "verb": "POST",
+ "@timestamp": 1550862319539,
+ "@value": 3.0
+ },
+ {
+ "@metric": "container_memory_failures_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/cloud-init-local.service",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "scope": "container",
+ "service": "prometheus-operator-kubelet",
+ "type": "pgmajfault",
+ "@timestamp": 1550862316539,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_memory_failures_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/cloud-init-local.service",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "scope": "container",
+ "service": "prometheus-operator-kubelet",
+ "type": "pgmajfault",
+ "@timestamp": 1550862317538,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_memory_failures_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/cloud-init-local.service",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "scope": "container",
+ "service": "prometheus-operator-kubelet",
+ "type": "pgmajfault",
+ "@timestamp": 1550862318539,
+ "@value": 5.0
+ },
+ {
+ "@metric": "container_memory_failures_total",
+ "container_name": "alertmanager",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod8c6c4471-3601-11e9-9dd9-06b661a2d196/cd53938117c67e758182fa8758a8c021e4dbe0a383264e4fd4403e5576e86d6c",
+ "image": "sha256:02e0d8e930da585bfe8eda8b0ea0aa50ce37fee3d4932cb57b68ac02d4ee1c18",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "name": "k8s_alertmanager_alertmanager-prometheus-operator-alertmanager-0_monitoring_8c6c4471-3601-11e9-9dd9-06b661a2d196_0",
+ "namespace": "monitoring",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "pod_name": "alertmanager-prometheus-operator-alertmanager-0",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "scope": "hierarchy",
+ "service": "prometheus-operator-kubelet",
+ "type": "pgfault",
+ "@timestamp": 1550862317538,
+ "@value": 5825.0
+ },
+ {
+ "@metric": "container_memory_max_usage_bytes",
+ "endpoint": "http-metrics",
+ "id": "/kubepods",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862316538,
+ "@value": 930537472.0
+ },
+ {
+ "@metric": "container_memory_max_usage_bytes",
+ "endpoint": "http-metrics",
+ "id": "/kubepods",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862317538,
+ "@value": 930537472.0
+ },
+ {
+ "@metric": "container_memory_max_usage_bytes",
+ "endpoint": "http-metrics",
+ "id": "/kubepods",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862318538,
+ "@value": 930537472.0
+ },
+ {
+ "@metric": "container_network_receive_errors_total",
+ "endpoint": "http-metrics",
+ "id": "/",
+ "instance": "172.20.49.89:10255",
+ "interface": "eth0",
+ "job": "kubelet",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862317538,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "container_name": "fluentd",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod9a0e1ca2-33ec-11e9-9dd9-06b661a2d196/36ebe85a66d5b66e1918876db5507e57e899794c48309362be77e535e6b9a460",
+ "image": "sumologic/fluentd-kubernetes-sumologic@sha256:a8ac3ea58b98f4e5be8977aafa80f793e9d94d51141619567cb9d4d840d59d45",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "name": "k8s_fluentd_fluentd-sumologic-29qwd_default_9a0e1ca2-33ec-11e9-9dd9-06b661a2d196_0",
+ "namespace": "default",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "pod_name": "fluentd-sumologic-29qwd",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "tcp_state": "close",
+ "@timestamp": 1550862317538,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_udp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod9a0e1ca2-33ec-11e9-9dd9-06b661a2d196",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "namespace": "default",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "pod_name": "fluentd-sumologic-29qwd",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "udp_state": "listen",
+ "@timestamp": 1550862317538,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_fs_reads_total",
+ "container_name": "POD",
+ "device": "/dev/xvda2",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/pod171fe2ce-36d3-11e9-9dd9-06b661a2d196/95ff5fba5362c26a39572e7b9bd192049cece14c21c7da1e8f1aa8028e2ecc47",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.49.89:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_fluentd-deployment-76c7759567-gxchb_sumologic_171fe2ce-36d3-11e9-9dd9-06b661a2d196_0",
+ "namespace": "sumologic",
+ "node": "ip-172-20-49-89.us-west-1.compute.internal",
+ "pod_name": "fluentd-deployment-76c7759567-gxchb",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862317538,
+ "@value": 0.0
+ },
+ {
+ "To": "e5987458574dc057",
+ "@metric": "etcd_network_peer_round_trip_time_seconds_bucket",
+ "endpoint": "http-metrics",
+ "instance": "172.20.60.195:4001",
+ "job": "kube-etcd",
+ "le": "0.1024",
+ "namespace": "kube-system",
+ "pod": "etcd-server-ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-etcd",
+ "@timestamp": 1550862317752,
+ "@value": 11115.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod971e7a8e-33cc-11e9-9dd9-06b661a2d196",
+ "instance": "172.20.77.239:10255",
+ "job": "kubelet",
+ "namespace": "kube-system",
+ "node": "ip-172-20-77-239.us-west-1.compute.internal",
+ "pod_name": "kube-dns-autoscaler-6b658bd4d5-d6trb",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "tcp_state": "established",
+ "@timestamp": 1550862317724,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_tasks_state",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/systemd-remount-fs.service",
+ "instance": "172.20.77.239:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-77-239.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "state": "sleeping",
+ "@timestamp": 1550862317724,
+ "@value": 0.0
+ },
+ {
+ "@metric": "http_request_size_bytes_sum",
+ "endpoint": "http-metrics",
+ "handler": "prometheus",
+ "instance": "172.20.36.191:10251",
+ "job": "kube-scheduler",
+ "namespace": "kube-system",
+ "pod": "kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-scheduler",
+ "@timestamp": 1550862319243,
+ "@value": 1620122.0
+ },
+ {
+ "@metric": "http_request_size_bytes_sum",
+ "endpoint": "http-metrics",
+ "handler": "prometheus",
+ "instance": "172.20.36.191:10251",
+ "job": "kube-scheduler",
+ "namespace": "kube-system",
+ "pod": "kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-scheduler",
+ "@timestamp": 1550862329243,
+ "@value": 1620222.0
+ },
+ {
+ "@metric": "http_request_size_bytes_sum",
+ "endpoint": "http-metrics",
+ "handler": "prometheus",
+ "instance": "172.20.36.191:10251",
+ "job": "kube-scheduler",
+ "namespace": "kube-system",
+ "pod": "kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-scheduler",
+ "@timestamp": 1550862339243,
+ "@value": 1620422.0
+ },
+ {
+ "@metric": "container_fs_inodes_total",
+ "container_name": "POD",
+ "device": "/dev/xvda2",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/podf4d4fdde0385643b22c6d2ed3cbda862/b5210f14587ef34d903c9ad08fdf291391d04fe0e01b7474b16671bca2861e13",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_etcd-server-ip-172-20-60-195.us-west-1.compute.internal_kube-system_f4d4fdde0385643b22c6d2ed3cbda862_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "pod_name": "etcd-server-ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862320644,
+ "@value": 1.0
+ },
+ {
+ "@metric": "container_fs_reads_total",
+ "container_name": "calico-node",
+ "device": "/dev/xvda",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/poda06948c0-33cc-11e9-9dd9-06b661a2d196/c7e635b9a3791a14630a0937229d90556ec94243e94e7d1c737ef015db592a0e",
+ "image": "quay.io/calico/node@sha256:da1a7ee6bb1c66950a8a73965cc11bd80fda99c97870c188b08ffb6c646b8ddc",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "name": "k8s_calico-node_calico-node-hnptc_kube-system_a06948c0-33cc-11e9-9dd9-06b661a2d196_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "pod_name": "calico-node-hnptc",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "@timestamp": 1550862320644,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/podb6219e20943498110e5f3f5ba8807a98",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "namespace": "kube-system",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "pod_name": "kube-controller-manager-ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "tcp_state": "timewait",
+ "@timestamp": 1550862320644,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/systemd-random-seed.service",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "tcp_state": "synsent",
+ "@timestamp": 1550862320644,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_cpu_schedstat_runqueue_seconds_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/cloud-init-local.service",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862320774,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_cpu_user_seconds_total",
+ "container_name": "node-exporter",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/besteffort/pod89224a32-3601-11e9-9dd9-06b661a2d196/d12b674aa4d31f0e630d14fb958fb07ea211496e4531f8354d0470bed677e91d",
+ "image": "sha256:b3e7f67a1480979f7c2068c14e4f0d3da6243631b9def8490ed68236a8311f11",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "name": "k8s_node-exporter_prometheus-operator-prometheus-node-exporter-4t9rg_monitoring_89224a32-3601-11e9-9dd9-06b661a2d196_0",
+ "namespace": "monitoring",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "pod_name": "prometheus-operator-prometheus-node-exporter-4t9rg",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862320774,
+ "@value": 38.12
+ },
+ {
+ "@metric": "container_memory_failures_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/cloud-init-local.service",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "scope": "hierarchy",
+ "service": "prometheus-operator-kubelet",
+ "type": "pgfault",
+ "@timestamp": 1550862320774,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_udp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/systemd-random-seed.service",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "udp_state": "txqueued",
+ "@timestamp": 1550862320774,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_udp_usage_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/systemd-udev-trigger.service",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "udp_state": "txqueued",
+ "@timestamp": 1550862320774,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_spec_cpu_shares",
+ "endpoint": "http-metrics",
+ "id": "/init.scope",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862320774,
+ "@value": 1024.0
+ },
+ {
+ "@metric": "container_spec_memory_swap_limit_bytes",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/lvm2-lvmetad.service",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862320774,
+ "@value": 0.0
+ },
+ {
+ "@metric": "rest_client_request_latency_seconds_bucket",
+ "endpoint": "http-metrics",
+ "instance": "172.20.60.195:10255",
+ "job": "kubelet",
+ "le": "0.016",
+ "namespace": "kube-system",
+ "node": "ip-172-20-60-195.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-prometheus-oper-kubelet",
+ "url": "https://127.0.0.1/%7Bprefix%7D",
+ "verb": "DELETE",
+ "@timestamp": 1550862321039,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_cpu_schedstat_run_periods_total",
+ "container_name": "POD",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod37d41543c66f623fda26346f61eb60ab/cb2208bfaad5140f58b4ce18ffc402506db8adf20a8b9b6bc4b9c31a2e6b0024",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.87.79:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_kube-proxy-ip-172-20-87-79.us-west-1.compute.internal_kube-system_37d41543c66f623fda26346f61eb60ab_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-87-79.us-west-1.compute.internal",
+ "pod_name": "kube-proxy-ip-172-20-87-79.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862322359,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_cpu_schedstat_runqueue_seconds_total",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/kmod-static-nodes.service",
+ "instance": "172.20.87.79:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-87-79.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862322359,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_memory_failures_total",
+ "container_name": "POD",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/besteffort/pod89388c78-3601-11e9-9dd9-06b661a2d196/d2fd5abaeb880b386be15ac462f04935212f88585a73971db91bff3f50debd39",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.87.79:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_prometheus-operator-prometheus-node-exporter-q8bm5_monitoring_89388c78-3601-11e9-9dd9-06b661a2d196_0",
+ "namespace": "monitoring",
+ "node": "ip-172-20-87-79.us-west-1.compute.internal",
+ "pod_name": "prometheus-operator-prometheus-node-exporter-q8bm5",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "scope": "container",
+ "service": "prometheus-operator-kubelet",
+ "type": "pgfault",
+ "@timestamp": 1550862322359,
+ "@value": 201.0
+ },
+ {
+ "@metric": "container_cpu_user_seconds_total",
+ "endpoint": "http-metrics",
+ "id": "/",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862324543,
+ "@value": 23870.22
+ },
+ {
+ "@metric": "container_fs_sector_writes_total",
+ "container_name": "POD",
+ "device": "/dev/xvda2",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod0f04a82db0cb7f4c92c333ce54f314f1/7e2fe9d6366adb209d3412164e0a4040306afc63ea0b177bf25ea1ed6f011951",
+ "image": "k8s.gcr.io/pause-amd64:3.0",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "name": "k8s_POD_kube-proxy-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0f04a82db0cb7f4c92c333ce54f314f1_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "pod_name": "kube-proxy-ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862324543,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_fs_writes_total",
+ "container_name": "calico-node",
+ "device": "/dev/xvda2",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod96cdc1a8-33cc-11e9-9dd9-06b661a2d196/a733137ae1360c4fd509703ae5fc9e354cda6d90ced0d2584e6b8ef5d1127a6e",
+ "image": "quay.io/calico/node@sha256:da1a7ee6bb1c66950a8a73965cc11bd80fda99c97870c188b08ffb6c646b8ddc",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "name": "k8s_calico-node_calico-node-t65cw_kube-system_96cdc1a8-33cc-11e9-9dd9-06b661a2d196_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "pod_name": "calico-node-t65cw",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862324543,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_network_tcp_usage_total",
+ "container_name": "kube-proxy",
+ "endpoint": "http-metrics",
+ "id": "/kubepods/burstable/pod0f04a82db0cb7f4c92c333ce54f314f1/b12799abf23b9c0886ee55d29e36b739226cc46d276a4e0136aa05272e0ed426",
+ "image": "k8s.gcr.io/kube-proxy@sha256:de320f2611b72465371292c87d892e64b01bf5e27b211b9e8969a239d0f2523a",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "name": "k8s_kube-proxy_kube-proxy-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0f04a82db0cb7f4c92c333ce54f314f1_0",
+ "namespace": "kube-system",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "pod_name": "kube-proxy-ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "tcp_state": "closing",
+ "@timestamp": 1550862324543,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_spec_cpu_shares",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/systemd-udev-trigger.service",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "@timestamp": 1550862324543,
+ "@value": 1024.0
+ },
+ {
+ "@metric": "container_tasks_state",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/cloud-config.service",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "state": "stopped",
+ "@timestamp": 1550862324543,
+ "@value": 0.0
+ },
+ {
+ "@metric": "container_tasks_state",
+ "endpoint": "http-metrics",
+ "id": "/system.slice/systemd-user-sessions.service",
+ "instance": "172.20.36.191:10255",
+ "job": "kubelet",
+ "node": "ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kubelet",
+ "state": "uninterruptible",
+ "@timestamp": 1550862324543,
+ "@value": 0.0
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/datapoint.json b/fluent-plugin-prometheus-format/test/resources/datapoint.json
new file mode 100644
index 0000000000..3c900d3492
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/datapoint.json
@@ -0,0 +1,18 @@
+{
+ "datapoints": [
+ {
+ "endpoint": "http-metrics",
+ "handler": "prometheus",
+ "instance": "172.20.36.191:10251",
+ "job": "kube-scheduler",
+ "namespace": "kube-system",
+ "pod": "kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-scheduler",
+ "@metric": "http_request_size_bytes_sum",
+ "@timestamp": 1550862304339,
+ "@value": 1619905.0
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/datapoint.nested.escape.json b/fluent-plugin-prometheus-format/test/resources/datapoint.nested.escape.json
new file mode 100644
index 0000000000..2cde59f0a0
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/datapoint.nested.escape.json
@@ -0,0 +1,27 @@
+{
+ "datapoints": [
+ {
+ "endpoint": "http-metrics",
+ "error": "No file:\n\"FILE.TXT\"",
+ "handler": "prometheus",
+ "instance": "172.20.36.191:10251",
+ "job": "kube-scheduler",
+ "namespace": "kube-system",
+ "kubernetes": {
+ "pod": {
+ "name": "kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal"
+ },
+ "service": {
+ "name": "kube-scheduler"
+ }
+ },
+ "path": "C:\\DIR\\FILE.TXT",
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-scheduler",
+ "@metric": "http_request_size_bytes_sum",
+ "@timestamp": 1550862304339,
+ "@value": 1619905.0
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/datapoint.nested.json b/fluent-plugin-prometheus-format/test/resources/datapoint.nested.json
new file mode 100644
index 0000000000..ecc7f8fddd
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/datapoint.nested.json
@@ -0,0 +1,25 @@
+{
+ "datapoints": [
+ {
+ "endpoint": "http-metrics",
+ "handler": "prometheus",
+ "instance": "172.20.36.191:10251",
+ "job": "kube-scheduler",
+ "namespace": "kube-system",
+ "kubernetes": {
+ "pod": {
+ "name": "kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal"
+ },
+ "service": {
+ "name": "kube-scheduler"
+ }
+ },
+ "prometheus": "monitoring/prometheus-operator-prometheus",
+ "prometheus_replica": "prometheus-prometheus-operator-prometheus-0",
+ "service": "prometheus-operator-kube-scheduler",
+ "@metric": "http_request_size_bytes_sum",
+ "@timestamp": 1550862304339,
+ "@value": 1619905.0
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/datapoint.nested.spaces.json b/fluent-plugin-prometheus-format/test/resources/datapoint.nested.spaces.json
new file mode 100644
index 0000000000..f4d07295ac
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/datapoint.nested.spaces.json
@@ -0,0 +1,25 @@
+{
+ "datapoints": [
+ {
+ "endpoint": "http- etrics",
+ "handler": "pro etheus",
+ "instance": "172.20.36.191:10251",
+ "job": "kube-scheduler",
+ "na espace": "kube-syste ",
+ "kubernetes": {
+ "pod": {
+ "na e": "kube-scheduler-ip-172-20-36-191.us-west-1.co pute.internal"
+ },
+ "service": {
+ "na e": "kube-scheduler"
+ }
+ },
+ "pro etheus": " onitoring/pro etheus-operator-pro etheus",
+ "pro etheus_replica": "pro etheus-pro etheus-operator-pro etheus-0",
+ "service": "pro etheus-operator-kube-scheduler",
+ "@metric": "http_request_size_bytes_su ",
+ "@timestamp": 1550862304339,
+ "@value": 1619905.0
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/output.batch.case1.json b/fluent-plugin-prometheus-format/test/resources/output.batch.case1.json
new file mode 100644
index 0000000000..c43484cb3a
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/output.batch.case1.json
@@ -0,0 +1,814 @@
+{
+ "outputs": [
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "container_fs_writes_total{container_name=\"calico-node\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod96cdc1a8-33cc-11e9-9dd9-06b661a2d196/a733137ae1360c4fd509703ae5fc9e354cda6d90ced0d2584e6b8ef5d1127a6e\",image=\"quay.io/calico/node@sha256:da1a7ee6bb1c66950a8a73965cc11bd80fda99c97870c188b08ffb6c646b8ddc\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_calico-node_calico-node-t65cw_kube-system_96cdc1a8-33cc-11e9-9dd9-06b661a2d196_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"calico-node-t65cw\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862324543"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "container_fs_writes_total{container_name=\"calico-node\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod96cdc1a8-33cc-11e9-9dd9-06b661a2d196/a733137ae1360c4fd509703ae5fc9e354cda6d90ced0d2584e6b8ef5d1127a6e\",image=\"quay.io/calico/node@sha256:da1a7ee6bb1c66950a8a73965cc11bd80fda99c97870c188b08ffb6c646b8ddc\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_calico-node_calico-node-t65cw_kube-system_96cdc1a8-33cc-11e9-9dd9-06b661a2d196_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"calico-node-t65cw\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862324543"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "container_fs_writes_total{container_name=\"calico-node\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod96cdc1a8-33cc-11e9-9dd9-06b661a2d196/a733137ae1360c4fd509703ae5fc9e354cda6d90ced0d2584e6b8ef5d1127a6e\",image=\"quay.io/calico/node@sha256:da1a7ee6bb1c66950a8a73965cc11bd80fda99c97870c188b08ffb6c646b8ddc\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_calico-node_calico-node-t65cw_kube-system_96cdc1a8-33cc-11e9-9dd9-06b661a2d196_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"calico-node-t65cw\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862324543"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "container_fs_writes_total{container_name=\"calico-node\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod96cdc1a8-33cc-11e9-9dd9-06b661a2d196/a733137ae1360c4fd509703ae5fc9e354cda6d90ced0d2584e6b8ef5d1127a6e\",image=\"quay.io/calico/node@sha256:da1a7ee6bb1c66950a8a73965cc11bd80fda99c97870c188b08ffb6c646b8ddc\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_calico-node_calico-node-t65cw_kube-system_96cdc1a8-33cc-11e9-9dd9-06b661a2d196_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"calico-node-t65cw\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862324543"
+ },
+ {
+ "message": "go_memstats_sys_bytes{endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 71760120.0 1550862311846"
+ },
+ {
+ "message": "node_network_receive_compressed_total{device=\"eth0\",endpoint=\"metrics\",instance=\"172.20.36.191:9100\",job=\"node-exporter\",namespace=\"monitoring\",pod.name=\"prometheus-operator-prometheus-node-exporter-52rrm\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862311846"
+ },
+ {
+ "message": "container_fs_reads_total{container_name=\"POD\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862317137"
+ },
+ {
+ "message": "container_memory_failures_total{container_name=\"POD\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod0127334d9523c644f89c1b13407d1af7/4400d14421f3a455ca83f2138184764096ad5cc99ddf930ba94510b88afae96f\",image=\"k8s.gcr.io/pause-amd64:3.0\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_POD_etcd-server-ip-172-20-36-191.us-west-1.compute.internal_kube-system_0127334d9523c644f89c1b13407d1af7_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"etcd-server-ip-172-20-36-191.us-west-1.compute.internal\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",scope=\"hierarchy\",type=\"pgfault\",_origin=\"kubernetes\"} 373.0 1550862317137"
+ },
+ {
+ "message": "container_fs_writes_total{container_name=\"calico-node\",device=\"/dev/xvda2\",endpoint=\"http-metrics\",id=\"/kubepods/burstable/pod96cdc1a8-33cc-11e9-9dd9-06b661a2d196/a733137ae1360c4fd509703ae5fc9e354cda6d90ced0d2584e6b8ef5d1127a6e\",image=\"quay.io/calico/node@sha256:da1a7ee6bb1c66950a8a73965cc11bd80fda99c97870c188b08ffb6c646b8ddc\",instance=\"172.20.36.191:10255\",job=\"kubelet\",name=\"k8s_calico-node_calico-node-t65cw_kube-system_96cdc1a8-33cc-11e9-9dd9-06b661a2d196_0\",namespace=\"kube-system\",node=\"ip-172-20-36-191.us-west-1.compute.internal\",pod.name=\"calico-node-t65cw\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 0.0 1550862324543"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/output.batch.case2.json b/fluent-plugin-prometheus-format/test/resources/output.batch.case2.json
new file mode 100644
index 0000000000..6d4f3125a5
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/output.batch.case2.json
@@ -0,0 +1,160 @@
+{
+ "outputs": [
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ },
+ {
+ "message": "apiserver_response_sizes_bucket{endpoint=\"https\",instance=\"172.20.60.195:443\",job=\"apiserver\",namespace=\"default\",prometheus.name=\"monitoring/prometheus-operator-prometheus\",resource=\"endpoints\",scope=\"namespace\",verb=\"GET\",_origin=\"kubernetes\"} 207875.0 1550862316737"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/output.datapoint.json b/fluent-plugin-prometheus-format/test/resources/output.datapoint.json
new file mode 100644
index 0000000000..d6e1dd0a80
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/output.datapoint.json
@@ -0,0 +1,7 @@
+{
+ "outputs": [
+ {
+ "message": "http_request_size_bytes_sum{endpoint=\"http-metrics\",handler=\"prometheus\",instance=\"172.20.36.191:10251\",job=\"kube-scheduler\",namespace=\"kube-system\",pod=\"kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal\",prometheus=\"monitoring/prometheus-operator-prometheus\",prometheus_replica=\"prometheus-prometheus-operator-prometheus-0\",service=\"prometheus-operator-kube-scheduler\",_origin=\"kubernetes\"} 1619905.0 1550862304339"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.escape.json b/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.escape.json
new file mode 100644
index 0000000000..16e12013cc
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.escape.json
@@ -0,0 +1,7 @@
+{
+ "outputs": [
+ {
+ "message": "http_request_size_bytes_sum{endpoint=\"http-metrics\",error=\"No file:\\n\\\"FILE.TXT\\\"\",handler=\"prometheus\",instance=\"172.20.36.191:10251\",job=\"kube-scheduler\",kubernetes.pod.name=\"kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal\",kubernetes.service.name=\"kube-scheduler\",namespace=\"kube-system\",path=\"C:\\\\DIR\\\\FILE.TXT\",prometheus=\"monitoring/prometheus-operator-prometheus\",prometheus_replica=\"prometheus-prometheus-operator-prometheus-0\",service=\"prometheus-operator-kube-scheduler\",_origin=\"kubernetes\"} 1619905.0 1550862304339"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.json b/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.json
new file mode 100644
index 0000000000..4c6b5b8ca6
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.json
@@ -0,0 +1,7 @@
+{
+ "outputs": [
+ {
+ "message": "http_request_size_bytes_sum{endpoint=\"http-metrics\",handler=\"prometheus\",instance=\"172.20.36.191:10251\",job=\"kube-scheduler\",kubernetes.pod.name=\"kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal\",kubernetes.service.name=\"kube-scheduler\",namespace=\"kube-system\",prometheus=\"monitoring/prometheus-operator-prometheus\",prometheus_replica=\"prometheus-prometheus-operator-prometheus-0\",service=\"prometheus-operator-kube-scheduler\",_origin=\"kubernetes\"} 1619905.0 1550862304339"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.relabel.json b/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.relabel.json
new file mode 100644
index 0000000000..c31f807437
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.relabel.json
@@ -0,0 +1,7 @@
+{
+ "outputs": [
+ {
+ "message": "http_request_size_bytes_sum{endpoint=\"http-metrics\",handler=\"prometheus\",instance=\"172.20.36.191:10251\",job=\"kube-scheduler\",namespace=\"kube-system\",pod_name=\"kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal\",prometheus=\"monitoring/prometheus-operator-prometheus\",prometheus_replica=\"prometheus-prometheus-operator-prometheus-0\",service_name=\"kube-scheduler\",_origin=\"kubernetes\"} 1619905.0 1550862304339"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.spaces.relabel.json b/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.spaces.relabel.json
new file mode 100644
index 0000000000..8cff24bc91
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/output.datapoint.nested.spaces.relabel.json
@@ -0,0 +1,7 @@
+{
+ "outputs": [
+ {
+ "message": "http_request_size_bytes_su {endpoint=\"http- etrics\",handler=\"pro etheus\",instance=\"172.20.36.191:10251\",job=\"kube-scheduler\",na espace=\"kube-syste \",pod_na e=\"kube-scheduler-ip-172-20-36-191.us-west-1.co pute.internal\",pro etheus=\" onitoring/pro etheus-operator-pro etheus\",pro etheus_replica=\"pro etheus-pro etheus-operator-pro etheus-0\",service_na e=\"kube-scheduler\",_origin=\"kubernetes\"} 1619905.0 1550862304339"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/output.datapoint.relabel.json b/fluent-plugin-prometheus-format/test/resources/output.datapoint.relabel.json
new file mode 100644
index 0000000000..4dc890c92e
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/output.datapoint.relabel.json
@@ -0,0 +1,7 @@
+{
+ "outputs": [
+ {
+ "message": "http_request_size_bytes_sum{endpoint=\"http-metrics\",handler=\"prometheus\",instance=\"172.20.36.191:10251\",job=\"kube-scheduler\",namespace=\"kube-system\",pod=\"kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal\",prometheus=\"monitoring/prometheus-operator-prometheus\",prometheus.replica=\"prometheus-prometheus-operator-prometheus-0\",prometheus.service=\"prometheus-operator-kube-scheduler\",_origin=\"kubernetes\"} 1619905.0 1550862304339"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fluent-plugin-prometheus-format/test/resources/output.datapoint.relabel.remove.json b/fluent-plugin-prometheus-format/test/resources/output.datapoint.relabel.remove.json
new file mode 100644
index 0000000000..18eac2c7b8
--- /dev/null
+++ b/fluent-plugin-prometheus-format/test/resources/output.datapoint.relabel.remove.json
@@ -0,0 +1,7 @@
+{
+ "outputs": [
+ {
+ "message": "http_request_size_bytes_sum{endpoint=\"http-metrics\",handler=\"prometheus\",instance=\"172.20.36.191:10251\",job=\"kube-scheduler\",namespace=\"kube-system\",pod=\"kube-scheduler-ip-172-20-36-191.us-west-1.compute.internal\",prometheus=\"monitoring/prometheus-operator-prometheus\",_origin=\"kubernetes\"} 1619905.0 1550862304339"
+ }
+ ]
+}
\ No newline at end of file