diff --git a/.gitignore b/.gitignore index 14f8c5bac3..89e29e69fe 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ .ruby-version .rvmrc bin/* +coverage/ Gemfile.lock # Mac OSX ignores @@ -28,4 +29,4 @@ ubuntu-bionic-18.04-cloudimg-console.log .kube-config # ignore tmp direcotries -tmp \ No newline at end of file +tmp diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 2a8fec935f..8f00a19906 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -35,7 +35,6 @@ RUN gem install \ RUN gem install \ fluent-plugin-systemd:1.0.2 \ fluent-plugin-record-modifier:2.0.1 \ - fluent-plugin-kubernetes_metadata_filter:2.5.2 \ fluent-plugin-sumologic_output:1.7.1 \ fluent-plugin-concat:2.4.0 \ fluent-plugin-rewrite-tag-filter:2.2.0 \ @@ -55,6 +54,7 @@ RUN mkdir /tmp/terraform \ COPY gems/fluent-plugin*.gem ./ RUN gem install \ --local fluent-plugin-prometheus-format \ + --local fluent-plugin-kubernetes-metadata-filter \ --local fluent-plugin-kubernetes-sumologic \ --local fluent-plugin-enhance-k8s-metadata \ --local fluent-plugin-datapoint \ diff --git a/fluent-plugin-kubernetes-metadata-filter/Gemfile b/fluent-plugin-kubernetes-metadata-filter/Gemfile new file mode 100644 index 0000000000..9021694006 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/Gemfile @@ -0,0 +1,7 @@ +source 'https://rubygems.org' + +gem 'codeclimate-test-reporter', '<1.0.0', :group => :test, :require => nil +gem 'rubocop', require: false + +# Specify your gem's dependencies in fluent-plugin-add.gemspec +gemspec diff --git a/fluent-plugin-kubernetes-metadata-filter/README.md b/fluent-plugin-kubernetes-metadata-filter/README.md new file mode 100644 index 0000000000..f0290ac975 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/README.md @@ -0,0 +1,250 @@ +# fluent-plugin-kubernetes-metadata-filter, a plugin for [Fluentd](http://fluentd.org) + +This code was retrieved from at commit [84f66a8](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/commit/84f66a8f9e06ab5b5211053fcce4cd8ab4bd74ba). Original README follows. + +The Kubernetes metadata plugin filter enriches container log records with pod and namespace metadata. + +This plugin derives basic metadata about the container that emitted a given log record using the source of the log record. Records from journald provide metadata about the +container environment as named fields. Records from JSON files encode metadata about the container in the file name. The initial metadata derived from the source is used +to lookup additional metadata about the container's associated pod and namespace (e.g. UUIDs, labels, annotations) when the kubernetes_url is configured. If the plugin cannot +authoritatively determine the namespace of the container emitting a log record, it will use an 'orphan' namespace ID in the metadata. This behaviors supports multi-tenant systems +that rely on the authenticity of the namespace for proper log isolation. + +## Requirements + +| fluent-plugin-kubernetes_metadata_filter | fluentd | ruby | +|-------------------|---------|------| +| >= 2.5.0 | >= v1.10.0 | >= 2.5 | +| >= 2.0.0 | >= v0.14.20 | >= 2.1 | +| < 2.0.0 | >= v0.12.0 | >= 1.9 | + +NOTE: For v0.12 version, you should use 1.x.y version. Please send patch into v0.12 branch if you encountered 1.x version's bug. + +NOTE: This documentation is for fluent-plugin-kubernetes_metadata_filter-plugin-elasticsearch 2.x or later. For 1.x documentation, please see [v0.12 branch](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/tree/v0.12). + +## Installation + + gem install fluent-plugin-kubernetes_metadata_filter + +## Configuration + +Configuration options for fluent.conf are: + +* `kubernetes_url` - URL to the API server. Set this to retrieve further kubernetes metadata for logs from kubernetes API server. If not specified, environment variables `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT` will be used if both are present which is typically true when running fluentd in a pod. +* `apiVersion` - API version to use (default: `v1`) +* `ca_file` - path to CA file for Kubernetes server certificate validation +* `verify_ssl` - validate SSL certificates (default: `true`) +* `client_cert` - path to a client cert file to authenticate to the API server +* `client_key` - path to a client key file to authenticate to the API server +* `bearer_token_file` - path to a file containing the bearer token to use for authentication +* `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag. +This must used named capture groups for `container_name`, `pod_name` & `namespace` (default: `\.(?[^\._]+)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$)`) +* `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`) +* `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour) +* `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`) +* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`) +* `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`) +* *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver. If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields +if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format. +* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field (default: `'^(?[^_]+)_(?[^\._]+)(\.(?[^_]+))?_(?[^_]+)_(?[^_]+)_[^_]+_[^_]+$'` + * This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317) +* `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record. +* `allow_orphans` - Modify the namespace and namespace id to the values of `orphaned_namespace_name` and `orphaned_namespace_id` +when true (default: `true`) +* `orphaned_namespace_name` - The namespace to associate with records where the namespace can not be determined (default: `.orphaned`) +* `orphaned_namespace_id` - The namespace id to associate with records where the namespace can not be determined (default: `orphaned`) +* `lookup_from_k8s_field` - If the field `kubernetes` is present, lookup the metadata from the given subfields such as `kubernetes.namespace_name`, `kubernetes.pod_name`, etc. This allows you to avoid having to pass in metadata to lookup in an explicitly formatted tag name or in an explicitly formatted `CONTAINER_NAME` value. For example, set `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`, and `docker.id` in the record, and the filter will fill in the rest. (default: `true`) +* `ssl_partial_chain` - if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want + to trust the intermediate CA certs we do have, set this to `true` - this corresponds to + the `openssl s_client -partial_chain` flag and `X509_V_FLAG_PARTIAL_CHAIN` (default: `false`) +* `skip_labels` - Skip all label fields from the metadata. +* `skip_container_metadata` - Skip some of the container data of the metadata. The metadata will not contain the container_image and container_image_id fields. +* `skip_master_url` - Skip the master_url field from the metadata. +* `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less. +* `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`) + +**NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the +payload. The following configuration options are removed: + +* `merge_json_log` +* `preserve_json_log` + +One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser) + +**NOTE** As of this release, the use of `use_journal` is **DEPRECATED**. If this setting is not present, the plugin will +attempt to figure out the source of the metadata fields from the following: +- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record: +`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`, +then the plugin will use those values as the source to use to lookup the metadata +- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record, +then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata +- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to +lookup the metdata + +Reading from the JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser": + +``` +fluent-gem install fluent-plugin-multi-format-parser +``` + +The config block could look like this: +``` + + @type tail + path /var/log/containers/*.log + pos_file fluentd-docker.pos + read_from_head true + tag kubernetes.* + + @type multi_format + + format json + time_key time + time_type string + time_format "%Y-%m-%dT%H:%M:%S.%NZ" + keep_time_key false + + + format regexp + expression /^(? + + + + + @type kubernetes_metadata + + + + @type stdout + +``` + +Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` and `systemd-journal` plugins, and requires docker to use the `--log-driver=journald` log driver): +``` + + @type systemd + path /run/log/journal + pos_file journal.pos + tag journal + read_from_head true + + +# probably want to use something like fluent-plugin-rewrite-tag-filter to +# retag entries from k8s + + @type rewrite_tag_filter + rewriterule1 CONTAINER_NAME ^k8s_ kubernetes.journal.container + ... + + + + @type kubernetes_metadata + use_journal true + + + + @type stdout + +``` +## Log content as JSON +In former versions this plugin parsed the value of the key log as JSON. In the current version this feature was removed, to avoid duplicate features in the fluentd plugin ecosystem. It can parsed with the parser plugin like this: +``` + + @type parser + key_name log + + @type json + json_parser json + + replace_invalid_sequence true + reserve_data true # this preserves unparsable log lines + emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean + reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time. + +``` + +## Environment variables for Kubernetes + +If the name of the Kubernetes node the plugin is running on is set as +an environment variable with the name `K8S_NODE_NAME`, it will reduce cache +misses and needless calls to the Kubernetes API. + +In the Kubernetes container definition, this is easily accomplished by: + +```yaml +env: +- name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName +``` + +## Example input/output + +Kubernetes creates symlinks to Docker log files in `/var/log/containers/*.log`. Docker logs in JSON format. + +Assuming following inputs are coming from a log file named `/var/log/containers/fabric8-console-controller-98rqc_default_fabric8-console-container-df14e0d5ae4c07284fa636d739c8fc2e6b52bc344658de7d3f08c36a2e804115.log`: + +``` +{ + "log": "2015/05/05 19:54:41 \n", + "stream": "stderr", + "time": "2015-05-05T19:54:41.240447294Z" +} +``` + +Then output becomes as belows +``` +{ + "log": "2015/05/05 19:54:41 \n", + "stream": "stderr", + "docker": { + "id": "df14e0d5ae4c07284fa636d739c8fc2e6b52bc344658de7d3f08c36a2e804115", + } + "kubernetes": { + "host": "jimmi-redhat.localnet", + "pod_name":"fabric8-console-controller-98rqc", + "pod_id": "c76927af-f563-11e4-b32d-54ee7527188d", + "pod_ip": "172.17.0.8", + "container_name": "fabric8-console-container", + "namespace_name": "default", + "namespace_id": "23437884-8e08-4d95-850b-e94378c9b2fd", + "namespace_annotations": { + "fabric8.io/git-commit": "5e1116f63df0bac2a80bdae2ebdc563577bbdf3c" + }, + "namespace_labels": { + "product_version": "v1.0.0" + }, + "labels": { + "component": "fabric8Console" + } + } +} +``` + +If using journal input, from docker configured with `--log-driver=journald`, the input looks like the `journalctl -o export` format: +``` +# The stream identification is encoded into the PRIORITY field as an +# integer: 6, or github.com/coreos/go-systemd/journal.Info, marks stdout, +# while 3, or github.com/coreos/go-systemd/journal.Err, marks stderr. +PRIORITY=6 +CONTAINER_ID=b6cbb6e73c0a +CONTAINER_ID_FULL=b6cbb6e73c0ad63ab820e4baa97cdc77cec729930e38a714826764ac0491341a +CONTAINER_NAME=k8s_registry.a49f5318_docker-registry-1-hhoj0_default_ae3a9bdc-1f66-11e6-80a2-fa163e2fff3a_799e4035 +MESSAGE=172.17.0.1 - - [21/May/2016:16:52:05 +0000] "GET /healthz HTTP/1.1" 200 0 "" "Go-http-client/1.1" +``` + +## Contributing + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Add some feature'`) +4. Test it (`GEM_HOME=vendor bundle install; GEM_HOME=vendor bundle exec rake test`) +5. Push to the branch (`git push origin my-new-feature`) +6. Create new Pull Request + +## Copyright + Copyright (c) 2015 jimmidyson diff --git a/fluent-plugin-kubernetes-metadata-filter/Rakefile b/fluent-plugin-kubernetes-metadata-filter/Rakefile new file mode 100644 index 0000000000..d117a33c1c --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/Rakefile @@ -0,0 +1,37 @@ +require 'bundler/gem_tasks' +require 'rake/testtask' +require 'bump/tasks' + +task :test => [:base_test] + +task :default => [:test, :build] + +desc 'Run test_unit based test' +Rake::TestTask.new(:base_test) do |t| + # To run test for only one file (or file path pattern) + # $ bundle exec rake base_test TEST=test/test_specified_path.rb + # $ bundle exec rake base_test TEST=test/test_*.rb + t.libs << 'test' + t.test_files = Dir['test/**/test_*.rb'].sort + #t.verbose = true + t.warning = false +end + +desc 'Add copyright headers' +task :headers do + require 'rubygems' + require 'copyright_header' + + args = { + :license => 'Apache-2.0', + :copyright_software => 'Fluentd Kubernetes Metadata Filter Plugin', + :copyright_software_description => 'Enrich Fluentd events with Kubernetes metadata', + :copyright_holders => ['Red Hat, Inc.'], + :copyright_years => ['2015-2017'], + :add_path => 'lib:test', + :output_dir => '.' + } + + command_line = CopyrightHeader::CommandLine.new( args ) + command_line.execute +end diff --git a/fluent-plugin-kubernetes-metadata-filter/fluent-plugin-kubernetes-metadata-filter.gemspec b/fluent-plugin-kubernetes-metadata-filter/fluent-plugin-kubernetes-metadata-filter.gemspec new file mode 100644 index 0000000000..e39ebf6f77 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/fluent-plugin-kubernetes-metadata-filter.gemspec @@ -0,0 +1,33 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) + +Gem::Specification.new do |gem| + gem.name = "fluent-plugin-kubernetes-metadata-filter" + gem.version = "2.5.3" + gem.authors = ["Jimmi Dyson"] + gem.email = ["jimmidyson@gmail.com"] + gem.description = %q{Filter plugin to add Kubernetes metadata} + gem.summary = %q{Fluentd filter plugin to add Kubernetes metadata} + gem.homepage = "https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter" + gem.license = "Apache-2.0" + + gem.files = `git ls-files`.split($/) + + gem.required_ruby_version = '>= 2.5.0' + + gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.12'] + gem.add_runtime_dependency "lru_redux" + gem.add_runtime_dependency "kubeclient", '< 5' + + gem.add_development_dependency "bundler", "~> 2.0" + gem.add_development_dependency "rake" + gem.add_development_dependency "minitest", "~> 4.0" + gem.add_development_dependency "test-unit", "~> 3.0.2" + gem.add_development_dependency "test-unit-rr", "~> 1.0.3" + gem.add_development_dependency "copyright-header" + gem.add_development_dependency "webmock" + gem.add_development_dependency "vcr" + gem.add_development_dependency "bump" + gem.add_development_dependency "yajl-ruby" +end diff --git a/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/filter_kubernetes_metadata.rb b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/filter_kubernetes_metadata.rb new file mode 100644 index 0000000000..d0466edb74 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/filter_kubernetes_metadata.rb @@ -0,0 +1,379 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require_relative 'kubernetes_metadata_cache_strategy' +require_relative 'kubernetes_metadata_common' +require_relative 'kubernetes_metadata_stats' +require_relative 'kubernetes_metadata_watch_namespaces' +require_relative 'kubernetes_metadata_watch_pods' + +require 'fluent/plugin/filter' +require 'resolv' + +module Fluent::Plugin + class KubernetesMetadataFilter < Fluent::Plugin::Filter + K8_POD_CA_CERT = 'ca.crt' + K8_POD_TOKEN = 'token' + + include KubernetesMetadata::CacheStrategy + include KubernetesMetadata::Common + include KubernetesMetadata::WatchNamespaces + include KubernetesMetadata::WatchPods + + Fluent::Plugin.register_filter('kubernetes_metadata', self) + + config_param :kubernetes_url, :string, default: nil + config_param :cache_size, :integer, default: 1000 + config_param :cache_ttl, :integer, default: 60 * 60 + config_param :watch, :bool, default: true + config_param :apiVersion, :string, default: 'v1' + config_param :client_cert, :string, default: nil + config_param :client_key, :string, default: nil + config_param :ca_file, :string, default: nil + config_param :verify_ssl, :bool, default: true + config_param :tag_to_kubernetes_name_regexp, + :string, + :default => 'var\.log\.containers\.(?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$' + config_param :bearer_token_file, :string, default: nil + config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount' + config_param :de_dot, :bool, default: true + config_param :de_dot_separator, :string, default: '_' + # if reading from the journal, the record will contain the following fields in the following + # format: + # CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex + # CONTAINER_FULL_ID=dockeridassha256hexvalue + config_param :use_journal, :bool, default: nil + # Field 2 is the container_hash, field 5 is the pod_id, and field 6 is the pod_randhex + # I would have included them as named groups, but you can't have named groups that are + # non-capturing :P + # parse format is defined here: https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317 + config_param :container_name_to_kubernetes_regexp, + :string, + :default => '^(?[^_]+)_(?[^\._]+)(\.(?[^_]+))?_(?[^_]+)_(?[^_]+)_[^_]+_[^_]+$' + + config_param :annotation_match, :array, default: [] + config_param :stats_interval, :integer, default: 30 + config_param :allow_orphans, :bool, default: true + config_param :orphaned_namespace_name, :string, default: '.orphaned' + config_param :orphaned_namespace_id, :string, default: 'orphaned' + config_param :lookup_from_k8s_field, :bool, default: true + # if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want + # to trust the intermediate CA certs we do have, set this to `true` - this corresponds to + # the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN + config_param :ssl_partial_chain, :bool, default: false + config_param :skip_labels, :bool, default: false + config_param :skip_container_metadata, :bool, default: false + config_param :skip_master_url, :bool, default: false + config_param :skip_namespace_metadata, :bool, default: false + # The time interval in seconds for retry backoffs when watch connections fail. + config_param :watch_retry_interval, :integer, default: 1 + # The base number of exponential backoff for retries. + config_param :watch_retry_exponential_backoff_base, :integer, default: 2 + # The maximum number of times to retry pod and namespace watches. + config_param :watch_retry_max_times, :integer, default: 10 + + def fetch_pod_metadata(namespace_name, pod_name) + log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace? + pod_object = @client.get_pod(pod_name, namespace_name) + log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}") if log.trace? + metadata = parse_pod_metadata(pod_object) + @stats.bump(:pod_cache_api_updates) + log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace? + @cache[metadata['pod_id']] = metadata + rescue => e + @stats.bump(:pod_cache_api_nil_error) + log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}" + {} + end + + def dump_stats + @curr_time = Time.now + return if @curr_time.to_i - @prev_time.to_i < @stats_interval + @prev_time = @curr_time + @stats.set(:pod_cache_size, @cache.count) + @stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache + log.info(@stats) + if log.level == Fluent::Log::LEVEL_TRACE + log.trace(" id cache: #{@id_cache.to_a}") + log.trace(" pod cache: #{@cache.to_a}") + log.trace("namespace cache: #{@namespace_cache.to_a}") + end + end + + def fetch_namespace_metadata(namespace_name) + log.trace("fetching namespace metadata: #{namespace_name}") if log.trace? + namespace_object = @client.get_namespace(namespace_name) + log.trace("raw metadata for #{namespace_name}: #{namespace_object}") if log.trace? + metadata = parse_namespace_metadata(namespace_object) + @stats.bump(:namespace_cache_api_updates) + log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace? + @namespace_cache[metadata['namespace_id']] = metadata + rescue => kube_error + @stats.bump(:namespace_cache_api_nil_error) + log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}" + {} + end + + def initialize + super + @prev_time = Time.now + end + + def configure(conf) + super + + def log.trace? + level == Fluent::Log::LEVEL_TRACE + end + + require 'kubeclient' + require 'lru_redux' + @stats = KubernetesMetadata::Stats.new + + if @de_dot && @de_dot_separator.include?(".") + raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'" + end + + if @cache_ttl < 0 + log.info "Setting the cache TTL to :none because it was <= 0" + @cache_ttl = :none + end + + # Caches pod/namespace UID tuples for a given container UID. + @id_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl) + + # Use the container UID as the key to fetch a hash containing pod metadata + @cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl) + + # Use the namespace UID as the key to fetch a hash containing namespace metadata + @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl) + + @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp) + @container_name_to_kubernetes_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_regexp) + + # Use Kubernetes default service account if we're in a pod. + if @kubernetes_url.nil? + log.debug "Kubernetes URL is not set - inspecting environ" + + env_host = ENV['KUBERNETES_SERVICE_HOST'] + env_port = ENV['KUBERNETES_SERVICE_PORT'] + if present?(env_host) && present?(env_port) + if env_host =~ Resolv::IPv6::Regex + # Brackets are needed around IPv6 addresses + env_host = "[#{env_host}]" + end + @kubernetes_url = "https://#{env_host}:#{env_port}/api" + log.debug "Kubernetes URL is now '#{@kubernetes_url}'" + else + log.debug "No Kubernetes URL could be found in config or environ" + end + end + + # Use SSL certificate and bearer token from Kubernetes service account. + if Dir.exist?(@secret_dir) + log.debug "Found directory with secrets: #{@secret_dir}" + ca_cert = File.join(@secret_dir, K8_POD_CA_CERT) + pod_token = File.join(@secret_dir, K8_POD_TOKEN) + + if !present?(@ca_file) and File.exist?(ca_cert) + log.debug "Found CA certificate: #{ca_cert}" + @ca_file = ca_cert + end + + if !present?(@bearer_token_file) and File.exist?(pod_token) + log.debug "Found pod token: #{pod_token}" + @bearer_token_file = pod_token + end + end + + if present?(@kubernetes_url) + ssl_options = { + client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil, + client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil, + ca_file: @ca_file, + verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE + } + + if @ssl_partial_chain + # taken from the ssl.rb OpenSSL::SSL::SSLContext code for DEFAULT_CERT_STORE + require 'openssl' + ssl_store = OpenSSL::X509::Store.new + ssl_store.set_default_paths + if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN + flagval = OpenSSL::X509::V_FLAG_PARTIAL_CHAIN + else + # this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN + flagval = 0x80000 + end + ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval + ssl_options[:cert_store] = ssl_store + end + + auth_options = {} + + if present?(@bearer_token_file) + bearer_token = File.read(@bearer_token_file) + auth_options[:bearer_token] = bearer_token + end + + log.debug "Creating K8S client" + @client = Kubeclient::Client.new( + @kubernetes_url, + @apiVersion, + ssl_options: ssl_options, + auth_options: auth_options, + as: :parsed_symbolized + ) + + begin + @client.api_valid? + rescue KubeException => kube_error + raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{kube_error.message}" + end + + if @watch + pod_thread = Thread.new(self) { |this| this.set_up_pod_thread } + pod_thread.abort_on_exception = true + + namespace_thread = Thread.new(self) { |this| this.set_up_namespace_thread } + namespace_thread.abort_on_exception = true + end + end + @time_fields = [] + @time_fields.push('_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP') if @use_journal || @use_journal.nil? + @time_fields.push('time') unless @use_journal + @time_fields.push('@timestamp') if @lookup_from_k8s_field + + @annotations_regexps = [] + @annotation_match.each do |regexp| + begin + @annotations_regexps << Regexp.compile(regexp) + rescue RegexpError => e + log.error "Error: invalid regular expression in annotation_match: #{e}" + end + end + + end + + def get_metadata_for_record(namespace_name, pod_name, container_name, container_id, create_time, batch_miss_cache) + metadata = { + 'docker' => {'container_id' => container_id}, + 'kubernetes' => { + 'container_name' => container_name, + 'namespace_name' => namespace_name, + 'pod_name' => pod_name + } + } + if present?(@kubernetes_url) + pod_metadata = get_pod_metadata(container_id, namespace_name, pod_name, create_time, batch_miss_cache) + + if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id) && !@skip_container_metadata + metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_id]['image'] + metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_id]['image_id'] + end + + metadata['kubernetes'].merge!(pod_metadata) if pod_metadata + metadata['kubernetes'].delete('containers') + end + metadata + end + + def create_time_from_record(record, internal_time) + time_key = @time_fields.detect{ |ii| record.has_key?(ii) } + time = record[time_key] + if time.nil? || time.chop.empty? + # `internal_time` is a Fluent::EventTime, it can't compare with Time. + return Time.at(internal_time.to_f) + end + if ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP'].include?(time_key) + timei= time.to_i + return Time.at(timei / 1000000, timei % 1000000) + end + return Time.parse(time) + end + + def filter_stream(tag, es) + return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream) + new_es = Fluent::MultiEventStream.new + tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal + tag_metadata = nil + batch_miss_cache = {} + es.each do |time, record| + if tag_match_data && tag_metadata.nil? + tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'], + tag_match_data['docker_id'], create_time_from_record(record, time), batch_miss_cache) + end + metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata + if (@use_journal || @use_journal.nil?) && + (j_metadata = get_metadata_for_journal_record(record, time, batch_miss_cache)) + metadata = j_metadata + end + if @lookup_from_k8s_field && record.has_key?('kubernetes') && record.has_key?('docker') && + record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) && + record['kubernetes'].has_key?('namespace_name') && + record['kubernetes'].has_key?('pod_name') && + record['kubernetes'].has_key?('container_name') && + record['docker'].has_key?('container_id') && + (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'], + record['kubernetes']['container_name'], record['docker']['container_id'], + create_time_from_record(record, time), batch_miss_cache)) + metadata = k_metadata + end + + record = record.merge(metadata) if metadata + new_es.add(time, record) + end + dump_stats + new_es + end + + def get_metadata_for_journal_record(record, time, batch_miss_cache) + metadata = nil + if record.has_key?('CONTAINER_NAME') && record.has_key?('CONTAINER_ID_FULL') + metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data| + get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'], + record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache) + end + unless metadata + log.debug "Error: could not match CONTAINER_NAME from record #{record}" + @stats.bump(:container_name_match_failed) + end + elsif record.has_key?('CONTAINER_NAME') && record['CONTAINER_NAME'].start_with?('k8s_') + log.debug "Error: no container name and id in record #{record}" + @stats.bump(:container_name_id_missing) + end + metadata + end + + def de_dot!(h) + h.keys.each do |ref| + if h[ref] && ref =~ /\./ + v = h.delete(ref) + newref = ref.to_s.gsub('.', @de_dot_separator) + h[newref] = v + end + end + end + + # copied from activesupport + def present?(object) + object.respond_to?(:empty?) ? !object.empty? : !!object + end + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb new file mode 100644 index 0000000000..c1a8c497be --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb @@ -0,0 +1,98 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +module KubernetesMetadata + module CacheStrategy + def get_pod_metadata(key, namespace_name, pod_name, record_create_time, batch_miss_cache) + metadata = {} + ids = @id_cache[key] + if !ids.nil? + # FAST PATH + # Cache hit, fetch metadata from the cache + metadata = @cache.fetch(ids[:pod_id]) do + @stats.bump(:pod_cache_miss) + m = fetch_pod_metadata(namespace_name, pod_name) + (m.nil? || m.empty?) ? {'pod_id'=>ids[:pod_id]} : m + end + metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do + @stats.bump(:namespace_cache_miss) + m = fetch_namespace_metadata(namespace_name) unless @skip_namespace_metadata + (m.nil? || m.empty?) ? {'namespace_id'=>ids[:namespace_id]} : m + end) + else + # SLOW PATH + @stats.bump(:id_cache_miss) + return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}") + pod_metadata = fetch_pod_metadata(namespace_name, pod_name) + if @skip_namespace_metadata + ids = { :pod_id=> pod_metadata['pod_id'] } + @id_cache[key] = ids + return pod_metadata + end + namespace_metadata = fetch_namespace_metadata(namespace_name) + ids = { :pod_id=> pod_metadata['pod_id'], :namespace_id => namespace_metadata['namespace_id'] } + if !ids[:pod_id].nil? && !ids[:namespace_id].nil? + # pod found and namespace found + metadata = pod_metadata + metadata.merge!(namespace_metadata) + else + if ids[:pod_id].nil? && !ids[:namespace_id].nil? + # pod not found, but namespace found + @stats.bump(:id_cache_pod_not_found_namespace) + ns_time = Time.parse(namespace_metadata['creation_timestamp']) + if ns_time <= record_create_time + # namespace is older then record for pod + ids[:pod_id] = key + metadata = @cache.fetch(ids[:pod_id]) do + m = { 'pod_id' => ids[:pod_id] } + end + end + metadata.merge!(namespace_metadata) + else + if !ids[:pod_id].nil? && ids[:namespace_id].nil? + # pod found, but namespace NOT found + # this should NEVER be possible since pod meta can + # only be retrieved with a namespace + @stats.bump(:id_cache_namespace_not_found_pod) + else + # nothing found + @stats.bump(:id_cache_orphaned_record) + end + if @allow_orphans + log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace? + metadata = { + 'orphaned_namespace' => namespace_name, + 'namespace_name' => @orphaned_namespace_name, + 'namespace_id' => @orphaned_namespace_id + } + else + metadata = {} + end + batch_miss_cache["#{namespace_name}_#{pod_name}"] = metadata + end + end + @id_cache[key] = ids unless batch_miss_cache.key?("#{namespace_name}_#{pod_name}") + end + + # remove namespace info that is only used for comparison + metadata.delete('creation_timestamp') + metadata.delete_if{|k,v| v.nil?} + end + + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_common.rb b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_common.rb new file mode 100644 index 0000000000..f8e29c23f5 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_common.rb @@ -0,0 +1,119 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +module KubernetesMetadata + module Common + + class GoneError < StandardError + def initialize(msg="410 Gone") + super + end + end + + def match_annotations(annotations) + result = {} + @annotations_regexps.each do |regexp| + annotations.each do |key, value| + if ::Fluent::StringUtil.match_regexp(regexp, key.to_s) + result[key] = value + end + end + end + result + end + + def parse_namespace_metadata(namespace_object) + labels = String.new + labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless @skip_labels + + annotations = match_annotations(syms_to_strs(namespace_object[:metadata][:annotations].to_h)) + if @de_dot + self.de_dot!(labels) unless @skip_labels + self.de_dot!(annotations) + end + kubernetes_metadata = { + 'namespace_id' => namespace_object[:metadata][:uid], + 'creation_timestamp' => namespace_object[:metadata][:creationTimestamp] + } + kubernetes_metadata['namespace_labels'] = labels unless labels.empty? + kubernetes_metadata['namespace_annotations'] = annotations unless annotations.empty? + kubernetes_metadata + end + + def parse_pod_metadata(pod_object) + labels = String.new + labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless @skip_labels + + annotations = match_annotations(syms_to_strs(pod_object[:metadata][:annotations].to_h)) + if @de_dot + self.de_dot!(labels) unless @skip_labels + self.de_dot!(annotations) + end + + # collect container information + container_meta = {} + begin + pod_object[:status][:containerStatuses].each do|container_status| + # get plain container id (eg. docker://hash -> hash) + container_id = container_status[:containerID].sub /^[-_a-zA-Z0-9]+:\/\//, '' + unless @skip_container_metadata + container_meta[container_id] = { + 'name' => container_status[:name], + 'image' => container_status[:image], + 'image_id' => container_status[:imageID] + } + else + container_meta[container_id] = { + 'name' => container_status[:name] + } + end + end + rescue + log.debug("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]} ") + end + + kubernetes_metadata = { + 'namespace_name' => pod_object[:metadata][:namespace], + 'pod_id' => pod_object[:metadata][:uid], + 'pod_name' => pod_object[:metadata][:name], + 'containers' => syms_to_strs(container_meta), + 'host' => pod_object[:spec][:nodeName] + } + kubernetes_metadata['annotations'] = annotations unless annotations.empty? + kubernetes_metadata['labels'] = labels unless labels.empty? + kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url + kubernetes_metadata + end + + def syms_to_strs(hsh) + newhsh = {} + hsh.each_pair do |kk,vv| + if vv.is_a?(Hash) + vv = syms_to_strs(vv) + end + if kk.is_a?(Symbol) + newhsh[kk.to_s] = vv + else + newhsh[kk] = vv + end + end + newhsh + end + + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_stats.rb b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_stats.rb new file mode 100644 index 0000000000..bb6c416737 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_stats.rb @@ -0,0 +1,46 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'lru_redux' +module KubernetesMetadata + class Stats + + def initialize + @stats = ::LruRedux::TTL::ThreadSafeCache.new(1000, 3600) + end + + def bump(key) + @stats[key] = @stats.getset(key) { 0 } + 1 + end + + def set(key, value) + @stats[key] = value + end + + def [](key) + @stats[key] + end + + def to_s + "stats - " + [].tap do |a| + @stats.each {|k,v| a << "#{k.to_s}: #{v}"} + end.join(', ') + end + + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb new file mode 100644 index 0000000000..10f006830a --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb @@ -0,0 +1,154 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# TODO: this is mostly copy-paste from kubernetes_metadata_watch_pods.rb unify them +require_relative 'kubernetes_metadata_common' + +module KubernetesMetadata + module WatchNamespaces + + include ::KubernetesMetadata::Common + + def set_up_namespace_thread + # Any failures / exceptions in the initial setup should raise + # Fluent:ConfigError, so that users can inspect potential errors in + # the configuration. + namespace_watcher = start_namespace_watch + Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval + Thread.current[:namespace_watch_retry_count] = 0 + + # Any failures / exceptions in the followup watcher notice + # processing will be swallowed and retried. These failures / + # exceptions could be caused by Kubernetes API being temporarily + # down. We assume the configuration is correct at this point. + while true + begin + namespace_watcher ||= get_namespaces_and_start_watcher + process_namespace_watcher_notices(namespace_watcher) + rescue GoneError => e + # Expected error. Quietly go back through the loop in order to + # start watching from the latest resource versions + @stats.bump(:namespace_watch_gone_errors) + log.info("410 Gone encountered. Restarting namespace watch to reset resource versions.", e) + namespace_watcher = nil + rescue => e + @stats.bump(:namespace_watch_failures) + if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times + # Instead of raising exceptions and crashing Fluentd, swallow + # the exception and reset the watcher. + log.info( + "Exception encountered parsing namespace watch event. " \ + "The connection might have been closed. Sleeping for " \ + "#{Thread.current[:namespace_watch_retry_backoff_interval]} " \ + "seconds and resetting the namespace watcher.", e) + sleep(Thread.current[:namespace_watch_retry_backoff_interval]) + Thread.current[:namespace_watch_retry_count] += 1 + Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base + namespace_watcher = nil + else + # Since retries failed for many times, log as errors instead + # of info and raise exceptions and trigger Fluentd to restart. + message = + "Exception encountered parsing namespace watch event. The " \ + "connection might have been closed. Retried " \ + "#{@watch_retry_max_times} times yet still failing. Restarting." + log.error(message, e) + raise Fluent::UnrecoverableError.new(message) + end + end + end + end + + def start_namespace_watch + get_namespaces_and_start_watcher + rescue => e + message = "start_namespace_watch: Exception encountered setting up " \ + "namespace watch from Kubernetes API #{@apiVersion} endpoint " \ + "#{@kubernetes_url}: #{e.message}" + message += " (#{e.response})" if e.respond_to?(:response) + log.debug(message) + + raise Fluent::ConfigError, message + end + + # List all namespaces, record the resourceVersion and return a watcher + # starting from that resourceVersion. + def get_namespaces_and_start_watcher + options = { + resource_version: '0' # Fetch from API server cache instead of etcd quorum read + } + namespaces = @client.get_namespaces(options) + namespaces[:items].each do |namespace| + cache_key = namespace[:metadata][:uid] + @namespace_cache[cache_key] = parse_namespace_metadata(namespace) + @stats.bump(:namespace_cache_host_updates) + end + + # continue watching from most recent resourceVersion + options[:resource_version] = namespaces[:metadata][:resourceVersion] + + watcher = @client.watch_namespaces(options) + reset_namespace_watch_retry_stats + watcher + end + + # Reset namespace watch retry count and backoff interval as there is a + # successful watch notice. + def reset_namespace_watch_retry_stats + Thread.current[:namespace_watch_retry_count] = 0 + Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval + end + + # Process a watcher notice and potentially raise an exception. + def process_namespace_watcher_notices(watcher) + watcher.each do |notice| + case notice[:type] + when 'MODIFIED' + reset_namespace_watch_retry_stats + cache_key = notice[:object][:metadata][:uid] + cached = @namespace_cache[cache_key] + if cached + @namespace_cache[cache_key] = parse_namespace_metadata(notice[:object]) + @stats.bump(:namespace_cache_watch_updates) + else + @stats.bump(:namespace_cache_watch_misses) + end + when 'DELETED' + reset_namespace_watch_retry_stats + # ignore and let age out for cases where + # deleted but still processing logs + @stats.bump(:namespace_cache_watch_deletes_ignored) + when 'ERROR' + if notice[:object] && notice[:object][:code] == 410 + @stats.bump(:namespace_watch_gone_notices) + raise GoneError + else + @stats.bump(:namespace_watch_error_type_notices) + message = notice[:object][:message] if notice[:object] && notice[:object][:message] + raise "Error while watching namespaces: #{message}" + end + else + reset_namespace_watch_retry_stats + # Don't pay attention to creations, since the created namespace may not + # be used by any namespace on this node. + @stats.bump(:namespace_cache_watch_ignored) + end + end + end + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb new file mode 100644 index 0000000000..fa8a87d66c --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb @@ -0,0 +1,173 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# TODO: this is mostly copy-paste from kubernetes_metadata_watch_namespaces.rb unify them +require_relative 'kubernetes_metadata_common' + +module KubernetesMetadata + + module WatchPods + + include ::KubernetesMetadata::Common + + def set_up_pod_thread + # Any failures / exceptions in the initial setup should raise + # Fluent:ConfigError, so that users can inspect potential errors in + # the configuration. + pod_watcher = start_pod_watch + + Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval + Thread.current[:pod_watch_retry_count] = 0 + + # Any failures / exceptions in the followup watcher notice + # processing will be swallowed and retried. These failures / + # exceptions could be caused by Kubernetes API being temporarily + # down. We assume the configuration is correct at this point. + while true + begin + pod_watcher ||= get_pods_and_start_watcher + process_pod_watcher_notices(pod_watcher) + rescue GoneError => e + # Expected error. Quietly go back through the loop in order to + # start watching from the latest resource versions + @stats.bump(:pod_watch_gone_errors) + log.info("410 Gone encountered. Restarting pod watch to reset resource versions.", e) + pod_watcher = nil + rescue => e + @stats.bump(:pod_watch_failures) + if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times + # Instead of raising exceptions and crashing Fluentd, swallow + # the exception and reset the watcher. + log.info( + "Exception encountered parsing pod watch event. The " \ + "connection might have been closed. Sleeping for " \ + "#{Thread.current[:pod_watch_retry_backoff_interval]} " \ + "seconds and resetting the pod watcher.", e) + sleep(Thread.current[:pod_watch_retry_backoff_interval]) + Thread.current[:pod_watch_retry_count] += 1 + Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base + pod_watcher = nil + else + # Since retries failed for many times, log as errors instead + # of info and raise exceptions and trigger Fluentd to restart. + message = + "Exception encountered parsing pod watch event. The " \ + "connection might have been closed. Retried " \ + "#{@watch_retry_max_times} times yet still failing. Restarting." + log.error(message, e) + raise Fluent::UnrecoverableError.new(message) + end + end + end + end + + def start_pod_watch + get_pods_and_start_watcher + rescue => e + message = "start_pod_watch: Exception encountered setting up pod watch " \ + "from Kubernetes API #{@apiVersion} endpoint " \ + "#{@kubernetes_url}: #{e.message}" + message += " (#{e.response})" if e.respond_to?(:response) + log.debug(message) + + raise Fluent::ConfigError, message + end + + # List all pods, record the resourceVersion and return a watcher starting + # from that resourceVersion. + def get_pods_and_start_watcher + options = { + resource_version: '0' # Fetch from API server cache instead of etcd quorum read + } + if ENV['K8S_NODE_NAME'] + options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME'] + end + if @last_seen_resource_version + options[:resource_version] = @last_seen_resource_version + else + pods = @client.get_pods(options) + pods[:items].each do |pod| + cache_key = pod[:metadata][:uid] + @cache[cache_key] = parse_pod_metadata(pod) + @stats.bump(:pod_cache_host_updates) + end + + # continue watching from most recent resourceVersion + options[:resource_version] = pods[:metadata][:resourceVersion] + end + + watcher = @client.watch_pods(options) + reset_pod_watch_retry_stats + watcher + end + + # Reset pod watch retry count and backoff interval as there is a + # successful watch notice. + def reset_pod_watch_retry_stats + Thread.current[:pod_watch_retry_count] = 0 + Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval + end + + # Process a watcher notice and potentially raise an exception. + def process_pod_watcher_notices(watcher) + watcher.each do |notice| + # store version we processed to not reprocess it ... do not unset when there is no version in response + version = ( # TODO: replace with &.dig once we are on ruby 2.5+ + notice[:object] && notice[:object][:metadata] && notice[:object][:metadata][:resourceVersion] + ) + @last_seen_resource_version = version if version + + case notice[:type] + when 'MODIFIED' + reset_pod_watch_retry_stats + cache_key = notice.dig(:object, :metadata, :uid) + cached = @cache[cache_key] + if cached + @cache[cache_key] = parse_pod_metadata(notice[:object]) + @stats.bump(:pod_cache_watch_updates) + elsif ENV['K8S_NODE_NAME'] == notice[:object][:spec][:nodeName] then + @cache[cache_key] = parse_pod_metadata(notice[:object]) + @stats.bump(:pod_cache_host_updates) + else + @stats.bump(:pod_cache_watch_misses) + end + when 'DELETED' + reset_pod_watch_retry_stats + # ignore and let age out for cases where pods + # deleted but still processing logs + @stats.bump(:pod_cache_watch_delete_ignored) + when 'ERROR' + if notice[:object] && notice[:object][:code] == 410 + @last_seen_resource_version = nil # requested resourceVersion was too old, need to reset + @stats.bump(:pod_watch_gone_notices) + raise GoneError + else + @stats.bump(:pod_watch_error_type_notices) + message = notice[:object][:message] if notice[:object] && notice[:object][:message] + raise "Error while watching pods: #{message}" + end + else + reset_pod_watch_retry_stats + # Don't pay attention to creations, since the created pod may not + # end up on this node. + @stats.bump(:pod_cache_watch_ignored) + end + end + end + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/invalid_api_server_config.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/invalid_api_server_config.yml new file mode 100644 index 0000000000..0284c84d00 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/invalid_api_server_config.yml @@ -0,0 +1,53 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + Authorization: + - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWF + response: + status: + code: 401 + message: Unauthorized + headers: + Content-Type: + - text/plain; charset=utf-8 + Date: + - Sat, 09 May 2015 14:04:39 GMT + Content-Length: + - '13' + body: + encoding: UTF-8 + string: | + Unauthorized + http_version: + recorded_at: Sat, 09 May 2015 14:04:39 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_docker_metadata_annotations.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_docker_metadata_annotations.yml new file mode 100644 index 0000000000..a8d0c64f8f --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_docker_metadata_annotations.yml @@ -0,0 +1,205 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "fabric8-console-controller-98rqc", + "generateName": "fabric8-console-controller-", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc", + "uid": "c76927af-f563-11e4-b32d-54ee7527188d", + "resourceVersion": "122", + "creationTimestamp": "2015-05-08T09:22:42Z", + "labels": { + "component": "fabric8Console" + }, + "annotations": { + "kubernetes.io/config.hash": "c171c44f5b9345c6dc17b0e95030318c", + "kubernetes.io/config.mirror": "c171c44f5b9345c6dc17b0e95030318c", + "kubernetes.io/config.seen": "2016-06-06T08:08:35.680437994Z", + "kubernetes.io/config.source": "file", + "custom.field1": "hello_kitty", + "field.two": "value" + } + }, + "spec": { + "volumes": [ + { + "name": "openshift-cert-secrets", + "hostPath": null, + "emptyDir": null, + "gcePersistentDisk": null, + "gitRepo": null, + "secret": { + "secretName": "openshift-cert-secrets" + }, + "nfs": null, + "iscsi": null, + "glusterfs": null + } + ], + "containers": [ + { + "name": "fabric8-console-container", + "image": "fabric8/hawtio-kubernetes:latest", + "ports": [ + { + "containerPort": 9090, + "protocol": "TCP" + } + ], + "env": [ + { + "name": "OAUTH_CLIENT_ID", + "value": "fabric8-console" + }, + { + "name": "OAUTH_AUTHORIZE_URI", + "value": "https://localhost:8443/oauth/authorize" + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "openshift-cert-secrets", + "readOnly": true, + "mountPath": "/etc/secret-volume" + } + ], + "terminationMessagePath": "/dev/termination-log", + "imagePullPolicy": "IfNotPresent", + "capabilities": {} + } + ], + "restartPolicy": "Always", + "dnsPolicy": "ClusterFirst", + "nodeName": "jimmi-redhat.localnet" + }, + "status": { + "phase": "Running", + "Condition": [ + { + "type": "Ready", + "status": "True" + } + ], + "hostIP": "172.17.42.1", + "podIP": "172.17.0.8", + "containerStatuses": [ + { + "name": "fabric8-console-container", + "state": { + "running": { + "startedAt": "2015-05-08T09:22:44Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "fabric8/hawtio-kubernetes:latest", + "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303", + "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459" + } + ] + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/default + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Namespace", + "apiVersion": "v1", + "metadata": { + "name": "default", + "selfLink": "/api/v1/namespaces/default", + "uid": "898268c8-4a36-11e5-9d81-42010af0194c", + "resourceVersion": "6", + "creationTimestamp": "2015-05-08T09:22:01Z", + "annotations": { + "workspaceId": "myWorkspaceName" + } + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_docker_metadata_dotted_labels.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_docker_metadata_dotted_labels.yml new file mode 100644 index 0000000000..1a590cbbdb --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_docker_metadata_dotted_labels.yml @@ -0,0 +1,197 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "fabric8-console-controller-98rqc", + "generateName": "fabric8-console-controller-", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc", + "uid": "c76927af-f563-11e4-b32d-54ee7527188d", + "resourceVersion": "122", + "creationTimestamp": "2015-05-08T09:22:42Z", + "labels": { + "kubernetes.io/test": "somevalue" + } + }, + "spec": { + "volumes": [ + { + "name": "openshift-cert-secrets", + "hostPath": null, + "emptyDir": null, + "gcePersistentDisk": null, + "gitRepo": null, + "secret": { + "secretName": "openshift-cert-secrets" + }, + "nfs": null, + "iscsi": null, + "glusterfs": null + } + ], + "containers": [ + { + "name": "fabric8-console-container", + "image": "fabric8/hawtio-kubernetes:latest", + "ports": [ + { + "containerPort": 9090, + "protocol": "TCP" + } + ], + "env": [ + { + "name": "OAUTH_CLIENT_ID", + "value": "fabric8-console" + }, + { + "name": "OAUTH_AUTHORIZE_URI", + "value": "https://localhost:8443/oauth/authorize" + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "openshift-cert-secrets", + "readOnly": true, + "mountPath": "/etc/secret-volume" + } + ], + "terminationMessagePath": "/dev/termination-log", + "imagePullPolicy": "IfNotPresent", + "capabilities": {} + } + ], + "restartPolicy": "Always", + "dnsPolicy": "ClusterFirst", + "nodeName": "jimmi-redhat.localnet" + }, + "status": { + "phase": "Running", + "Condition": [ + { + "type": "Ready", + "status": "True" + } + ], + "hostIP": "172.17.42.1", + "podIP": "172.17.0.8", + "containerStatuses": [ + { + "name": "fabric8-console-container", + "state": { + "running": { + "startedAt": "2015-05-08T09:22:44Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "fabric8/hawtio-kubernetes:latest", + "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303", + "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459" + } + ] + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/default + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Namespace", + "apiVersion": "v1", + "metadata": { + "name": "default", + "selfLink": "/api/v1/namespaces/default", + "uid": "898268c8-4a36-11e5-9d81-42010af0194c", + "resourceVersion": "6", + "creationTimestamp": "2015-05-08T09:22:01Z", + "labels": { + "kubernetes.io/namespacetest": "somevalue" + } + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_api_v1.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_api_v1.yml new file mode 100644 index 0000000000..55f640eefe --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_api_v1.yml @@ -0,0 +1,193 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1 + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "APIResourceList", + "groupVersion": "v1", + "resources": [ + { + "name": "bindings", + "singularName": "", + "namespaced": true, + "kind": "Binding", + "verbs": [ + "create" + ] + }, + { + "name": "namespaces", + "singularName": "", + "namespaced": false, + "kind": "Namespace", + "verbs": [ + "create", + "delete", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ns" + ] + }, + { + "name": "namespaces/finalize", + "singularName": "", + "namespaced": false, + "kind": "Namespace", + "verbs": [ + "update" + ] + }, + { + "name": "namespaces/status", + "singularName": "", + "namespaced": false, + "kind": "Namespace", + "verbs": [ + "get", + "patch", + "update" + ] + }, + { + "name": "pods", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "po" + ], + "categories": [ + "all" + ] + }, + { + "name": "pods/attach", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [] + }, + { + "name": "pods/binding", + "singularName": "", + "namespaced": true, + "kind": "Binding", + "verbs": [ + "create" + ] + }, + { + "name": "pods/eviction", + "singularName": "", + "namespaced": true, + "group": "policy", + "version": "v1beta1", + "kind": "Eviction", + "verbs": [ + "create" + ] + }, + { + "name": "pods/exec", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [] + }, + { + "name": "pods/log", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [ + "get" + ] + }, + { + "name": "pods/portforward", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [] + }, + { + "name": "pods/proxy", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [] + }, + { + "name": "pods/status", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [ + "get", + "patch", + "update" + ] + } + ] + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_api_v1_using_token.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_api_v1_using_token.yml new file mode 100644 index 0000000000..6c2cf9d4b1 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_api_v1_using_token.yml @@ -0,0 +1,195 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1 + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + Authorization: + - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "APIResourceList", + "groupVersion": "v1", + "resources": [ + { + "name": "bindings", + "singularName": "", + "namespaced": true, + "kind": "Binding", + "verbs": [ + "create" + ] + }, + { + "name": "namespaces", + "singularName": "", + "namespaced": false, + "kind": "Namespace", + "verbs": [ + "create", + "delete", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ns" + ] + }, + { + "name": "namespaces/finalize", + "singularName": "", + "namespaced": false, + "kind": "Namespace", + "verbs": [ + "update" + ] + }, + { + "name": "namespaces/status", + "singularName": "", + "namespaced": false, + "kind": "Namespace", + "verbs": [ + "get", + "patch", + "update" + ] + }, + { + "name": "pods", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "po" + ], + "categories": [ + "all" + ] + }, + { + "name": "pods/attach", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [] + }, + { + "name": "pods/binding", + "singularName": "", + "namespaced": true, + "kind": "Binding", + "verbs": [ + "create" + ] + }, + { + "name": "pods/eviction", + "singularName": "", + "namespaced": true, + "group": "policy", + "version": "v1beta1", + "kind": "Eviction", + "verbs": [ + "create" + ] + }, + { + "name": "pods/exec", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [] + }, + { + "name": "pods/log", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [ + "get" + ] + }, + { + "name": "pods/portforward", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [] + }, + { + "name": "pods/proxy", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [] + }, + { + "name": "pods/status", + "singularName": "", + "namespaced": true, + "kind": "Pod", + "verbs": [ + "get", + "patch", + "update" + ] + } + ] + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_namespace_default.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_namespace_default.yml new file mode 100644 index 0000000000..619083f52e --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_namespace_default.yml @@ -0,0 +1,69 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/default + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Namespace", + "apiVersion": "v1", + "metadata": { + "name": "default", + "selfLink": "/api/v1/namespaces/default", + "uid": "898268c8-4a36-11e5-9d81-42010af0194c", + "resourceVersion": "6", + "creationTimestamp": "2015-05-08T09:22:01Z" + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_namespace_default_using_token.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_namespace_default_using_token.yml new file mode 100644 index 0000000000..144acc29db --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_namespace_default_using_token.yml @@ -0,0 +1,71 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/default + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + Authorization: + - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Namespace", + "apiVersion": "v1", + "metadata": { + "name": "default", + "selfLink": "/api/v1/namespaces/default", + "uid": "898268c8-4a36-11e5-9d81-42010af0194c", + "resourceVersion": "6", + "creationTimestamp": "2015-05-08T09:22:01Z" + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_pod.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_pod.yml new file mode 100644 index 0000000000..651b0d22d8 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_pod.yml @@ -0,0 +1,146 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "fabric8-console-controller-98rqc", + "generateName": "fabric8-console-controller-", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc", + "uid": "c76927af-f563-11e4-b32d-54ee7527188d", + "resourceVersion": "122", + "creationTimestamp": "2015-05-08T09:22:42Z", + "labels": { + "component": "fabric8Console" + } + }, + "spec": { + "volumes": [ + { + "name": "openshift-cert-secrets", + "hostPath": null, + "emptyDir": null, + "gcePersistentDisk": null, + "gitRepo": null, + "secret": { + "secretName": "openshift-cert-secrets" + }, + "nfs": null, + "iscsi": null, + "glusterfs": null + } + ], + "containers": [ + { + "name": "fabric8-console-container", + "image": "fabric8/hawtio-kubernetes:latest", + "ports": [ + { + "containerPort": 9090, + "protocol": "TCP" + } + ], + "env": [ + { + "name": "OAUTH_CLIENT_ID", + "value": "fabric8-console" + }, + { + "name": "OAUTH_AUTHORIZE_URI", + "value": "https://localhost:8443/oauth/authorize" + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "openshift-cert-secrets", + "readOnly": true, + "mountPath": "/etc/secret-volume" + } + ], + "terminationMessagePath": "/dev/termination-log", + "imagePullPolicy": "IfNotPresent", + "capabilities": {} + } + ], + "restartPolicy": "Always", + "dnsPolicy": "ClusterFirst", + "nodeName": "jimmi-redhat.localnet" + }, + "status": { + "phase": "Running", + "Condition": [ + { + "type": "Ready", + "status": "True" + } + ], + "hostIP": "172.17.42.1", + "podIP": "172.17.0.8", + "containerStatuses": [ + { + "name": "fabric8-console-container", + "state": { + "running": { + "startedAt": "2015-05-08T09:22:44Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "fabric8/hawtio-kubernetes:latest", + "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303", + "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459" + } + ] + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_pod_using_token.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_pod_using_token.yml new file mode 100644 index 0000000000..31238adef5 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/kubernetes_get_pod_using_token.yml @@ -0,0 +1,148 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + Authorization: + - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "fabric8-console-controller-98rqc", + "generateName": "fabric8-console-controller-", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc", + "uid": "c76927af-f563-11e4-b32d-54ee7527188d", + "resourceVersion": "122", + "creationTimestamp": "2015-05-08T09:22:42Z", + "labels": { + "component": "fabric8Console" + } + }, + "spec": { + "volumes": [ + { + "name": "openshift-cert-secrets", + "hostPath": null, + "emptyDir": null, + "gcePersistentDisk": null, + "gitRepo": null, + "secret": { + "secretName": "openshift-cert-secrets" + }, + "nfs": null, + "iscsi": null, + "glusterfs": null + } + ], + "containers": [ + { + "name": "fabric8-console-container", + "image": "fabric8/hawtio-kubernetes:latest", + "ports": [ + { + "containerPort": 9090, + "protocol": "TCP" + } + ], + "env": [ + { + "name": "OAUTH_CLIENT_ID", + "value": "fabric8-console" + }, + { + "name": "OAUTH_AUTHORIZE_URI", + "value": "https://localhost:8443/oauth/authorize" + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "openshift-cert-secrets", + "readOnly": true, + "mountPath": "/etc/secret-volume" + } + ], + "terminationMessagePath": "/dev/termination-log", + "imagePullPolicy": "IfNotPresent", + "capabilities": {} + } + ], + "restartPolicy": "Always", + "dnsPolicy": "ClusterFirst", + "nodeName": "jimmi-redhat.localnet" + }, + "status": { + "phase": "Running", + "Condition": [ + { + "type": "Ready", + "status": "True" + } + ], + "hostIP": "172.17.42.1", + "podIP": "172.17.0.8", + "containerStatuses": [ + { + "name": "fabric8-console-container", + "state": { + "running": { + "startedAt": "2015-05-08T09:22:44Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "fabric8/hawtio-kubernetes:latest", + "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303", + "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459" + } + ] + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/metadata_from_tag_and_journald_fields.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/metadata_from_tag_and_journald_fields.yml new file mode 100644 index 0000000000..7374233bbf --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/metadata_from_tag_and_journald_fields.yml @@ -0,0 +1,153 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/journald-namespace-name/pods/journald-pod-name + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "journald-pod-name", + "generateName": "journald-pod-name-", + "namespace": "journald-namespace-name", + "selfLink": "/api/v1/namespaces/journald-namespace-name/pods/journald-pod-name", + "uid": "5e1c1e27-b637-4e81-80b6-6d8a8c404d3b", + "resourceVersion": "122", + "creationTimestamp": "2015-05-08T09:22:42Z", + "labels": { + "component": "journald-test" + } + }, + "spec": { + "containers": [ + { + "name": "journald-container-name", + "image": "journald-container-image:latest", + "terminationMessagePath": "/dev/termination-log", + "imagePullPolicy": "IfNotPresent", + "capabilities": {} + } + ], + "nodeName": "jimmi-redhat.localnet" + }, + "status": { + "phase": "Running", + "Condition": [ + { + "type": "Ready", + "status": "True" + } + ], + "hostIP": "172.17.42.1", + "podIP": "172.17.0.8", + "containerStatuses": [ + { + "name": "journald-container-name", + "state": { + "running": { + "startedAt": "2015-05-08T09:22:44Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "journald-container-image:latest", + "imageID": "docker://dda4c95705fb7050b701b10a7fe928ca5bc971a1dcb521ae3c339194cbf36b47", + "containerID": "docker://838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b" + } + ] + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/journald-namespace-name + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Namespace", + "apiVersion": "v1", + "metadata": { + "name": "journald-namespace-name", + "selfLink": "/api/v1/namespaces/journald-namespace-name", + "uid": "8282888f-733f-4f23-a3d3-1fdfa3bdacf2", + "resourceVersion": "6", + "creationTimestamp": "2015-05-08T09:22:01Z" + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml new file mode 100644 index 0000000000..cffddca6fe --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml @@ -0,0 +1,285 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/journald-namespace-name/pods/journald-pod-name + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "journald-pod-name", + "generateName": "journald-pod-name-", + "namespace": "journald-namespace-name", + "selfLink": "/api/v1/namespaces/journald-namespace-name/pods/journald-pod-name", + "uid": "5e1c1e27-b637-4e81-80b6-6d8a8c404d3b", + "resourceVersion": "122", + "creationTimestamp": "2015-05-08T09:22:42Z", + "labels": { + "component": "journald-test" + } + }, + "spec": { + "containers": [ + { + "name": "journald-container-name", + "image": "journald-container-image:latest", + "terminationMessagePath": "/dev/termination-log", + "imagePullPolicy": "IfNotPresent", + "capabilities": {} + } + ], + "nodeName": "jimmi-redhat.localnet" + }, + "status": { + "phase": "Running", + "Condition": [ + { + "type": "Ready", + "status": "True" + } + ], + "hostIP": "172.17.42.1", + "podIP": "172.17.0.8", + "containerStatuses": [ + { + "name": "journald-container-name", + "state": { + "running": { + "startedAt": "2015-05-08T09:22:44Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "journald-container-image:latest", + "imageID": "docker://dda4c95705fb7050b701b10a7fe928ca5bc971a1dcb521ae3c339194cbf36b47", + "containerID": "docker://838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b" + } + ] + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/journald-namespace-name + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Namespace", + "apiVersion": "v1", + "metadata": { + "name": "journald-namespace-name", + "selfLink": "/api/v1/namespaces/journald-namespace-name", + "uid": "8282888f-733f-4f23-a3d3-1fdfa3bdacf2", + "resourceVersion": "6", + "creationTimestamp": "2015-05-08T09:22:01Z" + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/k8s-namespace-name/pods/k8s-pod-name + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "k8s-pod-name", + "generateName": "k8s-pod-name-", + "namespace": "k8s-namespace-name", + "selfLink": "/api/v1/namespaces/k8s-namespace-name/pods/k8s-pod-name", + "uid": "ebabf749-5fcd-4750-a3f0-aedd89476da8", + "resourceVersion": "122", + "creationTimestamp": "2015-05-08T09:22:42Z", + "labels": { + "component": "k8s-test" + } + }, + "spec": { + "containers": [ + { + "name": "k8s-container-name", + "image": "k8s-container-image:latest", + "terminationMessagePath": "/dev/termination-log", + "imagePullPolicy": "IfNotPresent", + "capabilities": {} + } + ], + "nodeName": "jimmi-redhat.localnet" + }, + "status": { + "phase": "Running", + "Condition": [ + { + "type": "Ready", + "status": "True" + } + ], + "hostIP": "172.17.42.1", + "podIP": "172.17.0.8", + "containerStatuses": [ + { + "name": "k8s-container-name", + "state": { + "running": { + "startedAt": "2015-05-08T09:22:44Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "k8s-container-image:latest", + "imageID": "docker://d78c5217c41e9af08d37d9ae2cb070afa1fe3da6bc77bfb18879a8b4bfdf8a34", + "containerID": "docker://e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95" + } + ] + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +- request: + method: get + uri: https://localhost:8443/api/v1/namespaces/k8s-namespace-name + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:35:37 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: |- + { + "kind": "Namespace", + "apiVersion": "v1", + "metadata": { + "name": "k8s-namespace-name", + "selfLink": "/api/v1/namespaces/k8s-namespace-name", + "uid": "8e0dc8fc-59f2-49f7-a3e2-ed0913e19d9f", + "resourceVersion": "6", + "creationTimestamp": "2015-05-08T09:22:01Z" + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + } + http_version: + recorded_at: Fri, 08 May 2015 10:35:37 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/valid_kubernetes_api_server.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/valid_kubernetes_api_server.yml new file mode 100644 index 0000000000..7110693fe2 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/valid_kubernetes_api_server.yml @@ -0,0 +1,55 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:13:54 GMT + Content-Length: + - '67' + body: + encoding: UTF-8 + string: |- + { + "versions": [ + "v1" + ] + } + http_version: + recorded_at: Fri, 08 May 2015 10:13:54 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/cassettes/valid_kubernetes_api_server_using_token.yml b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/valid_kubernetes_api_server_using_token.yml new file mode 100644 index 0000000000..ea3efc15cb --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/cassettes/valid_kubernetes_api_server_using_token.yml @@ -0,0 +1,57 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +http_interactions: +- request: + method: get + uri: https://localhost:8443/api + body: + encoding: US-ASCII + string: '' + headers: + Accept: + - "*/*; q=0.5, application/xml" + Accept-Encoding: + - gzip, deflate + User-Agent: + - Ruby + Authorization: + - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Date: + - Fri, 08 May 2015 10:13:54 GMT + Content-Length: + - '67' + body: + encoding: UTF-8 + string: |- + { + "versions": [ + "v1" + ] + } + http_version: + recorded_at: Fri, 08 May 2015 10:13:54 GMT +recorded_with: VCR 2.9.3 diff --git a/fluent-plugin-kubernetes-metadata-filter/test/helper.rb b/fluent-plugin-kubernetes-metadata-filter/test/helper.rb new file mode 100644 index 0000000000..175f37d235 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/helper.rb @@ -0,0 +1,80 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'bundler/setup' +require 'codeclimate-test-reporter' +SimpleCov.start do + formatter SimpleCov::Formatter::MultiFormatter.new [ + SimpleCov::Formatter::HTMLFormatter, + CodeClimate::TestReporter::Formatter + ] +end + +require 'rr' +require 'test/unit' +require 'test/unit/rr' +require 'fileutils' +require 'fluent/log' +require 'fluent/test' +require 'minitest/autorun' +require 'vcr' +require 'ostruct' +require 'fluent/plugin/filter_kubernetes_metadata' +require 'fluent/test/driver/filter' +require 'kubeclient' + +require 'webmock/test_unit' +WebMock.disable_net_connect! + +VCR.configure do |config| + config.cassette_library_dir = 'test/cassettes' + config.hook_into :webmock # or :fakeweb + config.ignore_hosts 'codeclimate.com' +end + +unless defined?(Test::Unit::AssertionFailedError) + class Test::Unit::AssertionFailedError < StandardError + end +end + +def unused_port + s = TCPServer.open(0) + port = s.addr[1] + s.close + port +end + +def ipv6_enabled? + require 'socket' + + begin + TCPServer.open('::1', 0) + true + rescue + false + end +end + +# TEST_NAME='foo' ruby test_file.rb to run a single test case +if ENV["TEST_NAME"] + (class << Test::Unit::TestCase; self; end).prepend(Module.new do + def test(name) + super if name == ENV["TEST_NAME"] + end + end) +end diff --git a/fluent-plugin-kubernetes-metadata-filter/test/plugin/test.token b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test.token new file mode 100644 index 0000000000..bd41cba781 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test.token @@ -0,0 +1 @@ +12345 \ No newline at end of file diff --git a/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_cache_stats.rb b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_cache_stats.rb new file mode 100644 index 0000000000..fde588433d --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_cache_stats.rb @@ -0,0 +1,33 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require_relative '../helper' + +class KubernetesMetadataCacheStatsTest < Test::Unit::TestCase + + test 'watch stats' do + require 'lru_redux' + stats = KubernetesMetadata::Stats.new + stats.bump(:missed) + stats.bump(:deleted) + stats.bump(:deleted) + + assert_equal("stats - deleted: 2, missed: 1", stats.to_s) + end + +end diff --git a/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_cache_strategy.rb b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_cache_strategy.rb new file mode 100644 index 0000000000..11498447d2 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_cache_strategy.rb @@ -0,0 +1,193 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require_relative '../helper' + +class TestCacheStrategy + include KubernetesMetadata::CacheStrategy + + def initialize + @stats = KubernetesMetadata::Stats.new + @cache = LruRedux::TTL::ThreadSafeCache.new(100,3600) + @id_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600) + @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600) + @orphaned_namespace_name = '.orphaned' + @orphaned_namespace_id = 'orphaned' + end + + attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans + + def fetch_pod_metadata(namespace_name, pod_name) + {} + end + + def fetch_namespace_metadata(namespace_name) + {} + end + + def log + logger = {} + def logger.trace? + true + end + def logger.trace(message) + end + logger + end + +end + +class KubernetesMetadataCacheStrategyTest < Test::Unit::TestCase + + def setup + @strategy = TestCacheStrategy.new + @cache_key = 'some_long_container_id' + @namespace_name = 'some_namespace_name' + @namespace_uuid = 'some_namespace_uuid' + @pod_name = 'some_pod_name' + @pod_uuid = 'some_pod_uuid' + @time = Time.now + @pod_meta = {'pod_id'=> @pod_uuid, 'labels'=> {'meta'=>'pod'}} + @namespace_meta = {'namespace_id'=> @namespace_uuid, 'creation_timestamp'=>@time.to_s} + end + + test 'when cached metadata is found' do + exp = @pod_meta.merge(@namespace_meta) + exp.delete('creation_timestamp') + @strategy.id_cache[@cache_key] = { + pod_id: @pod_uuid, + namespace_id: @namespace_uuid + } + @strategy.cache[@pod_uuid] = @pod_meta + @strategy.namespace_cache[@namespace_uuid] = @namespace_meta + assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {})) + end + + test 'when previously processed record for pod but metadata is not cached and can not be fetched' do + exp = { + 'pod_id'=> @pod_uuid, + 'namespace_id'=> @namespace_uuid + } + @strategy.id_cache[@cache_key] = { + pod_id: @pod_uuid, + namespace_id: @namespace_uuid + } + @strategy.stub :fetch_pod_metadata, {} do + @strategy.stub :fetch_namespace_metadata, nil do + assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {})) + end + end + end + + test 'when metadata is not cached and is fetched' do + exp = @pod_meta.merge(@namespace_meta) + exp.delete('creation_timestamp') + @strategy.stub :fetch_pod_metadata, @pod_meta do + @strategy.stub :fetch_namespace_metadata, @namespace_meta do + assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {})) + assert_true(@strategy.id_cache.key?(@cache_key)) + end + end + end + + test 'when metadata is not cached and pod is deleted and namespace metadata is fetched' do + # this is the case for a record from a deleted pod where no other + # records were read. using the container hash since that is all + # we ever will have and should allow us to process all the deleted + # pod records + exp = { + 'pod_id'=> @cache_key, + 'namespace_id'=> @namespace_uuid + } + @strategy.stub :fetch_pod_metadata, {} do + @strategy.stub :fetch_namespace_metadata, @namespace_meta do + assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {})) + assert_true(@strategy.id_cache.key?(@cache_key)) + end + end + end + + test 'when metadata is not cached and pod is deleted and namespace is for a different namespace with the same name' do + # this is the case for a record from a deleted pod from a deleted namespace + # where new namespace was created with the same name + exp = { + 'namespace_id'=> @namespace_uuid + } + @strategy.stub :fetch_pod_metadata, {} do + @strategy.stub :fetch_namespace_metadata, @namespace_meta do + assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {})) + assert_true(@strategy.id_cache.key?(@cache_key)) + end + end + end + + test 'when metadata is not cached and no metadata can be fetched and not allowing orphans' do + # we should never see this since pod meta should not be retrievable + # unless the namespace exists + @strategy.stub :fetch_pod_metadata, @pod_meta do + @strategy.stub :fetch_namespace_metadata, {} do + assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {})) + end + end + end + + test 'when metadata is not cached and no metadata can be fetched and allowing orphans' do + # we should never see this since pod meta should not be retrievable + # unless the namespace exists + @strategy.allow_orphans = true + exp = { + 'orphaned_namespace' => 'namespace', + 'namespace_name' => '.orphaned', + 'namespace_id' => 'orphaned' + } + @strategy.stub :fetch_pod_metadata, @pod_meta do + @strategy.stub :fetch_namespace_metadata, {} do + assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {})) + end + end + end + + test 'when metadata is not cached and no metadata can be fetched and not allowing orphans for multiple records' do + # processing a batch of records with no meta. ideally we only hit the api server once + batch_miss_cache = {} + @strategy.stub :fetch_pod_metadata, {} do + @strategy.stub :fetch_namespace_metadata, {} do + assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache)) + end + end + assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache)) + end + + test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do + # we should never see this since pod meta should not be retrievable + # unless the namespace exists + @strategy.allow_orphans = true + exp = { + 'orphaned_namespace' => 'namespace', + 'namespace_name' => '.orphaned', + 'namespace_id' => 'orphaned' + } + batch_miss_cache = {} + @strategy.stub :fetch_pod_metadata, {} do + @strategy.stub :fetch_namespace_metadata, {} do + assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache)) + end + end + assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache)) + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_filter_kubernetes_metadata.rb b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_filter_kubernetes_metadata.rb new file mode 100644 index 0000000000..b12f278a0f --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_filter_kubernetes_metadata.rb @@ -0,0 +1,999 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require_relative '../helper' + +class KubernetesMetadataFilterTest < Test::Unit::TestCase + include Fluent + + setup do + Fluent::Test.setup + @time = Fluent::Engine.now + end + + DEFAULT_TAG = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log' + + def create_driver(conf = '') + Test::Driver::Filter.new(Plugin::KubernetesMetadataFilter).configure(conf) + end + + sub_test_case 'configure' do + + test 'check default' do + d = create_driver + assert_equal(1000, d.instance.cache_size) + end + + test 'kubernetes url' do + VCR.use_cassette('valid_kubernetes_api_server') do + d = create_driver(' + kubernetes_url https://localhost:8443 + watch false + ') + assert_equal('https://localhost:8443', d.instance.kubernetes_url) + assert_equal(1000, d.instance.cache_size) + end + end + + test 'cache size' do + VCR.use_cassette('valid_kubernetes_api_server') do + d = create_driver(' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ') + assert_equal('https://localhost:8443', d.instance.kubernetes_url) + assert_equal(1, d.instance.cache_size) + end + end + + test 'invalid API server config' do + VCR.use_cassette('invalid_api_server_config') do + assert_raise Fluent::ConfigError do + create_driver(' + kubernetes_url https://localhost:8443 + bearer_token_file test/plugin/test.token + watch false + verify_ssl false + ') + end + end + end + + test 'service account credentials' do + VCR.use_cassette('valid_kubernetes_api_server') do + begin + ENV['KUBERNETES_SERVICE_HOST'] = 'localhost' + ENV['KUBERNETES_SERVICE_PORT'] = '8443' + + Dir.mktmpdir { |dir| + # Fake token file and CA crt. + expected_cert_path = File.join(dir, Plugin::KubernetesMetadataFilter::K8_POD_CA_CERT) + expected_token_path = File.join(dir, Plugin::KubernetesMetadataFilter::K8_POD_TOKEN) + + File.open(expected_cert_path, "w") {} + File.open(expected_token_path, "w") {} + + d = create_driver(" + watch false + secret_dir #{dir} + ") + + assert_equal(d.instance.kubernetes_url, "https://localhost:8443/api") + assert_equal(d.instance.ca_file, expected_cert_path) + assert_equal(d.instance.bearer_token_file, expected_token_path) + } + ensure + ENV['KUBERNETES_SERVICE_HOST'] = nil + ENV['KUBERNETES_SERVICE_PORT'] = nil + end + end + end + + test 'service account credential files are tested for existence' do + VCR.use_cassette('valid_kubernetes_api_server') do + begin + ENV['KUBERNETES_SERVICE_HOST'] = 'localhost' + ENV['KUBERNETES_SERVICE_PORT'] = '8443' + + Dir.mktmpdir { |dir| + d = create_driver(" + watch false + secret_dir #{dir} + ") + assert_equal(d.instance.kubernetes_url, "https://localhost:8443/api") + assert_nil(d.instance.ca_file, nil) + assert_nil(d.instance.bearer_token_file) + } + ensure + ENV['KUBERNETES_SERVICE_HOST'] = nil + ENV['KUBERNETES_SERVICE_PORT'] = nil + end + end + end + end + + sub_test_case 'filter_stream' do + + def emit(msg={}, config=' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ', d: nil) + d = create_driver(config) if d.nil? + d.run(default_tag: DEFAULT_TAG) { + d.feed(@time, msg) + } + d.filtered.map{|e| e.last} + end + + def emit_with_tag(tag, msg={}, config=' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ') + d = create_driver(config) + d.run(default_tag: tag) { + d.feed(@time, msg) + } + d.filtered.map{|e| e.last} + end + + test 'nil event stream' do + #not certain how this is possible but adding test to properly + #guard against this condition we have seen - test for nil, + #empty, no empty method, not an event stream + plugin = create_driver.instance + plugin.filter_stream('tag', nil) + plugin.filter_stream('tag', Fluent::MultiEventStream.new) + end + + test 'inability to connect to the api server handles exception and doensnt block pipeline' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}]) do + driver = create_driver(' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ') + stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc').to_raise(SocketError.new('error from pod fetch')) + stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_raise(SocketError.new('socket error from namespace fetch')) + filtered = emit({'time'=>'2015-05-08T09:22:01Z'}, '', :d => driver) + expected_kube_metadata = { + 'time'=>'2015-05-08T09:22:01Z', + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + "namespace_id"=>"orphaned", + 'namespace_name' => '.orphaned', + "orphaned_namespace"=>"default" + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with docker & kubernetes metadata where id cache hit and metadata miss' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}]) do + driver = create_driver(' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ') + cache = driver.instance.instance_variable_get(:@id_cache) + cache['49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'] = { + :pod_id =>'c76927af-f563-11e4-b32d-54ee7527188d', + :namespace_id =>'898268c8-4a36-11e5-9d81-42010af0194c' + } + stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc').to_timeout + stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_timeout + filtered = emit({'time'=>'2015-05-08T09:22:01Z'}, '', d:driver) + expected_kube_metadata = { + 'time'=>'2015-05-08T09:22:01Z', + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + } + } + + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with docker & kubernetes metadata where id cache hit and metadata is reloaded' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'}, {name: 'kubernetes_get_namespace_default'}]) do + driver = create_driver(' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ') + cache = driver.instance.instance_variable_get(:@id_cache) + cache['49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'] = { + :pod_id =>'c76927af-f563-11e4-b32d-54ee7527188d', + :namespace_id =>'898268c8-4a36-11e5-9d81-42010af0194c' + } + filtered = emit({'time'=>'2015-05-08T09:22:01Z'}, '', d:driver) + expected_kube_metadata = { + 'time'=>'2015-05-08T09:22:01Z', + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + } + + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with docker & kubernetes metadata' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'}, {name: 'kubernetes_get_namespace_default'}]) do + filtered = emit({'time'=>'2015-05-08T09:22:01Z'}) + expected_kube_metadata = { + 'time'=>'2015-05-08T09:22:01Z', + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + } + + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with docker & kubernetes metadata & namespace_id enabled' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'}, + {name: 'kubernetes_get_namespace_default', options: {allow_playback_repeats: true}}]) do + filtered = emit({}, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with docker & kubernetes metadata using bearer token' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server_using_token'}, {name: 'kubernetes_get_api_v1_using_token'}, + {name: 'kubernetes_get_pod_using_token'}, {name: 'kubernetes_get_namespace_default_using_token'}]) do + filtered = emit({}, ' + kubernetes_url https://localhost:8443 + verify_ssl false + watch false + bearer_token_file test/plugin/test.token + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with docker & kubernetes metadata but no configured api server' do + filtered = emit({}, '') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'namespace_name' => 'default', + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + + test 'with docker & inaccessible kubernetes metadata' do + stub_request(:any, 'https://localhost:8443/api').to_return( + 'body' => { + 'versions' => ['v1beta3', 'v1'] + }.to_json + ) + stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc').to_timeout + stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_timeout + filtered = emit() + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'namespace_name' => '.orphaned', + 'orphaned_namespace' => 'default', + 'namespace_id' => 'orphaned' + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + + test 'with dot in pod name' do + stub_request(:any, 'https://localhost:8443/api').to_return( + 'body' => { + 'versions' => ['v1beta3', 'v1'] + }.to_json + ) + stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller.98rqc').to_timeout + filtered = emit_with_tag('var.log.containers.fabric8-console-controller.98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log', {}, '') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'pod_name' => 'fabric8-console-controller.98rqc', + 'container_name' => 'fabric8-console-container', + 'namespace_name' => 'default' + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + + test 'with docker metadata, non-kubernetes' do + filtered = emit_with_tag('non-kubernetes', {}, '') + assert_false(filtered[0].has_key?(:kubernetes)) + end + + test 'ignores invalid json in log field' do + json_log = "{'foo':123}" + msg = { + 'log' => json_log + } + filtered = emit_with_tag('non-kubernetes', msg, '') + assert_equal(msg, filtered[0]) + end + + test 'with kubernetes dotted labels, de_dot enabled' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, + {name: 'kubernetes_docker_metadata_dotted_labels'}]) do + filtered = emit({}, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'namespace_labels' => { + 'kubernetes_io/namespacetest' => 'somevalue' + }, + 'namespace_name' => 'default', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'kubernetes_io/test' => 'somevalue' + } + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with kubernetes dotted labels, de_dot disabled' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, + {name: 'kubernetes_docker_metadata_dotted_labels'}]) do + filtered = emit({}, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + de_dot false + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'namespace_labels' => { + 'kubernetes.io/namespacetest' => 'somevalue' + }, + 'namespace_name' => 'default', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'kubernetes.io/test' => 'somevalue' + } + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'invalid de_dot_separator config' do + assert_raise Fluent::ConfigError do + create_driver(' + de_dot_separator contains. + ') + end + end + + test 'with records from journald and docker & kubernetes metadata' do + # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL + tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log' + msg = { + 'CONTAINER_NAME' => 'k8s_fabric8-console-container.db89db89_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_89db89db', + 'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459', + 'randomfield' => 'randomvalue' + } + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'}, + {name: 'kubernetes_get_namespace_default'}]) do + filtered = emit_with_tag(tag, msg, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + use_journal true + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + }.merge(msg) + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with records from journald and docker & kubernetes metadata & namespace_id enabled' do + # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL + tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log' + msg = { + 'CONTAINER_NAME' => 'k8s_fabric8-console-container.db89db89_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_89db89db', + 'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459', + 'randomfield' => 'randomvalue' + } + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'}, + {name: 'kubernetes_get_namespace_default', options: {allow_playback_repeats: true}}]) do + filtered = emit_with_tag(tag, msg, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + use_journal true + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + }.merge(msg) + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with records from journald and docker & kubernetes metadata with use_journal unset' do + # with use_journal unset, should still use the journal fields instead of tag fields + tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log' + msg = { + 'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db', + 'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b', + 'kubernetes' => { + 'namespace_name' => 'k8s-namespace-name', + 'pod_name' => 'k8s-pod-name', + 'container_name' => 'k8s-container-name' + }, + 'docker' => {'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95'}, + 'randomfield' => 'randomvalue' + } + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'}, + {name: 'kubernetes_get_namespace_default'}, + {name: 'metadata_from_tag_journald_and_kubernetes_fields'}]) do + es = emit_with_tag(tag, msg, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'k8s-pod-name', + 'container_name' => 'k8s-container-name', + 'container_image' => 'k8s-container-image:latest', + 'container_image_id' => 'docker://d78c5217c41e9af08d37d9ae2cb070afa1fe3da6bc77bfb18879a8b4bfdf8a34', + 'namespace_name' => 'k8s-namespace-name', + 'namespace_id' => '8e0dc8fc-59f2-49f7-a3e2-ed0913e19d9f', + 'pod_id' => 'ebabf749-5fcd-4750-a3f0-aedd89476da8', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'k8s-test' + } + } + }.merge(msg) {|key,oldval,newval| ((key == 'kubernetes') || (key == 'docker')) ? oldval : newval} + assert_equal(expected_kube_metadata, es[0]) + end + end + + test 'with records from journald and docker & kubernetes metadata with lookup_from_k8s_field false' do + # with use_journal unset, should still use the journal fields instead of tag fields + tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log' + msg = { + 'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db', + 'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b', + 'kubernetes' => { + 'namespace_name' => 'k8s-namespace-name', + 'pod_name' => 'k8s-pod-name', + 'container_name' => 'k8s-container-name' + }, + 'docker' => {'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95'}, + 'randomfield' => 'randomvalue' + } + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'}, + {name: 'kubernetes_get_namespace_default', options: {allow_playback_repeats: true}}, + {name: 'metadata_from_tag_and_journald_fields'}]) do + es = emit_with_tag(tag, msg, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + lookup_from_k8s_field false + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'journald-pod-name', + 'container_name' => 'journald-container-name', + 'container_image' => 'journald-container-image:latest', + 'container_image_id' => 'docker://dda4c95705fb7050b701b10a7fe928ca5bc971a1dcb521ae3c339194cbf36b47', + 'namespace_name' => 'journald-namespace-name', + 'namespace_id' => '8282888f-733f-4f23-a3d3-1fdfa3bdacf2', + 'pod_id' => '5e1c1e27-b637-4e81-80b6-6d8a8c404d3b', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'journald-test' + } + } + }.merge(msg) {|key,oldval,newval| ((key == 'kubernetes') || (key == 'docker')) ? oldval : newval} + assert_equal(expected_kube_metadata, es[0]) + end + end + + test 'uses metadata from tag if use_journal false and lookup_from_k8s_field false' do + # with use_journal unset, should still use the journal fields instead of tag fields + tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log' + msg = { + 'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db', + 'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b', + 'kubernetes' => { + 'namespace_name' => 'k8s-namespace-name', + 'pod_name' => 'k8s-pod-name', + 'container_name' => 'k8s-container-name' + }, + 'docker' => {'container_id' => 'e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95'}, + 'randomfield' => 'randomvalue' + } + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'}, + {name: 'kubernetes_get_namespace_default', options: {allow_playback_repeats: true}}, + {name: 'metadata_from_tag_and_journald_fields'}]) do + es = emit_with_tag(tag, msg, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + lookup_from_k8s_field false + use_journal false + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + }.merge(msg) {|key,oldval,newval| ((key == 'kubernetes') || (key == 'docker')) ? oldval : newval} + assert_equal(expected_kube_metadata, es[0]) + end + end + + test 'with kubernetes annotations' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, + {name: 'kubernetes_docker_metadata_annotations'}, + {name: 'kubernetes_get_namespace_default'}]) do + filtered = emit({},' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + annotation_match [ "^custom.+", "two"] + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + }, + 'annotations' => { + 'custom_field1' => 'hello_kitty', + 'field_two' => 'value' + } + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with records from journald and docker & kubernetes metadata, alternate form' do + # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL + tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log' + msg = { + 'CONTAINER_NAME' => 'alt_fabric8-console-container_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_0', + 'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459', + 'randomfield' => 'randomvalue' + } + VCR.use_cassettes([ + {name: 'valid_kubernetes_api_server'}, + {name: 'kubernetes_get_api_v1'}, + {name: 'kubernetes_get_pod'}, + {name: 'kubernetes_get_namespace_default'}, + {name: 'metadata_from_tag_and_journald_fields'} + ]) do + filtered = emit_with_tag(tag, msg, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + use_journal true + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + }.merge(msg) + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with kubernetes namespace annotations' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, + {name: 'kubernetes_docker_metadata_annotations'}, + {name: 'kubernetes_get_namespace_default'}]) do + filtered = emit({},' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + annotation_match [ "^custom.+", "two", "workspace*"] + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'namespace_name' => 'default', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + }, + 'annotations' => { + 'custom_field1' => 'hello_kitty', + 'field_two' => 'value' + }, + 'namespace_annotations' => { + 'workspaceId' => 'myWorkspaceName' + } + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with kubernetes namespace annotations no match' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, + {name: 'kubernetes_docker_metadata_annotations'}, + {name: 'kubernetes_get_namespace_default'}]) do + filtered = emit({},' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + annotation_match [ "noMatch*"] + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'namespace_name' => 'default', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with CONTAINER_NAME that does not match' do + tag = 'var.log.containers.junk4_junk5_junk6-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log' + msg = { + 'CONTAINER_NAME' => 'does_not_match', + 'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459', + 'randomfield' => 'randomvalue' + } + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, + {name: 'kubernetes_docker_metadata_annotations'}, + {name: 'kubernetes_get_namespace_default'}]) do + filtered = emit_with_tag(tag, msg, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + use_journal true + ') + expected_kube_metadata = { + 'CONTAINER_NAME' => 'does_not_match', + 'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459', + 'randomfield' => 'randomvalue' + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'with CONTAINER_NAME starts with k8s_ that does not match' do + tag = 'var.log.containers.junk4_junk5_junk6-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log' + msg = { + 'CONTAINER_NAME' => 'k8s_doesnotmatch', + 'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459', + 'randomfield' => 'randomvalue' + } + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, + {name: 'kubernetes_docker_metadata_annotations'}, + {name: 'kubernetes_get_namespace_default'}]) do + filtered = emit_with_tag(tag, msg, ' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + use_journal true + ') + expected_kube_metadata = { + 'CONTAINER_NAME' => 'k8s_doesnotmatch', + 'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459', + 'randomfield' => 'randomvalue' + } + assert_equal(expected_kube_metadata, filtered[0]) + end + end + + test 'processes all events when reading from MessagePackEventStream' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, + {name: 'kubernetes_get_pod'}, + {name: 'kubernetes_get_namespace_default'}]) do + entries = [[@time, {'time'=>'2015-05-08T09:22:01Z'}], [@time, {'time'=>'2015-05-08T09:22:01Z'}]] + array_stream = Fluent::ArrayEventStream.new(entries) + msgpack_stream = Fluent::MessagePackEventStream.new(array_stream.to_msgpack_stream) + + d = create_driver(' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + ') + d.run { + d.feed(DEFAULT_TAG, msgpack_stream) + } + filtered = d.filtered.map{|e| e.last} + + expected_kube_metadata = { + 'time'=>'2015-05-08T09:22:01Z', + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'container_image' => 'fabric8/hawtio-kubernetes:latest', + 'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + 'namespace_name' => 'default', + 'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d', + 'master_url' => 'https://localhost:8443', + 'labels' => { + 'component' => 'fabric8Console' + } + } + } + + assert_equal(expected_kube_metadata, filtered[0]) + assert_equal(expected_kube_metadata, filtered[1]) + end + end + + test 'with docker & kubernetes metadata using skip config params' do + VCR.use_cassettes([{name: 'valid_kubernetes_api_server'}, {name: 'kubernetes_get_api_v1'}, {name: 'kubernetes_get_pod'}, + {name: 'kubernetes_get_namespace_default'}]) do + filtered = emit({},' + kubernetes_url https://localhost:8443 + watch false + cache_size 1 + skip_labels true + skip_container_metadata true + skip_master_url true + skip_namespace_metadata true + ') + expected_kube_metadata = { + 'docker' => { + 'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + }, + 'kubernetes' => { + 'host' => 'jimmi-redhat.localnet', + 'pod_name' => 'fabric8-console-controller-98rqc', + 'container_name' => 'fabric8-console-container', + 'namespace_name' => 'default', + 'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d' + } + } + + assert_equal(expected_kube_metadata, filtered[0]) + end + end + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_watch_namespaces.rb b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_watch_namespaces.rb new file mode 100644 index 0000000000..a71b79dedc --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_watch_namespaces.rb @@ -0,0 +1,244 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require_relative '../helper' +require_relative 'watch_test' + +class WatchNamespacesTestTest < WatchTest + + include KubernetesMetadata::WatchNamespaces + + setup do + @initial = { + kind: 'NamespaceList', + metadata: {resourceVersion: '123'}, + items: [ + { + metadata: { + name: 'initial', + uid: 'initial_uid' + } + }, + { + metadata: { + name: 'modified', + uid: 'modified_uid' + } + } + ] + } + + @created = { + type: 'CREATED', + object: { + metadata: { + name: 'created', + uid: 'created_uid' + } + } + } + @modified = { + type: 'MODIFIED', + object: { + metadata: { + name: 'foo', + uid: 'modified_uid' + } + } + } + @deleted = { + type: 'DELETED', + object: { + metadata: { + name: 'deleteme', + uid: 'deleted_uid' + } + } + } + @error = { + type: 'ERROR', + object: { + message: 'some error message' + } + } + @gone = { + type: 'ERROR', + object: { + code: 410, + kind: 'Status', + message: 'too old resource version: 123 (391079)', + metadata: { + name: 'gone', + namespace: 'gone', + uid: 'gone_uid' + }, + reason: 'Gone' + } + } + end + + test 'namespace list caches namespaces' do + @client.stub :get_namespaces, @initial do + process_namespace_watcher_notices(start_namespace_watch) + assert_equal(true, @namespace_cache.key?('initial_uid')) + assert_equal(true, @namespace_cache.key?('modified_uid')) + assert_equal(2, @stats[:namespace_cache_host_updates]) + end + end + + test 'namespace list caches namespaces and watch updates' do + orig_env_val = ENV['K8S_NODE_NAME'] + ENV['K8S_NODE_NAME'] = 'aNodeName' + @client.stub :get_namespaces, @initial do + @client.stub :watch_namespaces, [@modified] do + process_namespace_watcher_notices(start_namespace_watch) + assert_equal(2, @stats[:namespace_cache_host_updates]) + assert_equal(1, @stats[:namespace_cache_watch_updates]) + end + end + ENV['K8S_NODE_NAME'] = orig_env_val + end + + test 'namespace watch ignores CREATED' do + @client.stub :watch_namespaces, [@created] do + process_namespace_watcher_notices(start_namespace_watch) + assert_equal(false, @namespace_cache.key?('created_uid')) + assert_equal(1, @stats[:namespace_cache_watch_ignored]) + end + end + + test 'namespace watch ignores MODIFIED when info not in cache' do + @client.stub :watch_namespaces, [@modified] do + process_namespace_watcher_notices(start_namespace_watch) + assert_equal(false, @namespace_cache.key?('modified_uid')) + assert_equal(1, @stats[:namespace_cache_watch_misses]) + end + end + + test 'namespace watch updates cache when MODIFIED is received and info is cached' do + @namespace_cache['modified_uid'] = {} + @client.stub :watch_namespaces, [@modified] do + process_namespace_watcher_notices(start_namespace_watch) + assert_equal(true, @namespace_cache.key?('modified_uid')) + assert_equal(1, @stats[:namespace_cache_watch_updates]) + end + end + + test 'namespace watch ignores DELETED' do + @namespace_cache['deleted_uid'] = {} + @client.stub :watch_namespaces, [@deleted] do + process_namespace_watcher_notices(start_namespace_watch) + assert_equal(true, @namespace_cache.key?('deleted_uid')) + assert_equal(1, @stats[:namespace_cache_watch_deletes_ignored]) + end + end + + test 'namespace watch raises Fluent::UnrecoverableError when cannot re-establish connection to k8s API server' do + # Stub start_namespace_watch to simulate initial successful connection to API server + stub(self).start_namespace_watch + # Stub watch_namespaces to simluate not being able to set up watch connection to API server + stub(@client).watch_namespaces { raise } + @client.stub :get_namespaces, @initial do + assert_raise Fluent::UnrecoverableError do + set_up_namespace_thread + end + end + assert_equal(3, @stats[:namespace_watch_failures]) + assert_equal(2, Thread.current[:namespace_watch_retry_count]) + assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval]) + assert_nil(@stats[:namespace_watch_error_type_notices]) + end + + test 'namespace watch resets watch retry count when exceptions are encountered and connection to k8s API server is re-established' do + @client.stub :get_namespaces, @initial do + @client.stub :watch_namespaces, [[@created, @exception_raised]] do + # Force the infinite watch loop to exit after 3 seconds. Verifies that + # no unrecoverable error was thrown during this period of time. + assert_raise Timeout::Error.new('execution expired') do + Timeout.timeout(3) do + set_up_namespace_thread + end + end + assert_operator(@stats[:namespace_watch_failures], :>=, 3) + assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1) + assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1) + end + end + end + + test 'namespace watch resets watch retry count when error is received and connection to k8s API server is re-established' do + @client.stub :get_namespaces, @initial do + @client.stub :watch_namespaces, [@error] do + # Force the infinite watch loop to exit after 3 seconds. Verifies that + # no unrecoverable error was thrown during this period of time. + assert_raise Timeout::Error.new('execution expired') do + Timeout.timeout(3) do + set_up_namespace_thread + end + end + assert_operator(@stats[:namespace_watch_failures], :>=, 3) + assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1) + assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1) + end + end + end + + test 'namespace watch continues after retries succeed' do + @client.stub :get_namespaces, @initial do + @client.stub :watch_namespaces, [@modified, @error, @modified] do + # Force the infinite watch loop to exit after 3 seconds. Verifies that + # no unrecoverable error was thrown during this period of time. + assert_raise Timeout::Error.new('execution expired') do + Timeout.timeout(3) do + set_up_namespace_thread + end + end + assert_operator(@stats[:namespace_watch_failures], :>=, 3) + assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1) + assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1) + assert_operator(@stats[:namespace_watch_error_type_notices], :>=, 3) + end + end + end + + test 'namespace watch raises a GoneError when a 410 Gone error is received' do + @cache['gone_uid'] = {} + @client.stub :watch_namespaces, [@gone] do + assert_raise KubernetesMetadata::Common::GoneError do + process_namespace_watcher_notices(start_namespace_watch) + end + assert_equal(1, @stats[:namespace_watch_gone_notices]) + end + end + + test 'namespace watch retries when 410 Gone errors are encountered' do + @client.stub :get_namespaces, @initial do + @client.stub :watch_namespaces, [@created, @gone, @modified] do + # Force the infinite watch loop to exit after 3 seconds. Verifies that + # no unrecoverable error was thrown during this period of time. + assert_raise Timeout::Error.new('execution expired') do + Timeout.timeout(3) do + set_up_namespace_thread + end + end + assert_operator(@stats[:namespace_watch_gone_errors], :>=, 3) + assert_operator(@stats[:namespace_watch_gone_notices], :>=, 3) + end + end + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_watch_pods.rb b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_watch_pods.rb new file mode 100644 index 0000000000..c24203f750 --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/plugin/test_watch_pods.rb @@ -0,0 +1,333 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require_relative '../helper' +require_relative 'watch_test' + +class DefaultPodWatchStrategyTest < WatchTest + + include KubernetesMetadata::WatchPods + + setup do + @initial = { + kind: 'PodList', + metadata: {resourceVersion: '123'}, + items: [ + { + metadata: { + name: 'initial', + namespace: 'initial_ns', + uid: 'initial_uid', + labels: {}, + }, + spec: { + nodeName: 'aNodeName', + containers: [{ + name: 'foo', + image: 'bar', + }, { + name: 'bar', + image: 'foo', + }] + } + }, + { + metadata: { + name: 'modified', + namespace: 'create', + uid: 'modified_uid', + labels: {}, + }, + spec: { + nodeName: 'aNodeName', + containers: [{ + name: 'foo', + image: 'bar', + }, { + name: 'bar', + image: 'foo', + }] + } + } + ] + } + @created = { + type: 'CREATED', + object: { + metadata: { + name: 'created', + namespace: 'create', + uid: 'created_uid', + resourceVersion: '122', + labels: {}, + }, + spec: { + nodeName: 'aNodeName', + containers: [{ + name: 'foo', + image: 'bar', + }, { + name: 'bar', + image: 'foo', + }] + } + } + } + @modified = { + type: 'MODIFIED', + object: { + metadata: { + name: 'foo', + namespace: 'modified', + uid: 'modified_uid', + resourceVersion: '123', + labels: {}, + }, + spec: { + nodeName: 'aNodeName', + containers: [{ + name: 'foo', + image: 'bar', + }, { + name: 'bar', + image: 'foo', + }] + }, + status: { + containerStatuses: [ + { + name: 'fabric8-console-container', + state: { + running: { + startedAt: '2015-05-08T09:22:44Z' + } + }, + lastState: {}, + ready: true, + restartCount: 0, + image: 'fabric8/hawtio-kubernetes:latest', + imageID: 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303', + containerID: 'docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459' + } + ] + } + } + } + @deleted = { + type: 'DELETED', + object: { + metadata: { + name: 'deleteme', + namespace: 'deleted', + uid: 'deleted_uid', + resourceVersion: '124' + } + } + } + @error = { + type: 'ERROR', + object: { + message: 'some error message' + } + } + @gone = { + type: 'ERROR', + object: { + code: 410, + kind: 'Status', + message: 'too old resource version: 123 (391079)', + metadata: { + name: 'gone', + namespace: 'gone', + uid: 'gone_uid' + }, + reason: 'Gone' + } + } + end + + test 'pod list caches pods' do + orig_env_val = ENV['K8S_NODE_NAME'] + ENV['K8S_NODE_NAME'] = 'aNodeName' + @client.stub :get_pods, @initial do + process_pod_watcher_notices(start_pod_watch) + assert_equal(true, @cache.key?('initial_uid')) + assert_equal(true, @cache.key?('modified_uid')) + assert_equal(2, @stats[:pod_cache_host_updates]) + end + ENV['K8S_NODE_NAME'] = orig_env_val + end + + test 'pod list caches pods and watch updates' do + orig_env_val = ENV['K8S_NODE_NAME'] + ENV['K8S_NODE_NAME'] = 'aNodeName' + @client.stub :get_pods, @initial do + @client.stub :watch_pods, [@modified] do + process_pod_watcher_notices(start_pod_watch) + assert_equal(2, @stats[:pod_cache_host_updates]) + assert_equal(1, @stats[:pod_cache_watch_updates]) + end + end + ENV['K8S_NODE_NAME'] = orig_env_val + assert_equal('123', @last_seen_resource_version) # from @modified + end + + test 'pod watch notice ignores CREATED' do + @client.stub :get_pods, @initial do + @client.stub :watch_pods, [@created] do + process_pod_watcher_notices(start_pod_watch) + assert_equal(false, @cache.key?('created_uid')) + assert_equal(1, @stats[:pod_cache_watch_ignored]) + end + end + end + + test 'pod watch notice is ignored when info not cached and MODIFIED is received' do + @client.stub :watch_pods, [@modified] do + process_pod_watcher_notices(start_pod_watch) + assert_equal(false, @cache.key?('modified_uid')) + assert_equal(1, @stats[:pod_cache_watch_misses]) + end + end + + test 'pod MODIFIED cached when hostname matches' do + orig_env_val = ENV['K8S_NODE_NAME'] + ENV['K8S_NODE_NAME'] = 'aNodeName' + @client.stub :watch_pods, [@modified] do + process_pod_watcher_notices(start_pod_watch) + assert_equal(true, @cache.key?('modified_uid')) + assert_equal(1, @stats[:pod_cache_host_updates]) + end + ENV['K8S_NODE_NAME'] = orig_env_val + end + + test 'pod watch notice is updated when MODIFIED is received' do + @cache['modified_uid'] = {} + @client.stub :watch_pods, [@modified] do + process_pod_watcher_notices(start_pod_watch) + assert_equal(true, @cache.key?('modified_uid')) + assert_equal(1, @stats[:pod_cache_watch_updates]) + end + end + + test 'pod watch notice is ignored when delete is received' do + @cache['deleted_uid'] = {} + @client.stub :watch_pods, [@deleted] do + process_pod_watcher_notices(start_pod_watch) + assert_equal(true, @cache.key?('deleted_uid')) + assert_equal(1, @stats[:pod_cache_watch_delete_ignored]) + end + end + + test 'pod watch raises Fluent::UnrecoverableError when cannot re-establish connection to k8s API server' do + # Stub start_pod_watch to simulate initial successful connection to API server + stub(self).start_pod_watch + # Stub watch_pods to simluate not being able to set up watch connection to API server + stub(@client).watch_pods { raise } + @client.stub :get_pods, @initial do + assert_raise Fluent::UnrecoverableError do + set_up_pod_thread + end + end + assert_equal(3, @stats[:pod_watch_failures]) + assert_equal(2, Thread.current[:pod_watch_retry_count]) + assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval]) + assert_nil(@stats[:pod_watch_error_type_notices]) + end + + test 'pod watch resets watch retry count when exceptions are encountered and connection to k8s API server is re-established' do + @client.stub :get_pods, @initial do + @client.stub :watch_pods, [[@created, @exception_raised]] do + # Force the infinite watch loop to exit after 3 seconds. Verifies that + # no unrecoverable error was thrown during this period of time. + assert_raise Timeout::Error.new('execution expired') do + Timeout.timeout(3) do + set_up_pod_thread + end + end + assert_operator(@stats[:pod_watch_failures], :>=, 3) + assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1) + assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1) + end + end + end + + test 'pod watch resets watch retry count when error is received and connection to k8s API server is re-established' do + @client.stub :get_pods, @initial do + @client.stub :watch_pods, [@error] do + # Force the infinite watch loop to exit after 3 seconds. Verifies that + # no unrecoverable error was thrown during this period of time. + assert_raise Timeout::Error.new('execution expired') do + Timeout.timeout(3) do + set_up_pod_thread + end + end + assert_operator(@stats[:pod_watch_failures], :>=, 3) + assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1) + assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1) + assert_operator(@stats[:pod_watch_error_type_notices], :>=, 3) + end + end + end + + test 'pod watch continues after retries succeed' do + @client.stub :get_pods, @initial do + @client.stub :watch_pods, [@modified, @error, @modified] do + # Force the infinite watch loop to exit after 3 seconds. Verifies that + # no unrecoverable error was thrown during this period of time. + assert_raise Timeout::Error.new('execution expired') do + Timeout.timeout(3) do + set_up_pod_thread + end + end + assert_operator(@stats[:pod_watch_failures], :>=, 3) + assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1) + assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1) + assert_operator(@stats[:pod_watch_error_type_notices], :>=, 3) + end + end + end + + test 'pod watch raises a GoneError when a 410 Gone error is received' do + @cache['gone_uid'] = {} + @client.stub :watch_pods, [@gone] do + @last_seen_resource_version = '100' + assert_raise KubernetesMetadata::Common::GoneError do + process_pod_watcher_notices(start_pod_watch) + end + assert_equal(1, @stats[:pod_watch_gone_notices]) + assert_nil @last_seen_resource_version # forced restart + end + end + + test 'pod watch retries when 410 Gone errors are encountered' do + @client.stub :get_pods, @initial do + @client.stub :watch_pods, [@created, @gone, @modified] do + # Force the infinite watch loop to exit after 3 seconds because the code sleeps 3 times. + # Verifies that no unrecoverable error was thrown during this period of time. + assert_raise Timeout::Error.new('execution expired') do + Timeout.timeout(3) do + set_up_pod_thread + end + end + assert_operator(@stats[:pod_watch_gone_errors], :>=, 3) + assert_operator(@stats[:pod_watch_gone_notices], :>=, 3) + end + end + end +end diff --git a/fluent-plugin-kubernetes-metadata-filter/test/plugin/watch_test.rb b/fluent-plugin-kubernetes-metadata-filter/test/plugin/watch_test.rb new file mode 100644 index 0000000000..2a50a12dff --- /dev/null +++ b/fluent-plugin-kubernetes-metadata-filter/test/plugin/watch_test.rb @@ -0,0 +1,68 @@ +# +# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with +# Kubernetes metadata +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require_relative '../helper' + +class WatchTest < Test::Unit::TestCase + + def thread_current_running? + true + end + + setup do + @annotations_regexps = [] + @namespace_cache = {} + @watch_retry_max_times = 2 + @watch_retry_interval = 1 + @watch_retry_exponential_backoff_base = 2 + @cache = {} + @stats = KubernetesMetadata::Stats.new + Thread.current[:pod_watch_retry_count] = 0 + Thread.current[:namespace_watch_retry_count] = 0 + + @client = OpenStruct.new + def @client.watch_pods(options = {}) + [] + end + def @client.watch_namespaces(options = {}) + [] + end + def @client.get_namespaces(options = {}) + {items: [], metadata: {resourceVersion: '12345'}} + end + def @client.get_pods(options = {}) + {items: [], metadata: {resourceVersion: '12345'}} + end + + @exception_raised = :blow_up_when_used + end + + def watcher=(value) + end + + def log + logger = {} + def logger.debug(message) + end + def logger.info(message, error) + end + def logger.error(message, error) + end + logger + end +end