diff --git a/.fossa.yml b/.fossa.yml new file mode 100644 index 0000000..8992b11 --- /dev/null +++ b/.fossa.yml @@ -0,0 +1,5 @@ +version: 3 +server: https://app.fossa.com +project: + id: "fluent-plugin-kubernetes-metrics" + team: "TA-Automation" diff --git a/.github/workflows/ci_build_test.yaml b/.github/workflows/ci_build_test.yaml index 660c7c2..45bd875 100644 --- a/.github/workflows/ci_build_test.yaml +++ b/.github/workflows/ci_build_test.yaml @@ -194,3 +194,36 @@ jobs: --splunk-password $CI_SPLUNK_PASSWORD \ --nodes-count $MINIKUBE_NODE_COUNTS\ -p no:warnings -s -n auto + fossa-scan: + continue-on-error: true + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: run fossa anlyze and create report + run: | + curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash + fossa analyze --include-unused-deps --debug + fossa report attribution --format text > /tmp/THIRDPARTY + env: + FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} + - name: upload THIRDPARTY file + uses: actions/upload-artifact@v2 + with: + name: THIRDPARTY + path: /tmp/THIRDPARTY + - name: run fossa test + run: | + fossa test --debug + env: + FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} + semgrep: + runs-on: ubuntu-latest + name: security-sast-semgrep + if: github.actor != 'dependabot[bot]' + steps: + - uses: actions/checkout@v3 + - name: Semgrep + id: semgrep + uses: returntocorp/semgrep-action@v1 + with: + publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} diff --git a/Gemfile.lock b/Gemfile.lock index 7231767..3995821 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,11 +1,11 @@ PATH remote: . specs: - fluent-plugin-kubernetes-metrics (1.1.12) + fluent-plugin-kubernetes-metrics (1.2.0) fluentd (>= 1.9.1) kubeclient (~> 4.9.3) - multi_json (~> 1.14.1) - oj (~> 3.10.2) + multi_json (~> 1.14) + oj (~> 3.10) GEM remote: https://rubygems.org/ @@ -23,12 +23,12 @@ GEM ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.14.6) + fluentd (1.15.1) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) - serverengine (>= 2.2.5, < 3.0.0) + serverengine (>= 2.3.0, < 3.0.0) sigdump (~> 0.2.2) strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) @@ -42,14 +42,14 @@ GEM http-form_data (~> 2.2) http-parser (~> 1.2.0) http-accept (1.7.0) - http-cookie (1.0.4) + http-cookie (1.0.5) domain_name (~> 0.5) http-form_data (2.3.0) http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.8.0) json (2.6.1) - jsonpath (1.1.0) + jsonpath (1.1.2) multi_json kubeclient (4.9.3) http (>= 3.0, < 5.0) @@ -59,10 +59,10 @@ GEM mime-types (3.4.1) mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) - msgpack (1.5.1) - multi_json (1.14.1) + msgpack (1.5.4) + multi_json (1.15.0) netrc (0.11.0) - oj (3.10.18) + oj (3.13.20) power_assert (2.0.1) public_suffix (4.0.6) rake (13.0.6) @@ -73,7 +73,7 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) rexml (3.2.5) - serverengine (2.2.5) + serverengine (2.3.0) sigdump (~> 0.2.2) sigdump (0.2.4) simplecov (0.16.1) @@ -84,19 +84,19 @@ GEM strptime (0.2.5) test-unit (3.3.9) power_assert - tzinfo (2.0.4) + tzinfo (2.0.5) concurrent-ruby (~> 1.0) - tzinfo-data (1.2022.1) + tzinfo-data (1.2022.2) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext - unf_ext (0.0.8.1) + unf_ext (0.0.8.2) webmock (3.5.1) addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff webrick (1.7.0) - yajl-ruby (1.4.2) + yajl-ruby (1.4.3) PLATFORMS ruby diff --git a/VERSION b/VERSION index d7f1518..867e524 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.12 \ No newline at end of file +1.2.0 \ No newline at end of file diff --git a/docker/Gemfile b/docker/Gemfile index 0b5244c..3d32316 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -3,16 +3,16 @@ source 'https://rubygems.org' # This is separate gemfile for building docker image that has all plugins # for kubernetes log collection agent # List all required gems here and install via bundler to resolve dependencies -gem "fluentd", ">=1.14.2" +gem "fluentd", ">=1.15.0" gem "fluent-plugin-prometheus", ">=2.0" gem "fluent-plugin-record-modifier", "=2.1.0" -gem "fluent-plugin-kubernetes_metadata_filter", ">=2.5.3" gem "fluent-plugin-jq", "=0.5.1" -gem "oj", "=3.10.18" -gem 'multi_json', '=1.14.1' +gem "oj", "~> 3.10" +gem 'multi_json', '~> 1.14' gem 'bigdecimal', '=3.0.0' gem 'kubeclient', '=4.9.3' gem 'http_parser.rb', '=0.8.0' +gem 'rack', '>= 2.2.3.1' -gem "fluent-plugin-splunk-hec", ">= 1.2.5" +gem "fluent-plugin-splunk-hec", "= 1.3.0" gem 'fluent-plugin-kubernetes-metrics', path: 'gem/' \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index d9e0ac9..5c23b67 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,18 +1,18 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-metrics (1.1.12) + fluent-plugin-kubernetes-metrics (1.2.0) fluentd (>= 1.9.1) kubeclient (~> 4.9.3) - multi_json (~> 1.14.1) - oj (~> 3.10.2) + multi_json (~> 1.14) + oj (~> 3.10) GEM remote: https://rubygems.org/ specs: - activemodel (7.0.2.4) - activesupport (= 7.0.2.4) - activesupport (7.0.2.4) + activemodel (7.0.3.1) + activesupport (= 7.0.3.1) + activesupport (7.0.3.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -35,27 +35,23 @@ GEM fluent-plugin-jq (0.5.1) fluentd (>= 0.14.10, < 2) multi_json (~> 1.13) - fluent-plugin-kubernetes_metadata_filter (2.9.3) - fluentd (>= 0.14.0, < 1.15) - kubeclient (>= 4.0.0, < 5.0.0) - lru_redux fluent-plugin-prometheus (2.0.2) fluentd (>= 1.9.1, < 2) prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.13) - fluentd (>= 1.4) + fluent-plugin-splunk-hec (1.3.0) + fluentd (>= 1.5) multi_json (~> 1.13) net-http-persistent (~> 4.0) openid_connect (~> 1.1.8) prometheus-client (>= 2.1.0) - fluentd (1.14.6) + fluentd (1.15.1) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) - serverengine (>= 2.2.5, < 3.0.0) + serverengine (>= 2.3.0, < 3.0.0) sigdump (~> 0.2.2) strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) @@ -75,12 +71,13 @@ GEM ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.8.0) httpclient (2.8.3) - i18n (1.10.0) + i18n (1.12.0) concurrent-ruby (~> 1.0) - json-jwt (1.13.0) + json-jwt (1.15.2) activesupport (>= 4.2) aes_key_wrap bindata + httpclient jsonpath (1.1.0) multi_json kubeclient (4.9.3) @@ -88,16 +85,15 @@ GEM jsonpath (~> 1.0) recursive-open-struct (~> 1.1, >= 1.1.1) rest-client (~> 2.0) - lru_redux (1.1.0) mail (2.7.1) mini_mime (>= 0.1.1) mime-types (3.4.1) mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) mini_mime (1.1.2) - minitest (5.15.0) - msgpack (1.5.1) - multi_json (1.14.1) + minitest (5.16.2) + msgpack (1.5.4) + multi_json (1.15.0) net-http-persistent (4.0.1) connection_pool (~> 2.2) netrc (0.11.0) @@ -114,8 +110,8 @@ GEM webfinger (>= 1.0.1) prometheus-client (4.0.0) public_suffix (4.0.7) - rack (2.2.3) - rack-oauth2 (1.19.0) + rack (2.2.4) + rack-oauth2 (1.21.2) activesupport attr_required httpclient @@ -128,7 +124,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - serverengine (2.2.5) + serverengine (2.3.0) sigdump (~> 0.2.2) sigdump (0.2.4) strptime (0.2.5) @@ -136,9 +132,9 @@ GEM activesupport (>= 3) attr_required (>= 0.0.5) httpclient (>= 2.4) - tzinfo (2.0.4) + tzinfo (2.0.5) concurrent-ruby (~> 1.0) - tzinfo-data (1.2022.1) + tzinfo-data (1.2022.2) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext @@ -146,14 +142,14 @@ GEM validate_email (0.1.6) activemodel (>= 3.0) mail (>= 2.2.5) - validate_url (1.0.13) + validate_url (1.0.15) activemodel (>= 3.0.0) public_suffix webfinger (1.2.0) activesupport httpclient (>= 2.4) webrick (1.7.0) - yajl-ruby (1.4.2) + yajl-ruby (1.4.3) PLATFORMS ruby @@ -162,15 +158,15 @@ DEPENDENCIES bigdecimal (= 3.0.0) fluent-plugin-jq (= 0.5.1) fluent-plugin-kubernetes-metrics! - fluent-plugin-kubernetes_metadata_filter (>= 2.5.3) fluent-plugin-prometheus (>= 2.0) fluent-plugin-record-modifier (= 2.1.0) - fluent-plugin-splunk-hec (>= 1.2.5) - fluentd (>= 1.14.2) + fluent-plugin-splunk-hec (= 1.3.0) + fluentd (>= 1.15.0) http_parser.rb (= 0.8.0) kubeclient (= 4.9.3) - multi_json (= 1.14.1) - oj (= 3.10.18) + multi_json (~> 1.14) + oj (~> 3.10) + rack (>= 2.2.3.1) BUNDLED WITH 2.1.4 diff --git a/fluent-plugin-kubernetes-metrics.gemspec b/fluent-plugin-kubernetes-metrics.gemspec index 8ea51e6..9f1153a 100644 --- a/fluent-plugin-kubernetes-metrics.gemspec +++ b/fluent-plugin-kubernetes-metrics.gemspec @@ -26,6 +26,6 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'webmock', '~> 3.5.1' spec.add_runtime_dependency 'fluentd', '>= 1.9.1' spec.add_runtime_dependency 'kubeclient', '~> 4.9.3' - spec.add_runtime_dependency 'multi_json', '~> 1.14.1' - spec.add_runtime_dependency 'oj', '~> 3.10.2' + spec.add_runtime_dependency 'multi_json', '~> 1.14' + spec.add_runtime_dependency 'oj', '~> 3.10' end diff --git a/lib/fluent/plugin/in_kubernetes_metrics.rb b/lib/fluent/plugin/in_kubernetes_metrics.rb index d50073e..24d3591 100644 --- a/lib/fluent/plugin/in_kubernetes_metrics.rb +++ b/lib/fluent/plugin/in_kubernetes_metrics.rb @@ -166,7 +166,7 @@ def init_without_kubeconfig(_options = {}) } auth_options = {} - auth_options[:bearer_token] = File.read(@bearer_token_file) if @bearer_token_file + auth_options[:bearer_token_file] = @bearer_token_file if @bearer_token_file @client = Kubeclient::Client.new( @kubernetes_url, 'v1', @@ -244,6 +244,11 @@ def set_ssl_options ssl_options end + # This method is used to refresh the authorization token for kubeclient + def update_kubeclient_header + @client.headers[:Authorization] = 'Bearer ' + File.read(@bearer_token_file) if @bearer_token_file + end + # This method is used to set the options for sending a request to the kubelet api def request_options options = { method: 'get', url: @kubelet_url } @@ -677,6 +682,7 @@ def scrape_metrics response = RestClient::Request.execute request_options handle_response(response) else + update_kubeclient_header @node_names.each do |node| response = summary_proxy_api(node).get(@client.headers) handle_response(response) @@ -688,6 +694,7 @@ def is_stats_endpoint_available? if @use_rest_client response_stats = RestClient::Request.execute request_options_stats else + update_kubeclient_header @node_names.each do |node| @node_name = node response_stats = stats_proxy_api(node).get(@client.headers) @@ -703,6 +710,7 @@ def scrape_stats_metrics response_stats = RestClient::Request.execute request_options_stats handle_stats_response(response_stats) else + update_kubeclient_header @node_names.each do |node| @node_name = node response_stats = stats_proxy_api(node).get(@client.headers) @@ -716,6 +724,7 @@ def scrape_cadvisor_metrics response_cadvisor = RestClient::Request.execute cadvisor_request_options handle_cadvisor_response(response_cadvisor) else + update_kubeclient_header @node_names.each do |node| response_cadvisor = cadvisor_proxy_api(node).get(@client.headers) handle_cadvisor_response(response_cadvisor) diff --git a/metrics-information.md b/metrics-information.md index a12530c..68489fa 100644 --- a/metrics-information.md +++ b/metrics-information.md @@ -1,6 +1,6 @@ # Metrics Information -## The foillowing table describes common terminology used in this topic. +## The following table describes common terminology used in this topic. | Terminology | Definition | | --- | --- |