From 472242af4536dbd5121271a12fd39abee4f65eba Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Tue, 13 Jul 2021 12:19:15 -0700 Subject: [PATCH 01/13] libupdate (#89) * libupdate --- Gemfile.lock | 28 ++++++++++++++------------- Makefile | 3 +-- VERSION | 2 +- docker/Dockerfile | 5 +---- docker/Gemfile | 2 +- docker/Gemfile.lock | 46 ++++++++++++++++++++++----------------------- docker/build.sh | 3 +-- 7 files changed, 42 insertions(+), 47 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3e9602a..b26684c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-metrics (1.1.5) + fluent-plugin-kubernetes-metrics (1.1.6) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) @@ -10,29 +10,30 @@ PATH GEM remote: https://rubygems.org/ specs: - addressable (2.7.0) + addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) - concurrent-ruby (1.1.8) + concurrent-ruby (1.1.9) cool.io (1.7.1) crack (0.4.5) rexml - docile (1.3.5) + docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.14.2) + ffi (1.15.3) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.12.1) + fluentd (1.13.2) bundler cool.io (>= 1.4.5, < 2.0.0) - http_parser.rb (>= 0.5.1, < 0.7.0) + http_parser.rb (>= 0.5.1, < 0.8.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) strptime (>= 0.2.2, < 1.0.0) tzinfo (>= 1.0, < 3.0) tzinfo-data (~> 1.0) + webrick (>= 1.4.2, < 1.8.0) yajl-ruby (~> 1.0) hashdiff (1.0.1) http (4.4.1) @@ -41,12 +42,12 @@ GEM http-form_data (~> 2.2) http-parser (~> 1.2.0) http-accept (1.7.0) - http-cookie (1.0.3) + http-cookie (1.0.4) domain_name (~> 0.5) http-form_data (2.3.0) http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) - http_parser.rb (0.6.0) + http_parser.rb (0.7.0) json (2.5.1) kubeclient (4.6.0) http (>= 3.0, < 5.0) @@ -54,22 +55,22 @@ GEM rest-client (~> 2.0) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0225) + mime-types-data (3.2021.0704) msgpack (1.4.2) multi_json (1.14.1) netrc (0.11.0) oj (3.10.18) power_assert (2.0.0) public_suffix (4.0.6) - rake (13.0.3) + rake (13.0.6) recursive-open-struct (1.1.3) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.2.4) - serverengine (2.2.3) + rexml (3.2.5) + serverengine (2.2.4) sigdump (~> 0.2.2) sigdump (0.2.4) simplecov (0.16.1) @@ -91,6 +92,7 @@ GEM addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff + webrick (1.7.0) yajl-ruby (1.4.1) PLATFORMS diff --git a/Makefile b/Makefile index 97cf261..46a55e3 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ VERSION := $(shell sh -c 'cat VERSION') -NODEJS_VERSION := 14.15.1 clean_pkg: @rm -rf pkg/* docker/*.gem @@ -17,7 +16,7 @@ docker: build install-deps @cp pkg/fluent-plugin-*.gem docker @mkdir -p docker/licenses @cp -rp LICENSE docker/licenses/ - @docker build --no-cache --pull --build-arg VERSION=$(VERSION) --build-arg NODEJS_VERSION=$(NODEJS_VERSION) -t splunk/k8s-metrics:$(VERSION) ./docker + @docker build --no-cache --pull --build-arg VERSION=$(VERSION) -t splunk/k8s-metrics:$(VERSION) ./docker unit-test: @bundle exec rake test diff --git a/VERSION b/VERSION index 314c3d7..ab67981 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.5 \ No newline at end of file +1.1.6 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index cd43c76..8016370 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,6 @@ FROM registry.access.redhat.com/ubi8/ruby-27 ARG VERSION -ARG NODEJS_VERSION LABEL name="Splunk Connect for Kubernetes Metrics container" \ maintainer="DataEdge@splunk.com" \ @@ -21,9 +20,7 @@ COPY licenses /licenses COPY Gemfile* ./ RUN yum update -y \ - && npm install -g n \ - && yum remove -y nodejs \ - && n ${NODEJS_VERSION} \ + && yum remove -y nodejs npm \ && gem install bundler \ && gem unpack /tmp/*.gem --target gem \ && bundle install diff --git a/docker/Gemfile b/docker/Gemfile index ad14bc0..ba730ee 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -4,7 +4,7 @@ source 'https://rubygems.org' # for kubernetes log collection agent # List all required gems here and install via bundler to resolve dependencies gem "fluentd", "=1.11.5" -gem "fluent-plugin-prometheus", "=1.8.5" +gem "fluent-plugin-prometheus", ">=2.0" gem "fluent-plugin-record-modifier", "=2.1.0" gem "fluent-plugin-kubernetes_metadata_filter", "=2.5.3" gem "fluent-plugin-jq", "=0.5.1" diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 93bcc0b..e41745c 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-metrics (1.1.5) + fluent-plugin-kubernetes-metrics (1.1.6) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) @@ -10,26 +10,26 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (6.1.3) - activesupport (= 6.1.3) - activesupport (6.1.3) + activemodel (6.1.4) + activesupport (= 6.1.4) + activesupport (6.1.4) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - addressable (2.7.0) + addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) aes_key_wrap (1.1.0) attr_required (1.0.1) bigdecimal (3.0.0) - bindata (2.4.8) - concurrent-ruby (1.1.8) - connection_pool (2.2.3) + bindata (2.4.10) + concurrent-ruby (1.1.9) + connection_pool (2.2.5) cool.io (1.7.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.14.2) + ffi (1.15.3) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake @@ -40,17 +40,17 @@ GEM fluentd (>= 0.14.0, < 1.12) kubeclient (< 5) lru_redux - fluent-plugin-prometheus (1.8.5) + fluent-plugin-prometheus (2.0.1) fluentd (>= 1.9.1, < 2) - prometheus-client (< 0.10) + prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.5) + fluent-plugin-splunk-hec (1.2.7) fluentd (>= 1.4) multi_json (~> 1.13) net-http-persistent (~> 3.1) openid_connect (~> 1.1.8) - prometheus-client (< 0.10.0) + prometheus-client (>= 2.1.0) fluentd (1.11.5) cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.7.0) @@ -67,14 +67,14 @@ GEM http-form_data (~> 2.2) http-parser (~> 1.2.0) http-accept (1.7.0) - http-cookie (1.0.3) + http-cookie (1.0.4) domain_name (~> 0.5) http-form_data (2.3.0) http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.5.3) httpclient (2.8.3) - i18n (1.8.9) + i18n (1.8.10) concurrent-ruby (~> 1.0) json-jwt (1.13.0) activesupport (>= 4.2) @@ -89,8 +89,8 @@ GEM mini_mime (>= 0.1.1) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0225) - mini_mime (1.0.2) + mime-types-data (3.2021.0704) + mini_mime (1.1.0) minitest (5.14.4) msgpack (1.4.2) multi_json (1.14.1) @@ -108,25 +108,23 @@ GEM validate_email validate_url webfinger (>= 1.0.1) - prometheus-client (0.9.0) - quantile (~> 0.2.1) + prometheus-client (2.1.0) public_suffix (4.0.6) - quantile (0.2.1) rack (2.2.3) - rack-oauth2 (1.16.0) + rack-oauth2 (1.17.0) activesupport attr_required httpclient json-jwt (>= 1.11.0) rack (>= 2.1.0) - rake (13.0.3) + rake (13.0.6) recursive-open-struct (1.1.3) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - serverengine (2.2.3) + serverengine (2.2.4) sigdump (~> 0.2.2) sigdump (0.2.4) strptime (0.2.5) @@ -161,7 +159,7 @@ DEPENDENCIES fluent-plugin-jq (= 0.5.1) fluent-plugin-kubernetes-metrics! fluent-plugin-kubernetes_metadata_filter (= 2.5.3) - fluent-plugin-prometheus (= 1.8.5) + fluent-plugin-prometheus (>= 2.0) fluent-plugin-record-modifier (= 2.1.0) fluent-plugin-splunk-hec (>= 1.2.5) fluentd (= 1.11.5) diff --git a/docker/build.sh b/docker/build.sh index fdf1ca9..a0601e1 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -e TAG=$1 -NODEJS_VERSION=14.15.1 # Install dependecies gem install bundler @@ -17,4 +16,4 @@ VERSION=`cat VERSION` echo "Copying licenses to be included in the docker image..." mkdir -p docker/licenses cp -rp LICENSE docker/licenses/ -docker build --no-cache --pull --build-arg VERSION=$VERSION --build-arg NODEJS_VERSION=$NODEJS_VERSION --no-cache -t splunk/k8s-metrics:$TAG ./docker +docker build --no-cache --pull --build-arg VERSION=$VERSION --no-cache -t splunk/k8s-metrics:$TAG ./docker From f3d67806616a81ddbfb14b0e9afbdcac09f72ca7 Mon Sep 17 00:00:00 2001 From: Mei Wang Date: Fri, 10 Sep 2021 16:00:07 -0700 Subject: [PATCH 02/13] Add github actions ci pipeline (#92) * Add github actions ci pipeline * removed CircleCi pipeline --- .circleci/config.yml | 88 ------------ .circleci/trigger_func_test.sh | 42 ------ .github/workflows/ci_build_test.yaml | 164 +++++++++++++++++++++++ README.md | 1 - ci_scripts/deploy_connector.sh | 27 ++++ {.circleci => ci_scripts}/install_dep.sh | 2 +- ci_scripts/k8s-splunk.yml | 31 +++++ 7 files changed, 223 insertions(+), 132 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100755 .circleci/trigger_func_test.sh create mode 100644 .github/workflows/ci_build_test.yaml create mode 100755 ci_scripts/deploy_connector.sh rename {.circleci => ci_scripts}/install_dep.sh (75%) create mode 100644 ci_scripts/k8s-splunk.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 5c0b0c3..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,88 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - checkout - - setup_remote_docker: - reusable: true - - run: - name: Install dependencies - command: | - .circleci/install_dep.sh - - run: - name: Builder - command: | - rake build -t -v - cp -R pkg /tmp - - persist_to_workspace: - root: /tmp - paths: - - pkg - - unit_test: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - attach_workspace: - at: /tmp - - checkout - - setup_remote_docker: - reusable: true - - run: - name: Install dependencies - command: | - .circleci/install_dep.sh - - run: - name: Run unit tests - command: | - bundle exec rake test -t -v - - func_test_main: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - checkout - - setup_remote_docker: - reusable: true - - run: - name: trigger - command: | - .circleci/trigger_func_test.sh main - - func_test_develop: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - checkout - - setup_remote_docker: - reusable: true - - run: - name: trigger - command: | - .circleci/trigger_func_test.sh develop - -workflows: - version: 2 - build_test_push: - jobs: - - build: - filters: - branches: - ignore: - - /^release\/.*/ - - main - - unit_test: - requires: - - build - - func_test_main: - requires: - - unit_test - - func_test_develop: - requires: - - unit_test \ No newline at end of file diff --git a/.circleci/trigger_func_test.sh b/.circleci/trigger_func_test.sh deleted file mode 100755 index a08745a..0000000 --- a/.circleci/trigger_func_test.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# trigger SCK 'main' branch to introduce this new image from this commit -# to working version of every other component. -ORGANIZATION=splunk -PROJECT=splunk-connect-for-kubernetes -BRANCH=$1 - -# Trigger functional test -curl -X POST --header "Content-Type: application/json" \ - -d '{"build_parameters": {"CIRCLE_JOB":"build_test", "TRIG_BRANCH":"'"$CIRCLE_BRANCH"'", "TRIG_PROJECT":"'"$CIRCLE_PROJECT_REPONAME"'", "TRIG_REPO":"'"$CIRCLE_REPOSITORY_URL"'"}}' "https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/tree/$BRANCH?circle-token=$CIRCLE_TOKEN" > build.json -cat build.json -BUILD_NUM=$(jq -r .build_num build.json) -sleep 10 - -# Wait until finish or maximum 20 minutes -TIMEOUT=20 -DONE="FALSE" -until [ "$TIMEOUT" -lt 0 ] || [ "$DONE" == "TRUE" ]; do - curl https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/$BUILD_NUM?circle-token=$CIRCLE_TOKEN > build_progress.json - STATUS=$(jq -r .status build_progress.json) - echo "STATUS = $STATUS" - if [ "$STATUS" != "running" ] && [ "$STATUS" != "queued" ]; then - DONE="TRUE" - else - let TIMEOUT-- - sleep 60 - fi -done - -BUILD_URL=$(jq -r .build_url build_progress.json) -if [ "$DONE" == "FALSE" ]; then - # Cancel hanging job and fail - curl -X POST https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/$BUILD_NUM/cancel?circle-token=$CIRCLE_TOKEN -else - if [ "$STATUS" != "success" ] && [ "$STATUS" != "fixed" ]; then - echo "Functional test have failed please see:" - echo $BUILD_URL - exit 1 - fi - exit 0 -fi diff --git a/.github/workflows/ci_build_test.yaml b/.github/workflows/ci_build_test.yaml new file mode 100644 index 0000000..6b65488 --- /dev/null +++ b/.github/workflows/ci_build_test.yaml @@ -0,0 +1,164 @@ +name: CI Build Test + +on: + pull_request: + branches-ignore: + - /^release\/.*/ + - main + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Ruby 2.6.1 + uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.6.1 + + - name: Install dependencies + run: | + sudo ci_scripts/install_dep.sh + + - name: Builder + run: | + rake build -t -v + cp -R pkg /tmp + + - name: Cache pkg + uses: actions/cache@v1 + with: + path: /tmp + key: ${{ runner.os }}-build + + unit-test: + runs-on: ubuntu-20.04 + needs: + - build + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install dependencies + run: | + sudo ci_scripts/install_dep.sh + + - uses: actions/cache@v2 + with: + path: /tmp + key: ${{ runner.os }}-build + + - name: Run unit tests + run: | + bundle exec rake test -t -v + + func-test: + needs: + - unit-test + runs-on: ubuntu-20.04 + env: + CI_SPLUNK_PORT: 8089 + CI_SPLUNK_USERNAME: admin + CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959 + CI_SPLUNK_PASSWORD: helloworld + CI_INDEX_EVENTS: ci_events + CI_INDEX_OBJECTS: ci_objects + CI_INDEX_METRICS: ci_metrics + KUBERNETES_VERSION: v1.15.2 + MINIKUBE_VERSION: v1.21.0 + GITHUB_ACTIONS: true + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Build kubernetes-metrics image + run: | + sed -i 's/VERSION=$VERSION -t splunk\/k8s-metrics:$TAG .\/docker/VERSION=$VERSION -t k8s-metrics:$TAG .\/docker/' docker/build.sh + sudo docker/build.sh $GITHUB_RUN_ID + docker image ls + + - name: Setup Minikube + run: | + # Install Kubectl + curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ + mkdir -p ${HOME}/.kube + touch ${HOME}/.kube/config + # Install Minikube + curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64 + chmod +x minikube + sudo mv minikube /usr/local/bin/ + # Start Minikube and Wait + minikube start --driver=docker --container-runtime=docker --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check + export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' + until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do + sleep 1; + done + + - name: Install Splunk + run: | + # Wait until minikube is ready + kubectl apply -f https://docs.projectcalico.org/v3.14/manifests/calico.yaml + export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' + until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do + echo "wait for minikube ready ..." + sleep 1; + done + kubectl get nodes + # Install Splunk on minikube + kubectl apply -f ci_scripts/k8s-splunk.yml + # Wait until splunk is ready + until kubectl logs splunk --tail=2 | grep -q 'Ansible playbook complete'; do + sleep 1; + done + export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}}) + # Setup Indexes + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_EVENTS -d datatype=event + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_OBJECTS -d datatype=event + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_METRICS -d datatype=metric + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=default-events -d datatype=event + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=ns-anno -d datatype=event + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=pod-anno -d datatype=event + # Enable HEC services + curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable + # Create new HEC token + curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=a6b5e77f-d5f6-415a-bd43-930cecb12959&disabled=0&index=default-events&indexes=default-events,$CI_INDEX_METRICS,$CI_INDEX_OBJECTS,$CI_INDEX_EVENTS,ns-anno,pod-anno" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http + # Restart Splunk + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/server/control/restart -X POST + + - name: Deploy k8s connector + run: | + export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}}) + ci_scripts/deploy_connector.sh + + - name: Deploy log generator + run: | + cd /opt/splunk-connect-for-kubernetes + kubectl apply -f test/test_setup.yaml + sleep 60 + + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Run functional tests + run: | + echo "check the pods" + kubectl get pods -A + cd /opt/splunk-connect-for-kubernetes + kubectl get nodes + export PYTHONWARNINGS="ignore:Unverified HTTPS request" + export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}}) + cd test + pip install --upgrade pip + pip install -r requirements.txt + echo "Running functional tests....." + python -m pytest \ + --splunkd-url https://$CI_SPLUNK_HOST:8089 \ + --splunk-user admin \ + --splunk-password $CI_SPLUNK_PASSWORD \ + -p no:warnings -s diff --git a/README.md b/README.md index b87869c..3891de1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -[![CircleCI](https://circleci.com/gh/git-lfs/git-lfs.svg?style=shield&circle-token=856152c2b02bfd236f54d21e1f581f3e4ebf47ad)](https://circleci.com/gh/splunk/fluent-plugin-kubernetes-metrics) # Fluentd Plugin for Kubernetes Metrics The [Fluentd](https://fluentd.org/) input plugin collects Kubernetes cluster metrics which are exposed by the [Kubelet API](https://kubernetes.io/docs/admin/kubelet/) and forwards them to fluentd. diff --git a/ci_scripts/deploy_connector.sh b/ci_scripts/deploy_connector.sh new file mode 100755 index 0000000..2af97c1 --- /dev/null +++ b/ci_scripts/deploy_connector.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -e + +#Make sure to check and clean previously failed deployment +echo "Checking if previous deployment exist..." +if [ "`helm ls --short`" == "" ]; then + echo "Nothing to clean, ready for deployment" +else + helm delete $(helm ls --short) +fi + +# Clone splunk-connect-for-kubernetes repo +cd /opt +git clone https://github.com/splunk/splunk-connect-for-kubernetes.git +cd splunk-connect-for-kubernetes + +echo "Deploying k8s-connect with latest changes" +helm install ci-sck --set global.splunk.hec.token=$CI_SPLUNK_HEC_TOKEN \ +--set global.splunk.hec.host=$CI_SPLUNK_HOST \ +--set kubelet.serviceMonitor.https=true \ +--set splunk-kubernetes-metrics.image.repository=k8s-metrics \ +--set splunk-kubernetes-metrics.image.pullPolicy=IfNotPresent \ +-f ci_scripts/sck_values.yml helm-chart/splunk-connect-for-kubernetes +#wait for deployment to finish +until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do + sleep 1; +done \ No newline at end of file diff --git a/.circleci/install_dep.sh b/ci_scripts/install_dep.sh similarity index 75% rename from .circleci/install_dep.sh rename to ci_scripts/install_dep.sh index 7c4d131..4fb02aa 100755 --- a/.circleci/install_dep.sh +++ b/ci_scripts/install_dep.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +sudo gem update --system gem install bundler bundle update --bundler bundle install \ No newline at end of file diff --git a/ci_scripts/k8s-splunk.yml b/ci_scripts/k8s-splunk.yml new file mode 100644 index 0000000..543c6ca --- /dev/null +++ b/ci_scripts/k8s-splunk.yml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Pod +metadata: + name: splunk +spec: + hostNetwork: true + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: splunk + image: splunk/splunk:latest + ports: + - containerPort: 8000 + hostPort: 8000 + protocol: TCP + - containerPort: 8088 + hostPort: 8088 + protocol: TCP + - containerPort: 8089 + hostPort: 8089 + protocol: TCP + env: + - name: SPLUNK_START_ARGS + value: --accept-license + - name: SPLUNK_USER + value: root + - name: SPLUNK_PASSWORD + value: helloworld + - name: SPLUNK_LAUNCH_CONF + value: OPTIMISTIC_ABOUT_FILE_LOCKING=1 From 5140650670808b25f8063110c013df77d238a76e Mon Sep 17 00:00:00 2001 From: foram-splunk <89519924+foram-splunk@users.noreply.github.com> Date: Fri, 29 Oct 2021 01:43:49 +0530 Subject: [PATCH 03/13] Update python3 for Red Hat Enterprise Linux 8 (#94) Co-authored-by: foram-splunk --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8016370..2bfb15c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,7 +23,8 @@ RUN yum update -y \ && yum remove -y nodejs npm \ && gem install bundler \ && gem unpack /tmp/*.gem --target gem \ - && bundle install + && bundle install \ + && rpm -e --nodeps python3-pip-wheel RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ From 1d024a1375439c3dbe0ee5dbbb33d3db8f91964f Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Tue, 2 Nov 2021 16:25:31 -0700 Subject: [PATCH 04/13] remove unnecessary rpm packages (#95) --- Gemfile.lock | 2 +- docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b26684c..bbbe6b6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -107,4 +107,4 @@ DEPENDENCIES webmock (~> 3.5.1) BUNDLED WITH - 2.1.4 + 2.2.23 diff --git a/docker/Dockerfile b/docker/Dockerfile index 2bfb15c..f3691ec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,7 +24,7 @@ RUN yum update -y \ && gem install bundler \ && gem unpack /tmp/*.gem --target gem \ && bundle install \ - && rpm -e --nodeps python3-pip-wheel + && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ From 9eef30232ef395c7fc7d0abdf32e345bf18a02aa Mon Sep 17 00:00:00 2001 From: hbrewster Date: Wed, 3 Nov 2021 13:35:53 +1100 Subject: [PATCH 05/13] Feat: Multi-arch containers --- .github/workflows/ci_build_test.yaml | 34 +++++++++++++++++++++++++--- ci_scripts/deploy_connector.sh | 6 +++-- docker/Dockerfile | 18 ++++++++++++--- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_build_test.yaml b/.github/workflows/ci_build_test.yaml index 6b65488..44c1b3e 100644 --- a/.github/workflows/ci_build_test.yaml +++ b/.github/workflows/ci_build_test.yaml @@ -74,10 +74,38 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Build kubernetes-metrics image + - name: Prepare container build + id: prep + run: | + VERSION=`cat VERSION` + TAGS=splunk/k8s-metrics:recent + + echo ::set-output name=tags::${TAGS} + echo ::set-output name=version::${VERSION} + + - name: Set up QEMU + uses: docker/setup-qemu-action@master + with: + platforms: all + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@master + + - name: Build multi-arch kubernetes-metrics image + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: ./docker/Dockerfile + platforms: linux/amd64 + push: false + load: true + tags: ${{ steps.prep.outputs.tags }} + build-args: VERSION=${{ steps.prep.outputs.version }} + + - name: Check kubernetes-metrics image run: | - sed -i 's/VERSION=$VERSION -t splunk\/k8s-metrics:$TAG .\/docker/VERSION=$VERSION -t k8s-metrics:$TAG .\/docker/' docker/build.sh - sudo docker/build.sh $GITHUB_RUN_ID docker image ls - name: Setup Minikube diff --git a/ci_scripts/deploy_connector.sh b/ci_scripts/deploy_connector.sh index 2af97c1..b2d5a55 100755 --- a/ci_scripts/deploy_connector.sh +++ b/ci_scripts/deploy_connector.sh @@ -14,14 +14,16 @@ cd /opt git clone https://github.com/splunk/splunk-connect-for-kubernetes.git cd splunk-connect-for-kubernetes +minikube image load splunk/k8s-metrics:recent + echo "Deploying k8s-connect with latest changes" helm install ci-sck --set global.splunk.hec.token=$CI_SPLUNK_HEC_TOKEN \ --set global.splunk.hec.host=$CI_SPLUNK_HOST \ --set kubelet.serviceMonitor.https=true \ ---set splunk-kubernetes-metrics.image.repository=k8s-metrics \ +--set splunk-kubernetes-metrics.image.tag=recent \ --set splunk-kubernetes-metrics.image.pullPolicy=IfNotPresent \ -f ci_scripts/sck_values.yml helm-chart/splunk-connect-for-kubernetes #wait for deployment to finish until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do sleep 1; -done \ No newline at end of file +done diff --git a/docker/Dockerfile b/docker/Dockerfile index 8016370..910e4d4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,13 @@ +FROM ruby:2.7.4-buster as builder + +ADD ./ /app/ +WORKDIR /app +RUN gem install bundler +RUN bundle update --bundler +RUN bundle install + +RUN bundle exec rake build -t -v + FROM registry.access.redhat.com/ubi8/ruby-27 ARG VERSION @@ -15,10 +25,12 @@ ENV FLUENT_USER fluent USER root -COPY *.gem /tmp/ -COPY licenses /licenses +COPY --from=builder /app/pkg/fluent-plugin-*.gem /tmp/ + +RUN mkdir /licenses +COPY --from=builder /app/LICENSE /licenses/LICENSE -COPY Gemfile* ./ +COPY --from=builder /app/docker/Gemfile* ./ RUN yum update -y \ && yum remove -y nodejs npm \ && gem install bundler \ From 3e878efa5860c17fec9131ac201eca936b4332c0 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Wed, 3 Nov 2021 16:03:04 -0700 Subject: [PATCH 06/13] 1.1.7 (#97) --- Gemfile.lock | 18 ++++++++--------- Makefile | 8 +++----- VERSION | 2 +- docker/Gemfile | 4 ++-- docker/Gemfile.lock | 49 ++++++++++++++++++++++++--------------------- 5 files changed, 41 insertions(+), 40 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index bbbe6b6..5f60be1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-metrics (1.1.6) + fluent-plugin-kubernetes-metrics (1.1.7) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) @@ -19,18 +19,18 @@ GEM docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.15.3) + ffi (1.15.4) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.13.2) + fluentd (1.14.2) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.8.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) - strptime (>= 0.2.2, < 1.0.0) + strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) tzinfo-data (~> 1.0) webrick (>= 1.4.2, < 1.8.0) @@ -48,19 +48,19 @@ GEM http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.7.0) - json (2.5.1) + json (2.6.1) kubeclient (4.6.0) http (>= 3.0, < 5.0) recursive-open-struct (~> 1.0, >= 1.0.4) rest-client (~> 2.0) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0704) + mime-types-data (3.2021.0901) msgpack (1.4.2) multi_json (1.14.1) netrc (0.11.0) oj (3.10.18) - power_assert (2.0.0) + power_assert (2.0.1) public_suffix (4.0.6) rake (13.0.6) recursive-open-struct (1.1.3) @@ -83,11 +83,11 @@ GEM power_assert tzinfo (2.0.4) concurrent-ruby (~> 1.0) - tzinfo-data (1.2021.1) + tzinfo-data (1.2021.5) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext - unf_ext (0.0.7.7) + unf_ext (0.0.8) webmock (3.5.1) addressable (>= 2.3.6) crack (>= 0.3.2) diff --git a/Makefile b/Makefile index 46a55e3..8bad0df 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,9 @@ clean: clean_pkg clean_gems build: clean_pkg @bundle exec rake build -docker: build install-deps - @cp pkg/fluent-plugin-*.gem docker - @mkdir -p docker/licenses - @cp -rp LICENSE docker/licenses/ - @docker build --no-cache --pull --build-arg VERSION=$(VERSION) -t splunk/k8s-metrics:$(VERSION) ./docker +.PHONY: docker +docker: + @docker build --no-cache --pull --build-arg VERSION=$(VERSION) -t splunk/k8s-metrics:$(VERSION) . -f docker/Dockerfile unit-test: @bundle exec rake test diff --git a/VERSION b/VERSION index ab67981..a5ba932 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.6 \ No newline at end of file +1.1.7 \ No newline at end of file diff --git a/docker/Gemfile b/docker/Gemfile index ba730ee..84c7fb4 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -3,10 +3,10 @@ source 'https://rubygems.org' # This is separate gemfile for building docker image that has all plugins # for kubernetes log collection agent # List all required gems here and install via bundler to resolve dependencies -gem "fluentd", "=1.11.5" +gem "fluentd", ">=1.14.2" gem "fluent-plugin-prometheus", ">=2.0" gem "fluent-plugin-record-modifier", "=2.1.0" -gem "fluent-plugin-kubernetes_metadata_filter", "=2.5.3" +gem "fluent-plugin-kubernetes_metadata_filter", ">=2.5.3" gem "fluent-plugin-jq", "=0.5.1" gem "oj", "=3.10.18" gem 'multi_json', '=1.14.1' diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index e41745c..dc78522 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-metrics (1.1.6) + fluent-plugin-kubernetes-metrics (1.1.7) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) @@ -10,9 +10,9 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (6.1.4) - activesupport (= 6.1.4) - activesupport (6.1.4) + activemodel (6.1.4.1) + activesupport (= 6.1.4.1) + activesupport (6.1.4.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -29,18 +29,18 @@ GEM cool.io (1.7.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.15.3) + ffi (1.15.4) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake fluent-plugin-jq (0.5.1) fluentd (>= 0.14.10, < 2) multi_json (~> 1.13) - fluent-plugin-kubernetes_metadata_filter (2.5.3) - fluentd (>= 0.14.0, < 1.12) - kubeclient (< 5) + fluent-plugin-kubernetes_metadata_filter (2.9.2) + fluentd (>= 0.14.0, < 1.15) + kubeclient (>= 4.0.0, < 5.0.0) lru_redux - fluent-plugin-prometheus (2.0.1) + fluent-plugin-prometheus (2.0.2) fluentd (>= 1.9.1, < 2) prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) @@ -51,15 +51,17 @@ GEM net-http-persistent (~> 3.1) openid_connect (~> 1.1.8) prometheus-client (>= 2.1.0) - fluentd (1.11.5) + fluentd (1.14.2) + bundler cool.io (>= 1.4.5, < 2.0.0) - http_parser.rb (>= 0.5.1, < 0.7.0) + http_parser.rb (>= 0.5.1, < 0.8.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) - strptime (>= 0.2.2, < 1.0.0) + strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) tzinfo-data (~> 1.0) + webrick (>= 1.4.2, < 1.8.0) yajl-ruby (~> 1.0) http (4.4.1) addressable (~> 2.3) @@ -74,7 +76,7 @@ GEM ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.5.3) httpclient (2.8.3) - i18n (1.8.10) + i18n (1.8.11) concurrent-ruby (~> 1.0) json-jwt (1.13.0) activesupport (>= 4.2) @@ -89,8 +91,8 @@ GEM mini_mime (>= 0.1.1) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0704) - mini_mime (1.1.0) + mime-types-data (3.2021.0901) + mini_mime (1.1.2) minitest (5.14.4) msgpack (1.4.2) multi_json (1.14.1) @@ -111,7 +113,7 @@ GEM prometheus-client (2.1.0) public_suffix (4.0.6) rack (2.2.3) - rack-oauth2 (1.17.0) + rack-oauth2 (1.19.0) activesupport attr_required httpclient @@ -128,28 +130,29 @@ GEM sigdump (~> 0.2.2) sigdump (0.2.4) strptime (0.2.5) - swd (1.2.0) + swd (1.3.0) activesupport (>= 3) attr_required (>= 0.0.5) httpclient (>= 2.4) tzinfo (2.0.4) concurrent-ruby (~> 1.0) - tzinfo-data (1.2021.1) + tzinfo-data (1.2021.5) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext - unf_ext (0.0.7.7) + unf_ext (0.0.8) validate_email (0.1.6) activemodel (>= 3.0) mail (>= 2.2.5) validate_url (1.0.13) activemodel (>= 3.0.0) public_suffix - webfinger (1.1.0) + webfinger (1.2.0) activesupport httpclient (>= 2.4) + webrick (1.7.0) yajl-ruby (1.4.1) - zeitwerk (2.4.2) + zeitwerk (2.5.1) PLATFORMS ruby @@ -158,11 +161,11 @@ DEPENDENCIES bigdecimal (= 3.0.0) fluent-plugin-jq (= 0.5.1) fluent-plugin-kubernetes-metrics! - fluent-plugin-kubernetes_metadata_filter (= 2.5.3) + fluent-plugin-kubernetes_metadata_filter (>= 2.5.3) fluent-plugin-prometheus (>= 2.0) fluent-plugin-record-modifier (= 2.1.0) fluent-plugin-splunk-hec (>= 1.2.5) - fluentd (= 1.11.5) + fluentd (>= 1.14.2) http_parser.rb (= 0.5.3) kubeclient (= 4.6.0) multi_json (= 1.14.1) From 10744c9d7b865bdf504122366c9856672f68d8e1 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Wed, 3 Nov 2021 18:23:30 -0700 Subject: [PATCH 07/13] error handle (#99) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9eded12..84a6e0f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,7 +36,7 @@ RUN yum update -y \ && gem install bundler \ && gem unpack /tmp/*.gem --target gem \ && bundle install \ - && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* + && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* || true RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ From 65710ae032639d11d8a539d7b1360d0591a204f4 Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Tue, 16 Nov 2021 05:54:38 +0530 Subject: [PATCH 08/13] Check availability of stats endpoint before starting scrapping from stats endpoint (#102) * check stats endpoint availability before scraping * updated log message and added comment --- lib/fluent/plugin/in_kubernetes_metrics.rb | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/fluent/plugin/in_kubernetes_metrics.rb b/lib/fluent/plugin/in_kubernetes_metrics.rb index 5bd257b..12385e2 100644 --- a/lib/fluent/plugin/in_kubernetes_metrics.rb +++ b/lib/fluent/plugin/in_kubernetes_metrics.rb @@ -90,8 +90,13 @@ def start super timer_execute :metric_scraper, @interval, &method(:scrape_metrics) - timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics) timer_execute :cadvisor_metric_scraper, @interval, &method(:scrape_cadvisor_metrics) + # It is done to optionally fetch from 'stats' for k8s version <1.21 + if is_stats_endpoint_available? + timer_execute :stats_metric_scraper, @interval, &method(:scrape_stats_metrics) + else + log.info "'/stats' endpoint is not available. It has been deprecated since k8s v1.15, disabled since v1.18, and removed in v1.21 and onwards" + end end def close @@ -642,6 +647,20 @@ def scrape_metrics end end + def is_stats_endpoint_available? + if @use_rest_client + response_stats = RestClient::Request.execute request_options_stats + else + @node_names.each do |node| + @node_name = node + response_stats = stats_proxy_api(node).get(@client.headers) + end + end + true + rescue RestClient::NotFound + false + end + def scrape_stats_metrics if @use_rest_client response_stats = RestClient::Request.execute request_options_stats From 1c6ee72bfa839ca153e1d9c46d76724936ba90a2 Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Wed, 8 Dec 2021 06:33:21 +0530 Subject: [PATCH 09/13] updated emit_cadvisor_metrics (#103) * updated emit_cadvisor_metrics --- lib/fluent/plugin/in_kubernetes_metrics.rb | 29 +++++++++++----------- test/plugin/test_in_kubernetes_metrics.rb | 25 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/lib/fluent/plugin/in_kubernetes_metrics.rb b/lib/fluent/plugin/in_kubernetes_metrics.rb index 12385e2..bf5312a 100644 --- a/lib/fluent/plugin/in_kubernetes_metrics.rb +++ b/lib/fluent/plugin/in_kubernetes_metrics.rb @@ -602,33 +602,34 @@ def emit_stats_metrics(metrics) emit_stats_breakdown(metrics['stats']) unless metrics['stats'].nil? end + # Make sure regex has only one capturing group + def grep_using_regex(metric, regex) + match = metric.match(regex) + return nil if match.nil? + match[1] + end + def emit_cadvisor_metrics(metrics) metrics = metrics.split("\n") metrics.each do |metric| - next unless metric.include? 'container_name=' - - next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#' - + + next if metric[0] == '#' or not container_name = grep_using_regex(metric, /container(?:_name)?="([^"]*)"/) + metric_str, metric_val = metric.split(' ') metric_val = metric_val.to_f if metric_val.is_a? String first_occur = metric_str.index('{') metric_name = metric_str[0..first_occur - 1] - pod_name = metric.match(/pod_name="\S*"/).to_s - pod_name = pod_name.split('"')[1] - image_name = metric.match(/image="\S*"/).to_s - image_name = image_name.split('"')[1] - namespace = metric.match(/namespace="\S*"/).to_s - namespace = namespace.split('"')[1] + pod_name = grep_using_regex(metric, /pod(?:_name)?="([^"]*)"/).to_s + image_name = grep_using_regex(metric, /image="([^"]*)"/).to_s + namespace = grep_using_regex(metric, /namespace="([^"]*)"/).to_s metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name } - if metric =~ /^((?!container_name="POD").)*$/ + if container_name=="POD" tag = 'pod' tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}") tag = tag.gsub('container', '') else - container_name = metric.match(/container_name="\S*"/).to_s - container_name = container_name.split('"')[1] container_label = { 'container_name' => container_name } - metric_labels.merge(container_label) + metric_labels.merge!(container_label) tag = generate_tag(metric_name.tr('_', '.').to_s) end router.emit tag, @scraped_at_cadvisor, metric_labels diff --git a/test/plugin/test_in_kubernetes_metrics.rb b/test/plugin/test_in_kubernetes_metrics.rb index 3b2431d..a873875 100644 --- a/test/plugin/test_in_kubernetes_metrics.rb +++ b/test/plugin/test_in_kubernetes_metrics.rb @@ -35,6 +35,9 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase ).freeze setup do + stub_k8s_requests + + return unless @@hash_map_test.empty? Fluent::Test.setup @@parsed_unit_string = JSON.parse(get_unit_parsed_string) @@ -45,8 +48,6 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase get_cadvisor_parsed_string = f.read end.close - stub_k8s_requests - @@ca_driver = create_driver @@ca_driver.run timeout: 20, expect_emits: 1, shutdown: true @@ -56,8 +57,10 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase metrics = get_cadvisor_parsed_string.split("\n") metrics.each do |metric| next unless metric.include? 'container_name=' + next unless metric[0] != '#' - next unless metric.match(/^((?!container_name="").)*$/) && metric[0] != '#' + container_name = metric.match(/container_name="\S*"/).to_s + container_name = container_name.split('"')[1] metric_str, metric_val = metric.split(' ') metric_val = metric_val.to_f if metric_val.is_a? String @@ -70,13 +73,11 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase namespace = metric.match(/namespace="\S*"/).to_s namespace = namespace.split('"')[1] metric_labels = { 'pod_name' => pod_name, 'image' => image_name, 'namespace' => namespace, 'value' => metric_val, 'node' => @node_name } - if metric =~ /^((?!container_name="POD").)*$/ + if container_name == 'POD' tag = 'pod' tag = generate_tag("#{tag}#{metric_name.tr('_', '.')}", @@driver.instance.tag) tag = tag.gsub('container', '') else - container_name = metric.match(/container_name="\S*"/).to_s - container_name = container_name.split('"')[1] container_label = { 'container_name' => container_name } metric_labels.merge(container_label) tag = generate_tag(metric_name.tr('_', '.').to_s, @@driver.instance.tag) @@ -252,11 +253,10 @@ def create_driver(conf = CONFIG) assert_true @@hash_map_cadvisor.key?('kube.container.fs.read.seconds.total') assert_equal @@hash_map_cadvisor['kube.container.fs.read.seconds.total'], @@hash_map_test['kube.container.fs.read.seconds.total'][2]['value'] end - - # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production + test 'Test - metrics cadvisor: container_fs_reads_bytes_total' do - assert_false @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total') - # assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"] + assert_true @@hash_map_cadvisor.key?('kube.container.fs.reads.bytes.total') + assert_equal @@hash_map_cadvisor['kube.container.fs.reads.bytes.total'], @@hash_map_test["kube.container.fs.reads.bytes.total"][2]["value"] end test 'Test - metrics cadvisor: container_fs_reads_merged_total' do @@ -289,10 +289,9 @@ def create_driver(conf = CONFIG) assert_equal @@hash_map_cadvisor['kube.container.fs.write.seconds.total'], @@hash_map_test['kube.container.fs.write.seconds.total'][2]['value'] end - # TODO: Current Test does not work - metric present in metrics_cadvisor.txt but not being parsed by connector in test/working in production test 'Test - metrics cadvisor: container_fs_writes_bytes_total' do - assert_false @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total') - # assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"] + assert_true @@hash_map_cadvisor.key?('kube.container.fs.writes.bytes.total') + assert_equal @@hash_map_cadvisor['kube.container.fs.writes.bytes.total'], @@hash_map_test["kube.container.fs.writes.bytes.total"][2]["value"] end test 'Test - metrics cadvisor: container_fs_writes_merged_total' do From 994ddf3493227cad0a7dab18ac4bb2445372de9d Mon Sep 17 00:00:00 2001 From: harshit-splunk Date: Wed, 8 Dec 2021 12:39:50 +0530 Subject: [PATCH 10/13] Ignored cAdvisor metric with empty container name --- lib/fluent/plugin/in_kubernetes_metrics.rb | 1 + test/metrics_cadvisor.txt | 8 ++++++++ test/plugin/test_in_kubernetes_metrics.rb | 2 ++ 3 files changed, 11 insertions(+) diff --git a/lib/fluent/plugin/in_kubernetes_metrics.rb b/lib/fluent/plugin/in_kubernetes_metrics.rb index bf5312a..1face14 100644 --- a/lib/fluent/plugin/in_kubernetes_metrics.rb +++ b/lib/fluent/plugin/in_kubernetes_metrics.rb @@ -614,6 +614,7 @@ def emit_cadvisor_metrics(metrics) metrics.each do |metric| next if metric[0] == '#' or not container_name = grep_using_regex(metric, /container(?:_name)?="([^"]*)"/) + next if container_name.empty? metric_str, metric_val = metric.split(' ') metric_val = metric_val.to_f if metric_val.is_a? String diff --git a/test/metrics_cadvisor.txt b/test/metrics_cadvisor.txt index c19c85e..bda90aa 100644 --- a/test/metrics_cadvisor.txt +++ b/test/metrics_cadvisor.txt @@ -907,6 +907,7 @@ container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstab container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 1.786404787e+09 container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 1.786399729e+09 container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 1.786393994e+09 +container_network_receive_bytes_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad00container_network_receive_errors_total4a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 1.786399238e+09 # HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving # TYPE container_network_receive_errors_total counter container_network_receive_errors_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0 @@ -918,6 +919,7 @@ container_network_receive_errors_total{container_name="POD",id="/kubepods/bursta container_network_receive_errors_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_receive_errors_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_receive_errors_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0 +container_network_receive_errors_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0 # HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving # TYPE container_network_receive_packets_dropped_total counter container_network_receive_packets_dropped_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0 @@ -929,6 +931,7 @@ container_network_receive_packets_dropped_total{container_name="POD",id="/kubepo container_network_receive_packets_dropped_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_receive_packets_dropped_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_receive_packets_dropped_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0 +container_network_receive_packets_dropped_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0 # HELP container_network_receive_packets_total Cumulative count of packets received # TYPE container_network_receive_packets_total counter container_network_receive_packets_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 7.074054e+06 @@ -940,6 +943,7 @@ container_network_receive_packets_total{container_name="POD",id="/kubepods/burst container_network_receive_packets_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 7.073992e+06 container_network_receive_packets_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 7.073969e+06 container_network_receive_packets_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 7.073917e+06 +container_network_receive_packets_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 7.073237e+06 # HELP container_network_tcp_usage_total tcp connection usage statistic for container # TYPE container_network_tcp_usage_total gauge container_network_tcp_usage_total{container_name="",id="/",image="",name="",namespace="",pod_name="",tcp_state="close"} 0 @@ -1250,6 +1254,7 @@ container_network_transmit_bytes_total{container_name="POD",id="/kubepods/bursta container_network_transmit_bytes_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 3.253087401e+09 container_network_transmit_bytes_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 3.253065965e+09 container_network_transmit_bytes_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 3.253037394e+09 +container_network_transmit_bytes_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 3.253037394e+09 # HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting # TYPE container_network_transmit_errors_total counter container_network_transmit_errors_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0 @@ -1261,6 +1266,7 @@ container_network_transmit_errors_total{container_name="POD",id="/kubepods/burst container_network_transmit_errors_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_transmit_errors_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_transmit_errors_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0 +container_network_transmit_errors_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0 # HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting # TYPE container_network_transmit_packets_dropped_total counter container_network_transmit_packets_dropped_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0 @@ -1272,6 +1278,7 @@ container_network_transmit_packets_dropped_total{container_name="POD",id="/kubep container_network_transmit_packets_dropped_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_transmit_packets_dropped_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0 container_network_transmit_packets_dropped_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0 +container_network_transmit_packets_dropped_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0 # HELP container_network_transmit_packets_total Cumulative count of packets transmitted # TYPE container_network_transmit_packets_total counter container_network_transmit_packets_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 7.727546e+06 @@ -1283,6 +1290,7 @@ container_network_transmit_packets_total{container_name="POD",id="/kubepods/burs container_network_transmit_packets_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 7.727485e+06 container_network_transmit_packets_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 7.727451e+06 container_network_transmit_packets_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 7.727389e+06 +container_network_transmit_packets_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 7.727481e+06 # HELP container_network_udp_usage_total udp connection usage statistic for container # TYPE container_network_udp_usage_total gauge container_network_udp_usage_total{container_name="",id="/",image="",name="",namespace="",pod_name="",udp_state="dropped"} 0 diff --git a/test/plugin/test_in_kubernetes_metrics.rb b/test/plugin/test_in_kubernetes_metrics.rb index a873875..7ae81a2 100644 --- a/test/plugin/test_in_kubernetes_metrics.rb +++ b/test/plugin/test_in_kubernetes_metrics.rb @@ -61,6 +61,8 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase container_name = metric.match(/container_name="\S*"/).to_s container_name = container_name.split('"')[1] + + next if container_name.empty? metric_str, metric_val = metric.split(' ') metric_val = metric_val.to_f if metric_val.is_a? String From b608fa8e7820304ec14a11792a1e9ea199efa5ef Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Thu, 16 Dec 2021 01:06:57 -0800 Subject: [PATCH 11/13] 1.1.8 (#106) --- Gemfile.lock | 2 +- VERSION | 2 +- docker/Gemfile.lock | 24 +++++++++++------------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5f60be1..aebbcab 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-metrics (1.1.7) + fluent-plugin-kubernetes-metrics (1.1.8) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) diff --git a/VERSION b/VERSION index a5ba932..db15278 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.7 \ No newline at end of file +1.1.8 \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index dc78522..79d4669 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-metrics (1.1.7) + fluent-plugin-kubernetes-metrics (1.1.8) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) @@ -10,14 +10,13 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (6.1.4.1) - activesupport (= 6.1.4.1) - activesupport (6.1.4.1) + activemodel (7.0.0) + activesupport (= 7.0.0) + activesupport (7.0.0) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - zeitwerk (~> 2.3) addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) aes_key_wrap (1.1.0) @@ -36,7 +35,7 @@ GEM fluent-plugin-jq (0.5.1) fluentd (>= 0.14.10, < 2) multi_json (~> 1.13) - fluent-plugin-kubernetes_metadata_filter (2.9.2) + fluent-plugin-kubernetes_metadata_filter (2.9.3) fluentd (>= 0.14.0, < 1.15) kubeclient (>= 4.0.0, < 5.0.0) lru_redux @@ -45,16 +44,16 @@ GEM prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.7) + fluent-plugin-splunk-hec (1.2.9) fluentd (>= 1.4) multi_json (~> 1.13) net-http-persistent (~> 3.1) openid_connect (~> 1.1.8) prometheus-client (>= 2.1.0) - fluentd (1.14.2) + fluentd (1.14.3) bundler cool.io (>= 1.4.5, < 2.0.0) - http_parser.rb (>= 0.5.1, < 0.8.0) + http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) @@ -89,11 +88,11 @@ GEM lru_redux (1.1.0) mail (2.7.1) mini_mime (>= 0.1.1) - mime-types (3.3.1) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0901) + mime-types-data (3.2021.1115) mini_mime (1.1.2) - minitest (5.14.4) + minitest (5.15.0) msgpack (1.4.2) multi_json (1.14.1) net-http-persistent (3.1.0) @@ -152,7 +151,6 @@ GEM httpclient (>= 2.4) webrick (1.7.0) yajl-ruby (1.4.1) - zeitwerk (2.5.1) PLATFORMS ruby From 67d765753734d2fd95b9e7378712fffc01ef151a Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Wed, 26 Jan 2022 01:01:29 +0530 Subject: [PATCH 12/13] converted divisor to float for cpu metrics (#108) * converted divisor to float for cpu metrics * updated unit test --- lib/fluent/plugin/in_kubernetes_metrics.rb | 8 ++++---- test/plugin/test_in_kubernetes_metrics.rb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/fluent/plugin/in_kubernetes_metrics.rb b/lib/fluent/plugin/in_kubernetes_metrics.rb index 1face14..261192c 100644 --- a/lib/fluent/plugin/in_kubernetes_metrics.rb +++ b/lib/fluent/plugin/in_kubernetes_metrics.rb @@ -315,7 +315,7 @@ def emit_cpu_metrics(tag:, metrics:, labels:) unless metrics['time'].nil? time = parse_time metrics['time'] if usage_rate = metrics['usageNanoCores'] - router.emit generate_tag("#{tag}.cpu.usage_rate"), time, labels.merge('value' => usage_rate / 1_000_000) + router.emit generate_tag("#{tag}.cpu.usage_rate"), time, labels.merge('value' => usage_rate / 1_000_000.0) end if usage = metrics['usageNanoCores'] router.emit generate_tag("#{tag}.cpu.usage"), time, labels.merge('value' => usage) @@ -416,13 +416,13 @@ def emit_stats_breakdown(stats) def emit_cpu_metrics_stats(tag:, metrics:, labels:, time:) if cpu_usage_total = metrics['usage']['total'] - router.emit generate_tag("#{tag}.cpu.usage.total"), time, labels.merge('value' => cpu_usage_total / 1_000_000) + router.emit generate_tag("#{tag}.cpu.usage.total"), time, labels.merge('value' => cpu_usage_total / 1_000_000.0) end if cpu_usage_user = metrics['usage']['user'] - router.emit generate_tag("#{tag}.cpu.usage.user"), time, labels.merge('value' => cpu_usage_user / 1_000_000) + router.emit generate_tag("#{tag}.cpu.usage.user"), time, labels.merge('value' => cpu_usage_user / 1_000_000.0) end if cpu_usage_system = metrics['usage']['system'] - router.emit generate_tag("#{tag}.cpu.usage.system"), time, labels.merge('value' => cpu_usage_system / 1_000_000) + router.emit generate_tag("#{tag}.cpu.usage.system"), time, labels.merge('value' => cpu_usage_system / 1_000_000.0) end if cpu_cfs_periods = metrics['cfs']['periods'] diff --git a/test/plugin/test_in_kubernetes_metrics.rb b/test/plugin/test_in_kubernetes_metrics.rb index 7ae81a2..cc99f96 100644 --- a/test/plugin/test_in_kubernetes_metrics.rb +++ b/test/plugin/test_in_kubernetes_metrics.rb @@ -115,7 +115,7 @@ def create_driver(conf = CONFIG) assert_equal @@parsed_unit_string['node']['cpu']['usageNanoCores'], @@hash_map_test['kube.node.cpu.usage'][2]['value'] assert_not_nil @@hash_map_test.key?('kube.node.cpu.usage_rate') - assert_equal @@parsed_unit_string['node']['cpu']['usageNanoCores'] / 1_000_000, @@hash_map_test['kube.node.cpu.usage_rate'][2]['value'] + assert_equal @@parsed_unit_string['node']['cpu']['usageNanoCores'] / 1_000_000.0, @@hash_map_test['kube.node.cpu.usage_rate'][2]['value'] end test 'test_emit_memory_metrics' do From 0efec5b9b052de4203b777ad1e571fdd44ac51b7 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Tue, 25 Jan 2022 13:53:46 -0800 Subject: [PATCH 13/13] 1.1.9 (#109) --- Gemfile.lock | 12 ++++++------ VERSION | 2 +- docker/Gemfile.lock | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index aebbcab..05aa341 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-metrics (1.1.8) + fluent-plugin-kubernetes-metrics (1.1.9) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) @@ -23,10 +23,10 @@ GEM ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.14.2) + fluentd (1.14.4) bundler cool.io (>= 1.4.5, < 2.0.0) - http_parser.rb (>= 0.5.1, < 0.8.0) + http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) @@ -47,15 +47,15 @@ GEM http-form_data (2.3.0) http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) - http_parser.rb (0.7.0) + http_parser.rb (0.8.0) json (2.6.1) kubeclient (4.6.0) http (>= 3.0, < 5.0) recursive-open-struct (~> 1.0, >= 1.0.4) rest-client (~> 2.0) - mime-types (3.3.1) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0901) + mime-types-data (3.2022.0105) msgpack (1.4.2) multi_json (1.14.1) netrc (0.11.0) diff --git a/VERSION b/VERSION index db15278..a5e4282 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.8 \ No newline at end of file +1.1.9 \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 05491a5..c9d688e 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-metrics (1.1.8) + fluent-plugin-kubernetes-metrics (1.1.9) fluentd (>= 1.9.1) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) @@ -151,7 +151,6 @@ GEM httpclient (>= 2.4) webrick (1.7.0) yajl-ruby (1.4.1) - zeitwerk (2.5.1) PLATFORMS ruby