From 7db81ca30d90d2d1362084228e29d7b45582daa6 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Mon, 27 Jul 2020 19:54:21 -0700 Subject: [PATCH 1/2] Api fix and CI cleanup --- .circleci/build_and_push_to_dockerhub.sh | 16 ----- .circleci/build_and_push_to_github_release.sh | 11 --- .circleci/check_version_trigger_release.sh | 26 ------- .circleci/config.yml | 67 +------------------ .circleci/gem_credentials | 2 - .circleci/push_gem.sh | 8 --- .circleci/push_gems_to_rubygems.sh | 7 -- .circleci/trigger_func_test.sh | 1 + Makefile | 19 ++++-- docker/Gemfile | 2 +- docker/Gemfile.lock | 28 ++++---- lib/fluent/plugin/in_kubernetes_metrics.rb | 6 +- test/helper.rb | 4 +- 13 files changed, 38 insertions(+), 159 deletions(-) delete mode 100755 .circleci/build_and_push_to_dockerhub.sh delete mode 100755 .circleci/build_and_push_to_github_release.sh delete mode 100755 .circleci/check_version_trigger_release.sh delete mode 100644 .circleci/gem_credentials delete mode 100755 .circleci/push_gem.sh delete mode 100755 .circleci/push_gems_to_rubygems.sh diff --git a/.circleci/build_and_push_to_dockerhub.sh b/.circleci/build_and_push_to_dockerhub.sh deleted file mode 100755 index cda46d5..0000000 --- a/.circleci/build_and_push_to_dockerhub.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -e -echo "Building docker image..." -cp /tmp/pkg/fluent-plugin-kubernetes-metrics-*.gem docker -echo "Copying licenses to be included in the docker image..." -mkdir docker/licenses -cp -rp LICENSE docker/licenses/ -VERSION=`cat VERSION` -docker build --no-cache --pull --build-arg VERSION=$VERSION --no-cache -t splunk/fluent-plugin-kubernetes-metrics:ci ./docker -docker tag splunk/fluent-plugin-kubernetes-metrics:ci splunk/${DOCKERHUB_REPO_NAME}:${VERSION} -docker tag splunk/fluent-plugin-kubernetes-metrics:ci splunk/${DOCKERHUB_REPO_NAME}:latest -echo "Push docker image to splunk dockerhub..." -docker login --username=$DOCKERHUB_ACCOUNT_ID --password=$DOCKERHUB_ACCOUNT_PASS -docker push splunk/${DOCKERHUB_REPO_NAME}:${VERSION} | awk 'END{print}' -docker push splunk/${DOCKERHUB_REPO_NAME}:latest | awk 'END{print}' -echo "Docker image pushed successfully to docker-hub." \ No newline at end of file diff --git a/.circleci/build_and_push_to_github_release.sh b/.circleci/build_and_push_to_github_release.sh deleted file mode 100755 index 2845a59..0000000 --- a/.circleci/build_and_push_to_github_release.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -e -wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_amd64.tar.gz -tar -xzvf ghr_v0.12.0_linux_amd64.tar.gz -sudo chmod +x ghr_v0.12.0_linux_amd64 -sudo mv ghr_v0.12.0_linux_amd64/ghr /usr/local/bin/ghr - -VERSION=`cat VERSION` -echo "Pushing SCK release to github releases...${VERSION}" - -ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -n "${RELEASE_TITLE}" -b "${RELEASE_BODY}" -draft ${VERSION} /tmp/pkg/ \ No newline at end of file diff --git a/.circleci/check_version_trigger_release.sh b/.circleci/check_version_trigger_release.sh deleted file mode 100755 index 22f3a11..0000000 --- a/.circleci/check_version_trigger_release.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -set -e - -#!/usr/bin/env bash -LATEST_COMMIT=$(git rev-parse HEAD) -VERSION_COMMIT=$(git log -1 --format=format:%H VERSION) -if [ $VERSION_COMMIT = $LATEST_COMMIT ]; - then - if [ -s VERSION ] # Check if content is empty - then - VERSION=`cat VERSION` - echo "VERSION is changed to $VERSION" - else - echo "[ERROR] VERSION file is empty." - exit 1 - fi - git config user.email "splunk-oss-admin@splunk.com" - git config user.name "splunk-oss-admin" - git checkout develop - git pull origin develop - git checkout -b release/$VERSION origin/develop - git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/fluent-plugin-kubernetes-metrics.git release/$VERSION - git checkout master - git merge --no-edit -X theirs release/$VERSION - git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/fluent-plugin-kubernetes-metrics.git master -fi diff --git a/.circleci/config.yml b/.circleci/config.yml index d756154..f83c975 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,47 +66,6 @@ jobs: name: trigger command: | .circleci/trigger_func_test.sh develop - release: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - attach_workspace: - at: /tmp - - checkout - - setup_remote_docker: - reusable: true - - run: - name: Install dependencies - command: | - .circleci/install_dep.sh - - run: - name: Build and push docker image to dockerhub - command: | - .circleci/build_and_push_to_dockerhub.sh - - run: - name: Upload gem to Github - command: | - .circleci/build_and_push_to_github_release.sh - - run: - name: Upload gem to Ruby Gem - command: | - .circleci/push_gems_to_rubygems.sh - - check_version: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - attach_workspace: - at: /tmp - - checkout - - setup_remote_docker: - reusable: true - - run: - name: Check VERSION file for change - command: | - .circleci/check_version_trigger_release.sh workflows: version: 2 @@ -126,28 +85,4 @@ workflows: - unit_test - func_test_develop: requires: - - unit_test - - check_version: - requires: - - unit_test - filters: - branches: - only: develop - release: - jobs: - - build: - filters: - branches: - only: master - - unit_test: - requires: - - build - - func_test_master: - requires: - - unit_test - - func_test_develop: - requires: - - unit_test - - release: - requires: - - unit_test + - unit_test \ No newline at end of file diff --git a/.circleci/gem_credentials b/.circleci/gem_credentials deleted file mode 100644 index 18257d8..0000000 --- a/.circleci/gem_credentials +++ /dev/null @@ -1,2 +0,0 @@ ---- -:rubygems_api_key: __RUBYGEMS_API_KEY__ \ No newline at end of file diff --git a/.circleci/push_gem.sh b/.circleci/push_gem.sh deleted file mode 100755 index 73697e3..0000000 --- a/.circleci/push_gem.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -e -sudo apt-get update -sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 -echo "Installing aws cli..." -sudo pip install awscli > /dev/null 2>&1 -echo "Pushing metrics gem to s3..." -aws s3 cp /tmp/pkg/*.gem s3://k8s-ci-artifacts/ \ No newline at end of file diff --git a/.circleci/push_gems_to_rubygems.sh b/.circleci/push_gems_to_rubygems.sh deleted file mode 100755 index 6c82a60..0000000 --- a/.circleci/push_gems_to_rubygems.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -e -echo "Pushing metrics gem to rubygems.org..." -echo "gem `gem --version`" -cat .circleci/gem_credentials | sed -e "s/__RUBYGEMS_API_KEY__/${RUBYGEMS_API_KEY}/" > ~/.gem/credentials -chmod 0600 ~/.gem/credentials -gem push /tmp/pkg/fluent-plugin-kubernetes-metrics-*.gem \ No newline at end of file diff --git a/.circleci/trigger_func_test.sh b/.circleci/trigger_func_test.sh index 0529918..2c2e1c3 100755 --- a/.circleci/trigger_func_test.sh +++ b/.circleci/trigger_func_test.sh @@ -11,6 +11,7 @@ curl -X POST --header "Content-Type: application/json" \ -d '{"build_parameters": {"CIRCLE_JOB":"build_test", "TRIG_BRANCH":"'"$CIRCLE_BRANCH"'", "TRIG_PROJECT":"'"$CIRCLE_PROJECT_REPONAME"'", "TRIG_REPO":"'"$CIRCLE_REPOSITORY_URL"'"}}' "https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/tree/$BRANCH?circle-token=$CIRCLE_TOKEN" > build.json cat build.json BUILD_NUM=$(jq -r .build_num build.json) +sleep 10 # Wait until finish or maximum 20 minutes TIMEOUT=20 diff --git a/Makefile b/Makefile index 7859f62..1e00e3f 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,15 @@ VERSION := $(shell sh -c 'cat VERSION') -clean: - @rm -rf pkg/* docker/gem/ docker/gems/ docker/*.gem docker/licenses +clean_pkg: + @rm -rf pkg/* docker/*.gem -build: clean +clean_gems: + @rm -rf docker/gem/ docker/gems/ + +clean: clean_pkg clean_gems + @rm -rf docker/licenses + +build: clean_pkg @bundle exec rake build docker: build install-deps @@ -18,4 +24,9 @@ unit-test: install-deps: @gem install bundler @bundle update --bundler - @bundle install \ No newline at end of file + @bundle install + +unpack: build + @cp pkg/fluent-plugin-*.gem docker + @gem unpack docker/fluent-plugin-*.gem --target docker/gem + @cd docker && bundle install \ No newline at end of file diff --git a/docker/Gemfile b/docker/Gemfile index b8a322f..cec790e 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -14,5 +14,5 @@ gem 'bigdecimal', '=2.0.0' gem 'kubeclient', '=4.6.0' gem 'http_parser.rb', '=0.5.3' -gem "fluent-plugin-splunk-hec", "~> 1.2" +gem "fluent-plugin-splunk-hec", "= 1.2.3" gem 'fluent-plugin-kubernetes-metrics', path: 'gem/' \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 4bc2c0c..44a0f89 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,3 +1,12 @@ +PATH + remote: gem + specs: + fluent-plugin-kubernetes-metrics (1.1.3) + fluentd (~> 1.9.1) + kubeclient (~> 4.6.0) + multi_json (~> 1.14.1) + oj (~> 3.10.2) + GEM remote: https://rubygems.org/ specs: @@ -26,20 +35,13 @@ GEM fluent-plugin-jq (0.5.1) fluentd (>= 0.14.10, < 2) multi_json (~> 1.13) - fluent-plugin-kubernetes_metadata_filter (2.4.2) - fluentd (>= 0.14.0, < 2) - kubeclient (< 5) - lru_redux fluent-plugin-prometheus (1.7.0) fluentd (>= 0.14.20, < 2) prometheus-client (< 0.10) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.1) - activesupport (~> 5.2) - fluent-plugin-kubernetes_metadata_filter (~> 2.4.2) + fluent-plugin-splunk-hec (1.2.3) fluentd (>= 1.4) - http_parser.rb (= 0.5.3) multi_json (~> 1.13) net-http-persistent (~> 3.1) openid_connect (~> 1.1.8) @@ -77,7 +79,6 @@ GEM http (>= 3.0, < 5.0) recursive-open-struct (~> 1.0, >= 1.0.4) rest-client (~> 2.0) - lru_redux (1.1.0) mail (2.7.1) mini_mime (>= 0.1.1) mime-types (3.3.1) @@ -105,13 +106,13 @@ GEM quantile (~> 0.2.1) public_suffix (4.0.3) quantile (0.2.1) - rack (2.2.2) - rack-oauth2 (1.10.1) + rack (2.2.3) + rack-oauth2 (1.16.0) activesupport attr_required httpclient json-jwt (>= 1.11.0) - rack + rack (>= 2.1.0) rake (13.0.1) recursive-open-struct (1.1.0) rest-client (2.1.0) @@ -152,9 +153,10 @@ PLATFORMS DEPENDENCIES bigdecimal (= 2.0.0) fluent-plugin-jq (= 0.5.1) + fluent-plugin-kubernetes-metrics! fluent-plugin-prometheus (= 1.7.0) fluent-plugin-record-modifier (= 2.1.0) - fluent-plugin-splunk-hec (= 1.2.1) + fluent-plugin-splunk-hec (= 1.2.3) fluentd (= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (= 4.6.0) diff --git a/lib/fluent/plugin/in_kubernetes_metrics.rb b/lib/fluent/plugin/in_kubernetes_metrics.rb index acb5a96..5bd257b 100644 --- a/lib/fluent/plugin/in_kubernetes_metrics.rb +++ b/lib/fluent/plugin/in_kubernetes_metrics.rb @@ -202,11 +202,11 @@ def initialize_rest_client if env_host && env_port if @use_rest_client_ssl @kubelet_url = "https://#{env_host}:#{env_port}/stats/summary" - @kubelet_url_stats = "https://#{env_host}:#{env_port}/stats/" + @kubelet_url_stats = "https://#{env_host}:#{env_port}/stats" @cadvisor_url = "https://#{env_host}:#{env_port}/metrics/cadvisor" else @kubelet_url = "http://#{env_host}:#{env_port}/stats/summary" - @kubelet_url_stats = "http://#{env_host}:#{env_port}/stats/" + @kubelet_url_stats = "http://#{env_host}:#{env_port}/stats" @cadvisor_url = "http://#{env_host}:#{env_port}/metrics/cadvisor" end end @@ -275,7 +275,7 @@ def stats_proxy_api(node) @stats_api = begin @client.discover unless @client.discovered - @client.rest_client["/nodes/#{node}:#{@kubelet_port}/proxy/stats/"].tap do |endpoint| + @client.rest_client["/nodes/#{node}:#{@kubelet_port}/proxy/stats"].tap do |endpoint| log.info("Use URL #{endpoint.url} for scraping stats metrics") end end diff --git a/test/helper.rb b/test/helper.rb index dd820c8..cf4694d 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -34,7 +34,7 @@ def kubelet_summary_api_url end def kubelet_stats_api_url - 'http://generics-aws-node-name:10255/stats/' + 'http://generics-aws-node-name:10255/stats' end def kubelet_cadvisor_api_url @@ -113,7 +113,7 @@ def stub_metrics_stats def stub_metrics_proxy_stats open(File.expand_path('stats.json', __dir__)).tap do |f| - stub_request(:get, "#{k8s_url}/v1/nodes/generics-aws-node-name:10255/proxy/stats/") + stub_request(:get, "#{k8s_url}/v1/nodes/generics-aws-node-name:10255/proxy/stats") .to_return(body: f.read) end.close end From abca24071aa4115704e2a7dee80e8541ebc70c6c Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Tue, 28 Jul 2020 14:06:04 -0700 Subject: [PATCH 2/2] update json --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4979653..a5984f4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -46,7 +46,7 @@ GEM http-parser (1.2.1) ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.5.3) - json (2.1.0) + json (2.3.1) kubeclient (4.6.0) http (>= 3.0, < 5.0) recursive-open-struct (~> 1.0, >= 1.0.4)