From 1c52b602ee44fc78070692c6f812a98813a31efa Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Sun, 16 Feb 2020 10:50:47 -0800 Subject: [PATCH 1/2] whitesource --- .gitignore | 2 + Gemfile.lock | 9 ++- docker/Dockerfile | 13 +--- docker/Gemfile | 15 ++++ docker/Gemfile.lock | 95 ++++++++++++++++++++++++ fluent-plugin-kubernetes-metrics.gemspec | 1 + 6 files changed, 122 insertions(+), 13 deletions(-) create mode 100644 docker/Gemfile create mode 100644 docker/Gemfile.lock diff --git a/.gitignore b/.gitignore index 2b05080..89ee2e5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ vendor pkg coverage/ .idea/ +docker/gems/ +docker/licenses/ \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index bb3ddff..1d3d742 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,6 +3,7 @@ PATH specs: fluent-plugin-kubernetes-metrics (1.1.2) fluentd (~> 1.9.1) + http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) multi_json (~> 1.14.1) oj (~> 3.10.2) @@ -12,7 +13,7 @@ GEM specs: addressable (2.6.0) public_suffix (>= 2.0.2, < 4.0) - concurrent-ruby (1.1.5) + concurrent-ruby (1.1.6) cool.io (1.6.0) crack (0.4.3) safe_yaml (~> 1.0.0) @@ -23,7 +24,7 @@ GEM ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.9.1) + fluentd (1.9.2) cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.7.0) msgpack (>= 1.3.1, < 2.0.0) @@ -45,7 +46,7 @@ GEM http-form_data (2.2.0) http-parser (1.2.1) ffi-compiler (>= 1.0, < 2.0) - http_parser.rb (0.6.0) + http_parser.rb (0.5.3) json (2.1.0) kubeclient (4.6.0) http (>= 3.0, < 5.0) @@ -54,7 +55,7 @@ GEM mime-types (3.3.1) mime-types-data (~> 3.2015) mime-types-data (3.2019.1009) - msgpack (1.3.1) + msgpack (1.3.3) multi_json (1.14.1) netrc (0.11.0) oj (3.10.2) diff --git a/docker/Dockerfile b/docker/Dockerfile index 65bffa4..b7f21c4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,15 +19,10 @@ USER root COPY *.gem /tmp/ COPY licenses /licenses -RUN gem install -N fluentd:1.9.1 \ - fluent-plugin-prometheus:1.7.0 \ - fluent-plugin-record-modifier:2.1.0 \ - fluent-plugin-splunk-hec:${VERSION} \ - oj:3.10.2 \ - multi_json:1.14.1 \ - bigdecimal:2.0.0 \ - kubeclient:4.6.0 && \ - gem install -N /tmp/*.gem +COPY Gemfile* ./ +RUN gem install bundler \ + && bundle add fluent-plugin-splunk-hec -v ${VERSION} \ + && bundle exec gem install /tmp/*.gem RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ diff --git a/docker/Gemfile b/docker/Gemfile new file mode 100644 index 0000000..d7b6d26 --- /dev/null +++ b/docker/Gemfile @@ -0,0 +1,15 @@ +source 'https://rubygems.org' + +# This is separate gemfile for building docker image that has all plugins +# for kubernetes log collection agent +# List all required gems here and install via bundler to resolve dependencies +gem "fluentd", "=1.9.1" +gem "fluent-plugin-prometheus", "=1.7.0" +gem "fluent-plugin-record-modifier", "=2.1.0" + +gem "fluent-plugin-jq", "=0.5.1" +gem "oj", "=3.10.2" +gem 'multi_json', '=1.14.1' +gem 'bigdecimal', '=2.0.0' +gem 'kubeclient', '=4.6.0' +gem 'http_parser.rb', '=0.5.3' diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock new file mode 100644 index 0000000..2d6f338 --- /dev/null +++ b/docker/Gemfile.lock @@ -0,0 +1,95 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + bigdecimal (2.0.0) + concurrent-ruby (1.1.6) + cool.io (1.6.0) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + ffi (1.12.2) + ffi-compiler (1.0.1) + ffi (>= 1.0.0) + rake + fluent-plugin-jq (0.5.1) + fluentd (>= 0.14.10, < 2) + multi_json (~> 1.13) + fluent-plugin-prometheus (1.7.0) + fluentd (>= 0.14.20, < 2) + prometheus-client (< 0.10) + fluent-plugin-record-modifier (2.1.0) + fluentd (>= 1.0, < 2) + fluentd (1.9.1) + cool.io (>= 1.4.5, < 2.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.0.4, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + yajl-ruby (~> 1.0) + http (4.3.0) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + http-parser (~> 1.2.0) + http-accept (1.7.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (2.2.0) + http-parser (1.2.1) + ffi-compiler (>= 1.0, < 2.0) + http_parser.rb (0.5.3) + kubeclient (4.6.0) + http (>= 3.0, < 5.0) + recursive-open-struct (~> 1.0, >= 1.0.4) + rest-client (~> 2.0) + mime-types (3.3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2019.1009) + msgpack (1.3.3) + multi_json (1.14.1) + netrc (0.11.0) + oj (3.10.2) + prometheus-client (0.9.0) + quantile (~> 0.2.1) + public_suffix (4.0.3) + quantile (0.2.1) + rake (13.0.1) + recursive-open-struct (1.1.0) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + serverengine (2.2.1) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.3) + tzinfo (2.0.1) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2019.3) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.6) + yajl-ruby (1.4.1) + +PLATFORMS + ruby + +DEPENDENCIES + bigdecimal (= 2.0.0) + fluent-plugin-jq (= 0.5.1) + fluent-plugin-prometheus (= 1.7.0) + fluent-plugin-record-modifier (= 2.1.0) + fluentd (= 1.9.1) + http_parser.rb (= 0.5.3) + kubeclient (= 4.6.0) + multi_json (= 1.14.1) + oj (= 3.10.2) + +BUNDLED WITH + 2.1.4 diff --git a/fluent-plugin-kubernetes-metrics.gemspec b/fluent-plugin-kubernetes-metrics.gemspec index d9d8fd5..31f9cbf 100644 --- a/fluent-plugin-kubernetes-metrics.gemspec +++ b/fluent-plugin-kubernetes-metrics.gemspec @@ -26,4 +26,5 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'kubeclient', '~> 4.6.0' spec.add_runtime_dependency 'multi_json', '~> 1.14.1' spec.add_runtime_dependency 'oj', '~> 3.10.2' + spec.add_runtime_dependency 'http_parser.rb', '= 0.5.3' end From 163ca849c8284fceae5eacadf7a45f45c0401d8e Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Fri, 21 Feb 2020 10:26:45 -0800 Subject: [PATCH 2/2] Release 1.1.3 --- .circleci/build_and_push.sh | 11 ---- .circleci/build_and_push_to_dockerhub.sh | 7 +- .circleci/check_version_trigger_release.sh | 2 +- .circleci/config.yml | 6 +- .gitignore | 4 +- Gemfile.lock | 2 +- Rakefile | 2 +- VERSION | 2 +- docker/Dockerfile | 9 +-- docker/FLUENTD_HEC_GEM_VERSION | 1 - docker/Gemfile | 2 + docker/Gemfile.lock | 74 +++++++++++++++++++++- docker/README.md | 2 +- docker/build.sh | 4 +- fluent-plugin-kubernetes-metrics.gemspec | 14 ++-- 15 files changed, 102 insertions(+), 40 deletions(-) delete mode 100755 .circleci/build_and_push.sh delete mode 100644 docker/FLUENTD_HEC_GEM_VERSION diff --git a/.circleci/build_and_push.sh b/.circleci/build_and_push.sh deleted file mode 100755 index de4006f..0000000 --- a/.circleci/build_and_push.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -e -FLUENTD_HEC_GEM_VERSION=`cat docker/FLUENTD_HEC_GEM_VERSION` -aws ecr get-login --region $AWS_REGION --no-include-email | bash -echo "Building docker image..." -cp /tmp/pkg/fluent-plugin-kubernetes-metrics-*.gem docker -docker build --build-arg VERSION=$FLUENTD_HEC_GEM_VERSION --no-cache -t splunk/fluent-plugin-kubernetes-metrics:metrics ./docker -docker tag splunk/fluent-plugin-kubernetes-metrics:metrics $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-metrics:latest -echo "Push docker image to ecr..." -docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-metrics:latest | awk 'END{print}' -echo "Docker image pushed successfully." \ No newline at end of file diff --git a/.circleci/build_and_push_to_dockerhub.sh b/.circleci/build_and_push_to_dockerhub.sh index adde523..4a9765c 100755 --- a/.circleci/build_and_push_to_dockerhub.sh +++ b/.circleci/build_and_push_to_dockerhub.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash set -e -FLUENTD_HEC_GEM_VERSION=`cat docker/FLUENTD_HEC_GEM_VERSION` echo "Building docker image..." cp /tmp/pkg/fluent-plugin-kubernetes-metrics-*.gem docker echo "Copying licenses to be included in the docker image..." -mkdir licenses -cp -rp LICENSE licenses/ +mkdir docker/licenses +cp -rp LICENSE docker/licenses/ VERSION=`cat VERSION` -docker build --build-arg VERSION=$FLUENTD_HEC_GEM_VERSION --no-cache -t splunk/fluent-plugin-kubernetes-metrics:ci ./docker +docker build --build-arg VERSION=$VERSION --no-cache -t splunk/fluent-plugin-kubernetes-metrics:ci ./docker docker tag splunk/fluent-plugin-kubernetes-metrics:ci splunk/${DOCKERHUB_REPO_NAME}:${VERSION} docker tag splunk/fluent-plugin-kubernetes-metrics:ci splunk/${DOCKERHUB_REPO_NAME}:latest echo "Push docker image to splunk dockerhub..." diff --git a/.circleci/check_version_trigger_release.sh b/.circleci/check_version_trigger_release.sh index bc08e6a..22f3a11 100755 --- a/.circleci/check_version_trigger_release.sh +++ b/.circleci/check_version_trigger_release.sh @@ -21,6 +21,6 @@ if [ $VERSION_COMMIT = $LATEST_COMMIT ]; git checkout -b release/$VERSION origin/develop git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/fluent-plugin-kubernetes-metrics.git release/$VERSION git checkout master - git merge --no-edit release/$VERSION + git merge --no-edit -X theirs release/$VERSION git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/fluent-plugin-kubernetes-metrics.git master fi diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c1bb2a..d756154 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,8 +129,7 @@ workflows: - unit_test - check_version: requires: - - func_test_master - - func_test_develop + - unit_test filters: branches: only: develop @@ -151,5 +150,4 @@ workflows: - unit_test - release: requires: - - func_test_master - - func_test_develop \ No newline at end of file + - unit_test diff --git a/.gitignore b/.gitignore index 89ee2e5..a21f250 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ pkg coverage/ .idea/ docker/gems/ -docker/licenses/ \ No newline at end of file +docker/licenses/ +/.bundle/ +docker/.bundle/ \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index 1d3d742..374e830 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-metrics (1.1.2) + fluent-plugin-kubernetes-metrics (1.1.3) fluentd (~> 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) diff --git a/Rakefile b/Rakefile index 15001f9..a1eb595 100644 --- a/Rakefile +++ b/Rakefile @@ -24,6 +24,6 @@ namespace :docker do raise 'Argument `tag` was not provided.' unless args.tag cp Dir['pkg/fluent-plugin-kubernetes-metrics-*.gem'], 'docker/' - sh "docker build --no-cache -t splunk/connect-for-kubernetes:#{args.tag} ./docker" + sh "docker build --build-arg VERSION=$(cat VERSION) --no-cache -t splunk/connect-for-kubernetes:#{args.tag} ./docker" end end diff --git a/VERSION b/VERSION index 8428158..9c1218c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.2 \ No newline at end of file +1.1.3 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index b7f21c4..811b812 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,15 +2,16 @@ FROM registry.access.redhat.com/ubi8/ruby-25 MAINTAINER DataEdge DataEdge@splunk.com +ARG VERSION + LABEL name="Splunk Connect for Kubernetes Metrics container" \ maintainer="DataEdge@splunk.com" \ vendor="Splunk Inc." \ - version="1.1.3" \ - release="1.1.3" \ + version=${VERSION} \ + release=${VERSION} \ summary="Splunk Connect for Kubernetes Metrics container" \ description="Splunk Connect for Kubernetes Metrics container" -ARG VERSION ENV VERSION=${VERSION} ENV FLUENT_USER fluent @@ -21,7 +22,7 @@ COPY licenses /licenses COPY Gemfile* ./ RUN gem install bundler \ - && bundle add fluent-plugin-splunk-hec -v ${VERSION} \ + && bundle install \ && bundle exec gem install /tmp/*.gem RUN groupadd -r $FLUENT_USER && \ diff --git a/docker/FLUENTD_HEC_GEM_VERSION b/docker/FLUENTD_HEC_GEM_VERSION deleted file mode 100644 index 1cc5f65..0000000 --- a/docker/FLUENTD_HEC_GEM_VERSION +++ /dev/null @@ -1 +0,0 @@ -1.1.0 \ No newline at end of file diff --git a/docker/Gemfile b/docker/Gemfile index d7b6d26..538f8f7 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -13,3 +13,5 @@ gem 'multi_json', '=1.14.1' gem 'bigdecimal', '=2.0.0' gem 'kubeclient', '=4.6.0' gem 'http_parser.rb', '=0.5.3' + +gem "fluent-plugin-splunk-hec", "= 1.2.1" diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 2d6f338..ba3807d 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,10 +1,21 @@ GEM remote: https://rubygems.org/ specs: + activemodel (5.2.4.1) + activesupport (= 5.2.4.1) + activesupport (5.2.4.1) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) + aes_key_wrap (1.0.1) + attr_required (1.0.1) bigdecimal (2.0.0) + bindata (2.4.5) concurrent-ruby (1.1.6) + connection_pool (2.2.2) cool.io (1.6.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) @@ -15,11 +26,24 @@ GEM fluent-plugin-jq (0.5.1) fluentd (>= 0.14.10, < 2) multi_json (~> 1.13) + fluent-plugin-kubernetes_metadata_filter (2.4.2) + fluentd (>= 0.14.0, < 2) + kubeclient (< 5) + lru_redux fluent-plugin-prometheus (1.7.0) fluentd (>= 0.14.20, < 2) prometheus-client (< 0.10) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) + fluent-plugin-splunk-hec (1.2.1) + activesupport (~> 5.2) + fluent-plugin-kubernetes_metadata_filter (~> 2.4.2) + fluentd (>= 1.4) + http_parser.rb (= 0.5.3) + multi_json (~> 1.13) + net-http-persistent (~> 3.1) + openid_connect (~> 1.1.8) + prometheus-client (< 0.10.0) fluentd (1.9.1) cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.7.0) @@ -42,21 +66,52 @@ GEM http-parser (1.2.1) ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.5.3) + httpclient (2.8.3) + i18n (1.8.2) + concurrent-ruby (~> 1.0) + json-jwt (1.11.0) + activesupport (>= 4.2) + aes_key_wrap + bindata kubeclient (4.6.0) http (>= 3.0, < 5.0) recursive-open-struct (~> 1.0, >= 1.0.4) rest-client (~> 2.0) + lru_redux (1.1.0) + mail (2.7.1) + mini_mime (>= 0.1.1) mime-types (3.3.1) mime-types-data (~> 3.2015) mime-types-data (3.2019.1009) + mini_mime (1.0.2) + minitest (5.14.0) msgpack (1.3.3) multi_json (1.14.1) + net-http-persistent (3.1.0) + connection_pool (~> 2.2) netrc (0.11.0) oj (3.10.2) + openid_connect (1.1.8) + activemodel + attr_required (>= 1.0.0) + json-jwt (>= 1.5.0) + rack-oauth2 (>= 1.6.1) + swd (>= 1.0.0) + tzinfo + validate_email + validate_url + webfinger (>= 1.0.1) prometheus-client (0.9.0) quantile (~> 0.2.1) public_suffix (4.0.3) quantile (0.2.1) + rack (2.2.2) + rack-oauth2 (1.10.1) + activesupport + attr_required + httpclient + json-jwt (>= 1.11.0) + rack rake (13.0.1) recursive-open-struct (1.1.0) rest-client (2.1.0) @@ -68,13 +123,27 @@ GEM sigdump (~> 0.2.2) sigdump (0.2.4) strptime (0.2.3) - tzinfo (2.0.1) - concurrent-ruby (~> 1.0) + swd (1.1.2) + activesupport (>= 3) + attr_required (>= 0.0.5) + httpclient (>= 2.4) + thread_safe (0.3.6) + tzinfo (1.2.6) + thread_safe (~> 0.1) tzinfo-data (1.2019.3) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext unf_ext (0.0.7.6) + validate_email (0.1.6) + activemodel (>= 3.0) + mail (>= 2.2.5) + validate_url (1.0.8) + activemodel (>= 3.0.0) + public_suffix + webfinger (1.1.0) + activesupport + httpclient (>= 2.4) yajl-ruby (1.4.1) PLATFORMS @@ -85,6 +154,7 @@ DEPENDENCIES fluent-plugin-jq (= 0.5.1) fluent-plugin-prometheus (= 1.7.0) fluent-plugin-record-modifier (= 2.1.0) + fluent-plugin-splunk-hec (= 1.2.1) fluentd (= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (= 4.6.0) diff --git a/docker/README.md b/docker/README.md index 8b6d32a..78cf954 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,4 +1,4 @@ # Docker Image for Splunk Connect for Kubernetes Fluentd with input plugin for k8s metrics # To Build -`docker build --build-arg VERSION=$(cat docker/FLUENTD_HEC_GEM_VERSION) --no-cache -t splunk/fluent-plugin-kubernetes-metrics:ci ./docker` \ No newline at end of file +`docker build --build-arg VERSION=$(cat VERSION) --no-cache -t splunk/fluent-plugin-kubernetes-metrics:ci ./docker` \ No newline at end of file diff --git a/docker/build.sh b/docker/build.sh index 8f1c18a..8ef4c2b 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -12,8 +12,8 @@ bundle exec rake build -t -v cp pkg/fluent-plugin-kubernetes-metrics-*.gem docker # Build Docker Image -FLUENTD_HEC_GEM_VERSION=`cat docker/FLUENTD_HEC_GEM_VERSION` +VERSION=`cat VERSION` echo "Copying licenses to be included in the docker image..." mkdir -p docker/licenses cp -rp LICENSE docker/licenses/ -docker build --build-arg VERSION=$FLUENTD_HEC_GEM_VERSION --no-cache -t splunk/k8s-metrics:$TAG ./docker +docker build --build-arg VERSION=$VERSION --no-cache -t splunk/k8s-metrics:$TAG ./docker diff --git a/fluent-plugin-kubernetes-metrics.gemspec b/fluent-plugin-kubernetes-metrics.gemspec index 31f9cbf..49b0de8 100644 --- a/fluent-plugin-kubernetes-metrics.gemspec +++ b/fluent-plugin-kubernetes-metrics.gemspec @@ -10,12 +10,14 @@ Gem::Specification.new do |spec| spec.description = 'A fluentd input plugin that collects node and container metrics from a kubernetes cluster.' spec.homepage = 'https://github.com/splunk/fluent-plugin-kubernetes-metrics' spec.license = 'Apache-2.0' - test_files, files = `git ls-files -z`.split("\x0").partition do |f| - f.match(%r{^(test|spec|features)/}) - end - spec.files = files - spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) } - spec.test_files = test_files + + spec.files = %w[ + metrics-information.md README.md LICENSE + fluent-plugin-kubernetes-metrics.gemspec + Gemfile Gemfile.lock + Rakefile VERSION + ] + Dir.glob('lib/**/**').reject(&File.method(:directory?)) + spec.test_files = Dir.glob('test/**/**.rb') spec.require_paths = ['lib'] spec.add_development_dependency 'bundler', '~> 2.0' spec.add_development_dependency 'rake', '~> 12.0'