diff --git a/base-image/Dockerfile b/base-image/Dockerfile index fa06ba2..5731507 100644 --- a/base-image/Dockerfile +++ b/base-image/Dockerfile @@ -2,52 +2,54 @@ # SPDX-License-Identifier: BSD-2-Clause # Similar to https://github.com/drecom/docker-centos-ruby/blob/2.6.5-slim/Dockerfile +FROM photon:4.0 + ARG RVM_PATH=/usr/local/rvm ARG RUBY_VERSION=ruby-3.1.4 ARG RUBY_PATH=/usr/local/rvm/rubies/$RUBY_VERSION ARG RUBYOPT='-W:no-deprecated -W:no-experimental' -FROM photon:3.0 AS rubybuild -ARG RVM_PATH -ARG RUBY_PATH -ARG RUBY_VERSION -ARG RUBYOPT +ENV PATH $RUBY_PATH/bin:$PATH +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 +ENV BUILDDEPS="\ + gmp-devel \ + libffi-devel \ + bzip2 \ + shadow \ + wget \ + which \ + vim \ + git \ + less \ + tar \ + gzip \ + sed \ + gcc \ + build-essential \ + zlib-devel \ + libedit \ + libedit-devel \ + gdbm \ + gdbm-devel \ + openssl-devel" + RUN tdnf clean all && \ tdnf upgrade -y && \ tdnf erase -y toybox && \ tdnf install -y \ findutils \ procps-ng \ - bzip2 \ - shadow \ - wget \ - which \ - vim \ - less \ - tar \ - gzip \ util-linux \ - sed \ - gcc \ - build-essential \ - zlib-devel \ - libedit \ - libedit-devel \ - gdbm \ - gdbm-devel \ - openssl-devel \ systemd \ net-tools \ - git \ gpg && \ + tdnf install -y $BUILDDEPS && \ tdnf clean all -# Copy Gemfile.lock to pin versions further: -COPY basegems/Gemfile Gemfile -COPY basegems/Gemfile.lock Gemfile.lock - SHELL [ "/bin/bash", "-l", "-c" ] +COPY failsafe.conf entrypoint.sh Gemfile Gemfile.lock /fluentd/ + # Install the gems with bundler is better practice: RUN curl -sSL https://rvm.io/mpapis.asc | gpg --import \ && curl -sSL https://rvm.io/pkuczynski.asc | gpg --import \ @@ -57,50 +59,10 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg --import \ && rvm requirements \ && rvm install --disable-binary $RUBY_VERSION --default \ && gem update --system --no-document \ - && gem install bundler -v '>= 2.3.26' --default --no-document \ - && bundler install - -FROM photon:3.0 -ARG RUBY_PATH -ARG RUBYOPT -ENV PATH $RUBY_PATH/bin:$PATH -COPY --from=rubybuild $RUBY_PATH $RUBY_PATH -# Not sure why this is needed: see https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v1.13/debian-elasticsearch7/Dockerfile -# skip runtime bundler installation -ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 - -COPY failsafe.conf entrypoint.sh Gemfile Gemfile.lock /fluentd/ - -USER root - -ENTRYPOINT ["/fluentd/entrypoint.sh"] + && gem install bundler -v '>= 2.4.13' --default --no-document +## Install dependencies which are not up to date and we need a fork or local download RUN mkdir -p /fluentd/log /fluentd/etc /fluentd/plugins /usr/local/bundle/bin/ \ - && tdnf clean all \ - && tdnf upgrade -y \ - && tdnf erase -y toybox \ - && buildDeps="\ - gmp-devel \ - libffi-devel \ - bzip2 \ - shadow \ - wget \ - which \ - vim \ - git \ - less \ - tar \ - gzip \ - sed \ - gcc \ - build-essential \ - zlib-devel \ - libedit \ - libedit-devel \ - gdbm \ - gdbm-devel \ - openssl-devel" \ - && tdnf install -y $buildDeps util-linux systemd net-tools findutils \ && wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/docker-image/v1.15/debian-elasticsearch7/plugins/parser_kubernetes.rb -P /fluentd/plugins \ && wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/docker-image/v1.15/debian-elasticsearch7/plugins/parser_multiline_kubernetes.rb -P /fluentd/plugins \ && echo 'gem: --no-document' >> /etc/gemrc \ @@ -108,12 +70,6 @@ RUN mkdir -p /fluentd/log /fluentd/etc /fluentd/plugins /usr/local/bundle/bin/ \ && cd /fluentd \ && bundle install \ && cd /fluentd \ - && git clone https://github.com/slimm609/fluent-plugin-detect-exceptions.git fluent-plugin-detect-exceptions \ - && cd fluent-plugin-detect-exceptions \ - && gem build fluent-plugin-detect-exceptions.gemspec \ - && gem install fluent-plugin-detect-exceptions-*.gem \ - && rm -rf /fluentd/fluent-plugin-detect-exceptions \ - && cd /fluentd \ && git clone https://github.com/javiercri/fluent-plugin-google-cloud.git fluent-plugin-google-cloud \ && cd fluent-plugin-google-cloud \ && gem build fluent-plugin-google-cloud.gemspec \ @@ -124,25 +80,32 @@ RUN mkdir -p /fluentd/log /fluentd/etc /fluentd/plugins /usr/local/bundle/bin/ \ && cd fluent-plugin-loggly \ && gem build fluent-plugin-loggly.gemspec \ && gem install fluent-plugin-loggly-*.gem \ - && rm -rf /fluentd/fluent-plugin-loggly \ - && wget https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 -P /tmp \ + && rm -rf /fluentd/fluent-plugin-loggly + +COPY plugins /fluentd/plugins + +## Install jemalloc +RUN wget https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 -P /tmp \ && tar -C /tmp/ -xjvf /tmp/jemalloc-5.3.0.tar.bz2 \ && cd /tmp/jemalloc-5.3.0 \ && ./configure && make \ && mv -v lib/libjemalloc.so* /usr/lib \ - && rm -rf /tmp/jemalloc-5.3.0 /tmp/jemalloc-5.3.0.tar.bz2 \ - && cd /fluentd \ + && rm -rf /tmp/jemalloc-5.3.0 /tmp/jemalloc-5.3.0.tar.bz2 +# Make sure fluentd picks jemalloc 5.3.0 lib as default +ENV LD_PRELOAD="/usr/lib/libjemalloc.so" + +## Remove all fluentd build deps and non needit configs +RUN cd /fluentd \ && gem sources --clear-all \ && ln -s $(which fluentd) /usr/local/bundle/bin/fluentd \ - && tdnf remove -y $buildDeps \ + && tdnf remove -y $BUILDDEPS \ && tdnf clean all \ - && gem uninstall google-protobuf --version 3.17.3 --force \ && gem uninstall bundler \ && gem cleanup \ - && rm -rf $RUBY_PATH/lib/ruby/gems/2.7.0/cache $RUBY_PATH/lib/ruby/gems/2.7.0/doc/ /usr/share/doc /root/.bundle/cache - -# Make sure fluentd picks jemalloc 5.3.0 lib as default -ENV LD_PRELOAD="/usr/lib/libjemalloc.so" + && rm -rf $RUBY_PATH/lib/ruby/gems/3.1.4/cache $RUBY_PATH/lib/ruby/gems/3.1.4/doc/ /usr/share/doc /root/.bundle/cache EXPOSE 24444 5140 -COPY plugins /fluentd/plugins + +USER root + +ENTRYPOINT ["/fluentd/entrypoint.sh"] \ No newline at end of file diff --git a/base-image/Gemfile b/base-image/Gemfile index 1e0fe73..620d1b5 100644 --- a/base-image/Gemfile +++ b/base-image/Gemfile @@ -10,8 +10,8 @@ gem 'fluent-plugin-azure-loganalytics', "0.7.0" gem 'fluent-plugin-cloudwatch-logs', "0.14.3" gem 'fluent-plugin-concat', "2.5.0" gem 'fluent-plugin-datadog', "0.14.2" -gem 'fluent-plugin-elasticsearch', "5.2.4" -gem 'fluent-plugin-opensearch', "1.0.9" +gem 'fluent-plugin-elasticsearch', "5.3.0" +gem 'fluent-plugin-opensearch', "1.1.0" gem 'fluent-plugin-gelf-hs', "1.0.8" gem 'fluent-plugin-grafana-loki', "1.2.20" gem 'fluent-plugin-grok-parser', "2.6.2" @@ -51,5 +51,6 @@ gem 'gelf', "3.1.0" gem 'logfmt', "0.0.10" gem 'kubeclient', "~> 4.9.3" gem 'fluent-plugin-webhdfs', '1.5.0' +gem 'fluent-plugin-detect-exceptions', '~> 0.0.15' # webhdfs requires gssapi plugin to work gem 'gssapi', '1.3.1' diff --git a/base-image/Gemfile.lock b/base-image/Gemfile.lock index df236c6..40ab4fe 100644 --- a/base-image/Gemfile.lock +++ b/base-image/Gemfile.lock @@ -14,11 +14,11 @@ GEM amq-protocol (2.3.2) attr_required (1.0.1) aws-eventstream (1.2.0) - aws-partitions (1.668.0) + aws-partitions (1.777.0) aws-sdk-cloudwatchlogs (1.57.0) aws-sdk-core (~> 3, >= 3.165.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.168.2) + aws-sdk-core (3.174.0) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.5) @@ -66,35 +66,19 @@ GEM elasticsearch-api (= 8.5.2) elasticsearch-api (8.5.2) multi_json - excon (0.94.0) - faraday (1.10.2) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) + excon (0.100.0) + faraday (2.7.6) + faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.4) - multipart-post (~> 2) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) - faraday_middleware (1.2.0) - faraday (~> 1.0) - faraday_middleware-aws-sigv4 (0.6.1) + faraday-excon (2.1.0) + excon (>= 0.27.4) + faraday (~> 2.0) + faraday-follow_redirects (0.3.0) + faraday (>= 1, < 3) + faraday-net_http (3.0.2) + faraday_middleware-aws-sigv4 (1.0.1) aws-sigv4 (~> 1.0) - faraday (>= 1.8, < 2) + faraday (>= 2.0, < 3) ffi (1.15.5) ffi-compiler (1.0.1) ffi (>= 1.0.0) @@ -120,10 +104,13 @@ GEM fluent-plugin-datadog (0.14.2) fluentd (>= 1, < 2) net-http-persistent (~> 4.0.1) - fluent-plugin-elasticsearch (5.2.4) + fluent-plugin-detect-exceptions (0.0.15) + fluentd (>= 0.10) + fluent-plugin-elasticsearch (5.3.0) elasticsearch excon - faraday (~> 1.10) + faraday (>= 2.0.0) + faraday-excon (>= 2.0.0) fluentd (>= 0.14.22) fluent-plugin-gelf-hs (1.0.8) fluentd @@ -168,11 +155,12 @@ GEM fluent-plugin-mysqlslowquery (0.0.9) fluentd (>= 0.12.0, < 2) myslog (~> 0.0) - fluent-plugin-opensearch (1.0.9) + fluent-plugin-opensearch (1.1.0) aws-sdk-core (~> 3) excon - faraday (~> 1.10) - faraday_middleware-aws-sigv4 + faraday (>= 2.0.0) + faraday-excon (>= 2.0.0) + faraday_middleware-aws-sigv4 (~> 1.0.1) fluentd (>= 0.14.22) opensearch-ruby fluent-plugin-out-http (1.3.3) @@ -299,7 +287,6 @@ GEM bson (>= 4.3.0, < 5.0.0) msgpack (1.7.1) multi_json (1.15.0) - multipart-post (2.2.3) myslog (0.1.1) net-http-persistent (4.0.1) connection_pool (~> 2.2) @@ -315,12 +302,12 @@ GEM validate_email validate_url webfinger (>= 1.0.1) - opensearch-api (2.0.2) + opensearch-api (2.2.0) multi_json - opensearch-ruby (2.0.3) - opensearch-api (= 2.0.2) - opensearch-transport (~> 2.0.0) - opensearch-transport (2.0.1) + opensearch-ruby (2.1.0) + opensearch-api (~> 2.1) + opensearch-transport (~> 2.0) + opensearch-transport (2.1.0) faraday (>= 1.0, < 3) multi_json prometheus-client (4.0.0) @@ -376,10 +363,10 @@ GEM activemodel (>= 3.0.0) public_suffix vertica (1.0.3) - webfinger (2.0.0) + webfinger (2.1.2) activesupport - faraday (~> 1.7) - faraday_middleware (~> 1.1) + faraday (~> 2.0) + faraday-follow_redirects webhdfs (0.10.2) addressable webrick (1.8.1) @@ -397,7 +384,8 @@ DEPENDENCIES fluent-plugin-cloudwatch-logs (= 0.14.3) fluent-plugin-concat (= 2.5.0) fluent-plugin-datadog (= 0.14.2) - fluent-plugin-elasticsearch (= 5.2.4) + fluent-plugin-detect-exceptions (~> 0.0.15) + fluent-plugin-elasticsearch (= 5.3.0) fluent-plugin-gelf-hs (= 1.0.8) fluent-plugin-grafana-loki (= 1.2.20) fluent-plugin-grok-parser (= 2.6.2) @@ -413,7 +401,7 @@ DEPENDENCIES fluent-plugin-mongo (= 1.5.0) fluent-plugin-multi-format-parser (= 1.0.0) fluent-plugin-mysqlslowquery (= 0.0.9) - fluent-plugin-opensearch (= 1.0.9) + fluent-plugin-opensearch (= 1.1.0) fluent-plugin-out-http (= 1.3.3) fluent-plugin-papertrail (= 0.2.8) fluent-plugin-prometheus (= 2.0.3) diff --git a/base-image/basegems/Gemfile b/base-image/basegems/Gemfile deleted file mode 100644 index d42fbaa..0000000 --- a/base-image/basegems/Gemfile +++ /dev/null @@ -1,30 +0,0 @@ -source "https://rubygems.org" - -gem 'async', '1.30.3' -gem 'async-http', '0.59.2' -gem 'async-io', '1.34.0' -gem 'async-pool', '0.3.12' -gem 'date', '~> 3.2.2' -gem 'json', '2.6.2' -gem 'logger', '1.5.1' -gem 'matrix', '0.4.2' -gem 'mutex_m', '0.1.1' -gem 'nio4r', '2.5.8' -gem 'ostruct', '0.5.5' -gem 'power_assert', '2.0.2' -gem 'prime', '0.1.2' -gem 'protocol-hpack', '1.4.2' -gem 'protocol-http', '0.23.12' -gem 'protocol-http1', '0.14.6' -gem 'protocol-http2', '0.14.2' -gem 'rdoc', '6.4.0' -gem 'rexml', '3.2.5' -gem 'rss', '0.2.9' -gem 'sync', '0.5.0' -gem 'thwait', '0.2.0' -gem 'unf_ext', '0.0.8.2' -gem 'xmlrpc', '0.3.2' -gem 'webrick', '1.7.0' -gem 'rake', '13.0.6' -gem 'resolv', '0.2.1' -gem 'cgi', '0.3.6' diff --git a/base-image/basegems/Gemfile.lock b/base-image/basegems/Gemfile.lock deleted file mode 100644 index ee1b88a..0000000 --- a/base-image/basegems/Gemfile.lock +++ /dev/null @@ -1,100 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - async (1.30.3) - console (~> 1.10) - nio4r (~> 2.3) - timers (~> 4.1) - async-http (0.59.2) - async (>= 1.25) - async-io (>= 1.28) - async-pool (>= 0.2) - protocol-http (~> 0.23.1) - protocol-http1 (~> 0.14.0) - protocol-http2 (~> 0.14.0) - traces (>= 0.4.0) - async-io (1.34.0) - async - async-pool (0.3.12) - async (>= 1.25) - cgi (0.3.6) - console (1.15.3) - fiber-local - date (3.2.2) - e2mmap (0.1.0) - fiber-local (1.0.0) - forwardable (1.3.2) - json (2.6.2) - logger (1.5.1) - matrix (0.4.2) - mutex_m (0.1.1) - nio4r (2.5.8) - ostruct (0.5.5) - power_assert (2.0.2) - prime (0.1.2) - forwardable - singleton - protocol-hpack (1.4.2) - protocol-http (0.23.12) - protocol-http1 (0.14.6) - protocol-http (~> 0.22) - protocol-http2 (0.14.2) - protocol-hpack (~> 1.4) - protocol-http (~> 0.18) - psych (4.0.5) - stringio - rake (13.0.6) - rdoc (6.4.0) - psych (>= 4.0.0) - resolv (0.2.1) - rexml (3.2.5) - rss (0.2.9) - rexml - singleton (0.1.1) - stringio (3.0.2) - sync (0.5.0) - thwait (0.2.0) - e2mmap - timers (4.3.4) - traces (0.7.0) - unf_ext (0.0.8.2) - webrick (1.7.0) - xmlrpc (0.3.2) - webrick - -PLATFORMS - x86_64-darwin-21 - x86_64-linux - -DEPENDENCIES - async (= 1.30.3) - async-http (= 0.59.2) - async-io (= 1.34.0) - async-pool (= 0.3.12) - cgi (= 0.3.6) - date (~> 3.2.2) - json (= 2.6.2) - logger (= 1.5.1) - matrix (= 0.4.2) - mutex_m (= 0.1.1) - nio4r (= 2.5.8) - ostruct (= 0.5.5) - power_assert (= 2.0.2) - prime (= 0.1.2) - protocol-hpack (= 1.4.2) - protocol-http (= 0.23.12) - protocol-http1 (= 0.14.6) - protocol-http2 (= 0.14.2) - rake (= 13.0.6) - rdoc (= 6.4.0) - resolv (= 0.2.1) - rexml (= 3.2.5) - rss (= 0.2.9) - sync (= 0.5.0) - thwait (= 0.2.0) - unf_ext (= 0.0.8.2) - webrick (= 1.7.0) - xmlrpc (= 0.3.2) - -BUNDLED WITH - 2.3.21