Update fluent-plugin-google-cloud to work with Ruby 3. (#504)

fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |gem|
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.12.12'
+  gem.version       = '0.13.0'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.6')
@@ -23,28 +23,24 @@ Gem::Specification.new do |gem|
   # also the fluentd version in
   # https://github.com/GoogleCloudPlatform/google-fluentd/blob/master/config/software/fluentd.rb.
   gem.add_runtime_dependency 'fluentd', '1.13.3'
-  gem.add_runtime_dependency 'google-api-client', '0.44.2'
-  gem.add_runtime_dependency 'googleapis-common-protos', '1.3.10'
-  gem.add_runtime_dependency 'googleauth', '0.9.0'
-  gem.add_runtime_dependency 'google-cloud-logging', '1.6.6'
-  gem.add_runtime_dependency 'google-protobuf', '3.20.0'
-  gem.add_runtime_dependency 'grpc', '1.45.0'
+  gem.add_runtime_dependency 'google-api-client', '0.53.0'
+  gem.add_runtime_dependency 'googleapis-common-protos', '1.4.0'
+  gem.add_runtime_dependency 'googleauth', '1.3.0'
+  gem.add_runtime_dependency 'google-cloud-logging', '2.3.2'
+  gem.add_runtime_dependency 'google-cloud-monitoring-v3', '0.10.0'
+  gem.add_runtime_dependency 'google-protobuf', '3.22.1'
+  gem.add_runtime_dependency 'grpc', '1.52.0'
   gem.add_runtime_dependency 'json', '2.6.3'
   gem.add_runtime_dependency 'opencensus', '0.5.0'
   gem.add_runtime_dependency 'opencensus-stackdriver', '0.4.1'
 
   gem.add_development_dependency 'mocha', '1.9.0'
   # Keep this the same as in
   # https://github.com/fluent/fluent-plugin-prometheus/blob/master/fluent-plugin-prometheus.gemspec
-  gem.add_development_dependency 'prometheus-client', '< 0.10'
-  # TODO(qingling128): Upgrade rake to 11.0+ after the following issues are
-  # fixed because rake (11.0+) requires ALL variables to be explicitly
-  # initialized.
-  # https://github.com/googleapis/google-auth-library-ruby/issues/227
-  # https://github.com/farcaller/rly/issues/2
   gem.add_development_dependency 'coveralls', '0.8.23'
-  gem.add_development_dependency 'rake', '10.5.0'
+  gem.add_development_dependency 'prometheus-client', '< 0.10'
+  gem.add_development_dependency 'rake', '13.0.6'
   gem.add_development_dependency 'rubocop', '1.48.1'
   gem.add_development_dependency 'test-unit', '3.3.3'
-  gem.add_development_dependency 'webmock', '3.6.2'
+  gem.add_development_dependency 'webmock', '3.17.1'
 end

lib/fluent/plugin/filter_add_insert_ids.rb

@@ -51,9 +51,6 @@ module ConfigConstants
     desc 'The field name for insertIds in the log record.'
     config_param :insert_id_key, :string, default: DEFAULT_INSERT_ID_KEY
 
-    # Expose attr_readers for testing.
-    attr_reader :insert_id_key
-
     def start
       super
       @log = $log # rubocop:disable Style/GlobalVars

lib/fluent/plugin/out_google_cloud.rb

@@ -20,9 +20,9 @@
 require 'time'
 require 'yaml'
 require 'google/apis'
+require 'google/cloud/errors'
 require 'google/apis/logging_v2'
 require 'google/cloud/logging/v2'
-require 'google/gax'
 require 'google/logging/v2/logging_pb'
 require 'google/logging/v2/logging_services_pb'
 require 'google/logging/v2/log_entry_pb'
@@ -149,7 +149,7 @@ module InternalConstants
             %w[userAgent user_agent parse_string]
           ],
           # The grpc version class name.
-          'Google::Logging::Type::HttpRequest',
+          'Google::Cloud::Logging::Type::HttpRequest',
           # The non-grpc version class name.
           'Google::Apis::LoggingV2::HttpRequest'
         ],
@@ -161,7 +161,7 @@ module InternalConstants
             %w[first first parse_bool],
             %w[last last parse_bool]
           ],
-          'Google::Logging::V2::LogEntryOperation',
+          'Google::Cloud::Logging::V2::LogEntryOperation',
           'Google::Apis::LoggingV2::LogEntryOperation'
         ],
         'source_location' => [
@@ -171,7 +171,7 @@ module InternalConstants
             %w[function function parse_string],
             %w[line line parse_int]
           ],
-          'Google::Logging::V2::LogEntrySourceLocation',
+          'Google::Cloud::Logging::V2::LogEntrySourceLocation',
           'Google::Apis::LoggingV2::LogEntrySourceLocation'
         ]
       }.freeze
@@ -431,7 +431,7 @@ module InternalConstants
 
     # Expose attr_readers to make testing of metadata more direct than only
     # testing it indirectly through metadata sent with logs.
-    attr_reader :project_id, :zone, :vm_id, :resource, :common_labels, :monitoring_resource
+    attr_reader :resource, :common_labels, :monitoring_resource
 
     def initialize
       super
@@ -784,7 +784,7 @@ def write(chunk)
 
       if @split_logs_by_tag
         requests_to_send.each do |request|
-          @write_request.call(request)
+          @write_request.call(**request)
         end
       else
         # Combine all requests into one. The request level "log_name" will be
@@ -846,7 +846,7 @@ def construct_log_entry_in_grpc_format(labels,
                                            severity,
                                            ts_secs,
                                            ts_nanos)
-      entry = Google::Logging::V2::LogEntry.new(
+      entry = Google::Cloud::Logging::V2::LogEntry.new(
         labels: labels,
         resource: Google::Api::MonitoredResource.new(
           type: resource.type,
@@ -893,7 +893,7 @@ def write_request_via_grpc(entries:,
       client = api_client
       entries_count = entries.length
       client.write_log_entries(
-        entries,
+        entries: entries,
         log_name: log_name,
         # Leave resource nil if it's nil.
         resource: if resource
@@ -916,7 +916,7 @@ def write_request_via_grpc(entries:,
         @successful_call = true
         @log.info 'Successfully sent gRPC to Stackdriver Logging API.'
       end
-    rescue Google::Gax::GaxError => e
+    rescue Google::Cloud::Error => e
       # GRPC::BadStatus is wrapped in error.cause.
       error = e.cause
 
@@ -1007,7 +1007,7 @@ def write_request_via_grpc(entries:,
                    error: error.to_s, error_code: error_code.to_s
       end
 
-    # Got an unexpected error (not Google::Gax::GaxError) from the
+    # Got an unexpected error (not Google::Cloud::Error) from the
     # google-cloud-logging lib.
     rescue StandardError => e
       increment_failed_requests_count(GRPC::Core::StatusCodes::UNKNOWN)
@@ -1470,20 +1470,20 @@ def set_log_entry_fields(record, entry)
 
           extracted_subfields = subfields.each_with_object({}) \
             do |(original_key, destination_key, cast_fn), extracted_fields|
-            value = fields.delete(original_key)
-            next if value.nil?
-
-            begin
-              casted_value = send(cast_fn, value)
-            rescue TypeError
-              @log.error "Failed to #{cast_fn} for #{field_name}." \
-                         "#{original_key} with value #{value.inspect}.", err
-              next
-            end
-            next if casted_value.nil?
+              value = fields.delete(original_key)
+              next if value.nil?
+
+              begin
+                casted_value = send(cast_fn, value)
+              rescue TypeError
+                @log.error "Failed to #{cast_fn} for #{field_name}." \
+                           "#{original_key} with value #{value.inspect}.", err
+                next
+              end
+              next if casted_value.nil?
 
-            extracted_fields[destination_key] = casted_value
-          end
+              extracted_fields[destination_key] = casted_value
+            end
 
           next unless extracted_subfields
 
@@ -1599,30 +1599,30 @@ def parse_severity(severity_str)
     end
 
     GRPC_SEVERITY_MAPPING = {
-      'DEFAULT' => Google::Logging::Type::LogSeverity::DEFAULT,
-      'DEBUG' => Google::Logging::Type::LogSeverity::DEBUG,
-      'INFO' => Google::Logging::Type::LogSeverity::INFO,
-      'NOTICE' => Google::Logging::Type::LogSeverity::NOTICE,
-      'WARNING' => Google::Logging::Type::LogSeverity::WARNING,
-      'ERROR' => Google::Logging::Type::LogSeverity::ERROR,
-      'CRITICAL' => Google::Logging::Type::LogSeverity::CRITICAL,
-      'ALERT' => Google::Logging::Type::LogSeverity::ALERT,
-      'EMERGENCY' => Google::Logging::Type::LogSeverity::EMERGENCY,
-      0 => Google::Logging::Type::LogSeverity::DEFAULT,
-      100 => Google::Logging::Type::LogSeverity::DEBUG,
-      200 => Google::Logging::Type::LogSeverity::INFO,
-      300 => Google::Logging::Type::LogSeverity::NOTICE,
-      400 => Google::Logging::Type::LogSeverity::WARNING,
-      500 => Google::Logging::Type::LogSeverity::ERROR,
-      600 => Google::Logging::Type::LogSeverity::CRITICAL,
-      700 => Google::Logging::Type::LogSeverity::ALERT,
-      800 => Google::Logging::Type::LogSeverity::EMERGENCY
+      'DEFAULT' => Google::Cloud::Logging::Type::LogSeverity::DEFAULT,
+      'DEBUG' => Google::Cloud::Logging::Type::LogSeverity::DEBUG,
+      'INFO' => Google::Cloud::Logging::Type::LogSeverity::INFO,
+      'NOTICE' => Google::Cloud::Logging::Type::LogSeverity::NOTICE,
+      'WARNING' => Google::Cloud::Logging::Type::LogSeverity::WARNING,
+      'ERROR' => Google::Cloud::Logging::Type::LogSeverity::ERROR,
+      'CRITICAL' => Google::Cloud::Logging::Type::LogSeverity::CRITICAL,
+      'ALERT' => Google::Cloud::Logging::Type::LogSeverity::ALERT,
+      'EMERGENCY' => Google::Cloud::Logging::Type::LogSeverity::EMERGENCY,
+      0 => Google::Cloud::Logging::Type::LogSeverity::DEFAULT,
+      100 => Google::Cloud::Logging::Type::LogSeverity::DEBUG,
+      200 => Google::Cloud::Logging::Type::LogSeverity::INFO,
+      300 => Google::Cloud::Logging::Type::LogSeverity::NOTICE,
+      400 => Google::Cloud::Logging::Type::LogSeverity::WARNING,
+      500 => Google::Cloud::Logging::Type::LogSeverity::ERROR,
+      600 => Google::Cloud::Logging::Type::LogSeverity::CRITICAL,
+      700 => Google::Cloud::Logging::Type::LogSeverity::ALERT,
+      800 => Google::Cloud::Logging::Type::LogSeverity::EMERGENCY
     }.freeze
 
     def grpc_severity(severity)
       # TODO: find out why this doesn't work.
       # if severity.is_a? String
-      #   return Google::Logging::Type::LogSeverity.resolve(severity)
+      #   return Google::Cloud::Logging::Type::LogSeverity.resolve(severity)
       # end
       return GRPC_SEVERITY_MAPPING[severity] if GRPC_SEVERITY_MAPPING.key?(severity)
 
@@ -1700,9 +1700,9 @@ def delete_and_extract_labels(hash, label_map)
 
       label_map.each_with_object({}) \
         do |(original_label, new_label), extracted_labels|
-        value = hash.delete(original_label)
-        extracted_labels[new_label] = convert_to_utf8(value.to_s) if value
-      end
+          value = hash.delete(original_label)
+          extracted_labels[new_label] = convert_to_utf8(value.to_s) if value
+        end
     end
 
     def value_from_ruby(value)
@@ -1834,11 +1834,11 @@ def init_api_client
           "#{Google::Apis::OS_VERSION}"
         channel_args = { 'grpc.primary_user_agent' => user_agent }
                        .merge!(compression_channel_args)
-        @client = Google::Cloud::Logging::V2::LoggingServiceV2Client.new(
-          credentials: GRPC::Core::Channel.new(
+        @client = Google::Cloud::Logging::V2::LoggingService::Client.new do |config|
+          config.credentials = GRPC::Core::Channel.new(
             "#{host}#{port}", channel_args, creds
           )
-        )
+        end
       else
         # TODO: Use a non-default ClientOptions object.
         Google::Apis::ClientOptions.default.application_name = PLUGIN_NAME
@@ -2004,11 +2004,10 @@ def construct_error_details_map(error)
     # are a list of indexes of log entries that failed due to this error.
     #
     # A sample error looks like:
-    # <Google::Gax::RetryError:
-    #   message: 'GaxError Exception occurred in retry method that was not class
-    #             ified as transient, caused by 7:User not authorized.',
+    # <Google::Cloud::PermissionDeniedError:
+    #   message: 'User not authorized.',
     #   details: [
-    #     <Google::Logging::V2::WriteLogEntriesPartialErrors:
+    #     <Google::Cloud::Logging::V2::WriteLogEntriesPartialErrors:
     #       log_entry_errors: {
     #         0 => <Google::Rpc::Status:
     #                code: 7,
@@ -2038,13 +2037,14 @@ def construct_error_details_map(error)
     #   [3, 'Log name contains illegal character :']: [1, 3]
     # }
     def construct_error_details_map_grpc(gax_error)
+      @log.error "construct_error_details_map_grpc: #{gax_error}"
       error_details_map = Hash.new { |h, k| h[k] = [] }
       error_details = ensure_array(gax_error.status_details)
       raise JSON::ParserError, 'The error details are empty.' if
         error_details.empty?
       raise JSON::ParserError, 'No partial error info in error details.' unless
         error_details[0].is_a?(
-          Google::Logging::V2::WriteLogEntriesPartialErrors
+          Google::Cloud::Logging::V2::WriteLogEntriesPartialErrors
         )
 
       log_entry_errors = ensure_hash(error_details[0].log_entry_errors)

lib/fluent/plugin/statusz.rb

@@ -19,14 +19,16 @@
 module Statusz
   module_function
 
-  # NOTE: The plugin parameter is referenced in STATUSZ_TMPL.
   def response(plugin)
     uptime = Time.now - SERVER_START
     uptime_str = format('%<hours>d hr %<minutes>02d min %<seconds>02d sec',
                         hours: uptime / 3600,
                         minutes: (uptime / 60) % 60,
                         seconds: uptime % 60)
-    ERB.new(STATUSZ_TMPL).result(binding)
+    ERB.new(STATUSZ_TMPL).result_with_hash(
+      plugin: plugin,
+      uptime_str: uptime_str
+    )
   end
 end
 

test/plugin/base_test.rb

@@ -29,8 +29,10 @@
 
 module Monitoring
   # Prevent OpenCensus from writing to the network.
-  class OpenCensusMonitoringRegistry
-    def export
+  OpenCensusMonitoringRegistry.class_eval do
+    # Suppress redefine warning (https://bugs.ruby-lang.org/issues/17055).
+    alias_method :export, :export
+    define_method(:export) do
       nil
     end
   end
@@ -55,6 +57,7 @@ def setup
     registry.unregister(:stackdriver_retried_entries_count)
 
     setup_auth_stubs('https://www.googleapis.com/oauth2/v4/token')
+    setup_auth_stubs('https://oauth2.googleapis.com/token')
     @logs_sent = []
   end
 

test/plugin/constants.rb

@@ -12,20 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'google/rpc/error_details_pb'
+
 # Add some helper methods to standard classes.
-module Google
-  module Protobuf
-    Any.class_eval do
-      # TODO(igorpeshansky): Remove this once
-      # https://github.com/google/protobuf/pull/4719 gets released.
-      def self.pack(msg, type_url_prefix = 'type.googleapis.com/')
-        any = Google::Protobuf::Any.new
-        any.pack(msg, type_url_prefix)
-        any
-      end
-    end
-  end
-end
 String.class_eval do
   def inspect_octal
     specials = {
@@ -1018,7 +1007,7 @@ module Constants
   }.freeze
 
   PARTIAL_SUCCESS_GRPC_METADATA = begin
-    partial_errors = Google::Logging::V2::WriteLogEntriesPartialErrors.new(
+    partial_errors = Google::Cloud::Logging::V2::WriteLogEntriesPartialErrors.new(
       log_entry_errors: {
         0 => Google::Rpc::Status.new(
           code: GRPC::Core::StatusCodes::PERMISSION_DENIED,

test/plugin/test_out_google_cloud.rb

@@ -198,10 +198,10 @@ def test_parse_severity
     # known severities should translate to themselves, regardless of case
     %w[DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY].each \
       do |severity|
-      assert_equal(severity, test_obj.parse_severity(severity))
-      assert_equal(severity, test_obj.parse_severity(severity.downcase))
-      assert_equal(severity, test_obj.parse_severity(severity.capitalize))
-    end
+        assert_equal(severity, test_obj.parse_severity(severity))
+        assert_equal(severity, test_obj.parse_severity(severity.downcase))
+        assert_equal(severity, test_obj.parse_severity(severity.capitalize))
+      end
 
     # numeric levels
     assert_equal(0, test_obj.parse_severity('0'))