From 5818904252e15d5af0cff4af9dfc9c7ee5b79ceb Mon Sep 17 00:00:00 2001 From: igorpeshansky Date: Thu, 30 Mar 2023 18:39:16 -0400 Subject: [PATCH] Update fluent-plugin-google-cloud to work with Ruby 3. (#504) --- fluent-plugin-google-cloud.gemspec | 26 +++-- lib/fluent/plugin/filter_add_insert_ids.rb | 3 - lib/fluent/plugin/out_google_cloud.rb | 106 ++++++++++----------- lib/fluent/plugin/statusz.rb | 6 +- test/plugin/base_test.rb | 7 +- test/plugin/constants.rb | 17 +--- test/plugin/test_out_google_cloud.rb | 8 +- test/plugin/test_out_google_cloud_grpc.rb | 18 ++-- test/plugin/utils.rb | 3 +- 9 files changed, 91 insertions(+), 103 deletions(-) diff --git a/fluent-plugin-google-cloud.gemspec b/fluent-plugin-google-cloud.gemspec index 75ccee07..ba769ddd 100644 --- a/fluent-plugin-google-cloud.gemspec +++ b/fluent-plugin-google-cloud.gemspec @@ -10,7 +10,7 @@ Gem::Specification.new do |gem| gem.homepage = 'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud' gem.license = 'Apache-2.0' - gem.version = '0.12.12' + gem.version = '0.13.0' gem.authors = ['Stackdriver Agents Team'] gem.email = ['stackdriver-agents@google.com'] gem.required_ruby_version = Gem::Requirement.new('>= 2.6') @@ -23,12 +23,13 @@ Gem::Specification.new do |gem| # also the fluentd version in # https://github.com/GoogleCloudPlatform/google-fluentd/blob/master/config/software/fluentd.rb. gem.add_runtime_dependency 'fluentd', '1.13.3' - gem.add_runtime_dependency 'google-api-client', '0.44.2' - gem.add_runtime_dependency 'googleapis-common-protos', '1.3.10' - gem.add_runtime_dependency 'googleauth', '0.9.0' - gem.add_runtime_dependency 'google-cloud-logging', '1.6.6' - gem.add_runtime_dependency 'google-protobuf', '3.20.0' - gem.add_runtime_dependency 'grpc', '1.45.0' + gem.add_runtime_dependency 'google-api-client', '0.53.0' + gem.add_runtime_dependency 'googleapis-common-protos', '1.4.0' + gem.add_runtime_dependency 'googleauth', '1.3.0' + gem.add_runtime_dependency 'google-cloud-logging', '2.3.2' + gem.add_runtime_dependency 'google-cloud-monitoring-v3', '0.10.0' + gem.add_runtime_dependency 'google-protobuf', '3.22.1' + gem.add_runtime_dependency 'grpc', '1.52.0' gem.add_runtime_dependency 'json', '2.6.3' gem.add_runtime_dependency 'opencensus', '0.5.0' gem.add_runtime_dependency 'opencensus-stackdriver', '0.4.1' @@ -36,15 +37,10 @@ Gem::Specification.new do |gem| gem.add_development_dependency 'mocha', '1.9.0' # Keep this the same as in # https://github.com/fluent/fluent-plugin-prometheus/blob/master/fluent-plugin-prometheus.gemspec - gem.add_development_dependency 'prometheus-client', '< 0.10' - # TODO(qingling128): Upgrade rake to 11.0+ after the following issues are - # fixed because rake (11.0+) requires ALL variables to be explicitly - # initialized. - # https://github.com/googleapis/google-auth-library-ruby/issues/227 - # https://github.com/farcaller/rly/issues/2 gem.add_development_dependency 'coveralls', '0.8.23' - gem.add_development_dependency 'rake', '10.5.0' + gem.add_development_dependency 'prometheus-client', '< 0.10' + gem.add_development_dependency 'rake', '13.0.6' gem.add_development_dependency 'rubocop', '1.48.1' gem.add_development_dependency 'test-unit', '3.3.3' - gem.add_development_dependency 'webmock', '3.6.2' + gem.add_development_dependency 'webmock', '3.17.1' end diff --git a/lib/fluent/plugin/filter_add_insert_ids.rb b/lib/fluent/plugin/filter_add_insert_ids.rb index 0560b813..c7e4b2b5 100644 --- a/lib/fluent/plugin/filter_add_insert_ids.rb +++ b/lib/fluent/plugin/filter_add_insert_ids.rb @@ -51,9 +51,6 @@ module ConfigConstants desc 'The field name for insertIds in the log record.' config_param :insert_id_key, :string, default: DEFAULT_INSERT_ID_KEY - # Expose attr_readers for testing. - attr_reader :insert_id_key - def start super @log = $log # rubocop:disable Style/GlobalVars diff --git a/lib/fluent/plugin/out_google_cloud.rb b/lib/fluent/plugin/out_google_cloud.rb index b87b47ea..e9ee23f4 100644 --- a/lib/fluent/plugin/out_google_cloud.rb +++ b/lib/fluent/plugin/out_google_cloud.rb @@ -20,9 +20,9 @@ require 'time' require 'yaml' require 'google/apis' +require 'google/cloud/errors' require 'google/apis/logging_v2' require 'google/cloud/logging/v2' -require 'google/gax' require 'google/logging/v2/logging_pb' require 'google/logging/v2/logging_services_pb' require 'google/logging/v2/log_entry_pb' @@ -149,7 +149,7 @@ module InternalConstants %w[userAgent user_agent parse_string] ], # The grpc version class name. - 'Google::Logging::Type::HttpRequest', + 'Google::Cloud::Logging::Type::HttpRequest', # The non-grpc version class name. 'Google::Apis::LoggingV2::HttpRequest' ], @@ -161,7 +161,7 @@ module InternalConstants %w[first first parse_bool], %w[last last parse_bool] ], - 'Google::Logging::V2::LogEntryOperation', + 'Google::Cloud::Logging::V2::LogEntryOperation', 'Google::Apis::LoggingV2::LogEntryOperation' ], 'source_location' => [ @@ -171,7 +171,7 @@ module InternalConstants %w[function function parse_string], %w[line line parse_int] ], - 'Google::Logging::V2::LogEntrySourceLocation', + 'Google::Cloud::Logging::V2::LogEntrySourceLocation', 'Google::Apis::LoggingV2::LogEntrySourceLocation' ] }.freeze @@ -431,7 +431,7 @@ module InternalConstants # Expose attr_readers to make testing of metadata more direct than only # testing it indirectly through metadata sent with logs. - attr_reader :project_id, :zone, :vm_id, :resource, :common_labels, :monitoring_resource + attr_reader :resource, :common_labels, :monitoring_resource def initialize super @@ -784,7 +784,7 @@ def write(chunk) if @split_logs_by_tag requests_to_send.each do |request| - @write_request.call(request) + @write_request.call(**request) end else # Combine all requests into one. The request level "log_name" will be @@ -846,7 +846,7 @@ def construct_log_entry_in_grpc_format(labels, severity, ts_secs, ts_nanos) - entry = Google::Logging::V2::LogEntry.new( + entry = Google::Cloud::Logging::V2::LogEntry.new( labels: labels, resource: Google::Api::MonitoredResource.new( type: resource.type, @@ -893,7 +893,7 @@ def write_request_via_grpc(entries:, client = api_client entries_count = entries.length client.write_log_entries( - entries, + entries: entries, log_name: log_name, # Leave resource nil if it's nil. resource: if resource @@ -916,7 +916,7 @@ def write_request_via_grpc(entries:, @successful_call = true @log.info 'Successfully sent gRPC to Stackdriver Logging API.' end - rescue Google::Gax::GaxError => e + rescue Google::Cloud::Error => e # GRPC::BadStatus is wrapped in error.cause. error = e.cause @@ -1007,7 +1007,7 @@ def write_request_via_grpc(entries:, error: error.to_s, error_code: error_code.to_s end - # Got an unexpected error (not Google::Gax::GaxError) from the + # Got an unexpected error (not Google::Cloud::Error) from the # google-cloud-logging lib. rescue StandardError => e increment_failed_requests_count(GRPC::Core::StatusCodes::UNKNOWN) @@ -1470,20 +1470,20 @@ def set_log_entry_fields(record, entry) extracted_subfields = subfields.each_with_object({}) \ do |(original_key, destination_key, cast_fn), extracted_fields| - value = fields.delete(original_key) - next if value.nil? - - begin - casted_value = send(cast_fn, value) - rescue TypeError - @log.error "Failed to #{cast_fn} for #{field_name}." \ - "#{original_key} with value #{value.inspect}.", err - next - end - next if casted_value.nil? + value = fields.delete(original_key) + next if value.nil? + + begin + casted_value = send(cast_fn, value) + rescue TypeError + @log.error "Failed to #{cast_fn} for #{field_name}." \ + "#{original_key} with value #{value.inspect}.", err + next + end + next if casted_value.nil? - extracted_fields[destination_key] = casted_value - end + extracted_fields[destination_key] = casted_value + end next unless extracted_subfields @@ -1599,30 +1599,30 @@ def parse_severity(severity_str) end GRPC_SEVERITY_MAPPING = { - 'DEFAULT' => Google::Logging::Type::LogSeverity::DEFAULT, - 'DEBUG' => Google::Logging::Type::LogSeverity::DEBUG, - 'INFO' => Google::Logging::Type::LogSeverity::INFO, - 'NOTICE' => Google::Logging::Type::LogSeverity::NOTICE, - 'WARNING' => Google::Logging::Type::LogSeverity::WARNING, - 'ERROR' => Google::Logging::Type::LogSeverity::ERROR, - 'CRITICAL' => Google::Logging::Type::LogSeverity::CRITICAL, - 'ALERT' => Google::Logging::Type::LogSeverity::ALERT, - 'EMERGENCY' => Google::Logging::Type::LogSeverity::EMERGENCY, - 0 => Google::Logging::Type::LogSeverity::DEFAULT, - 100 => Google::Logging::Type::LogSeverity::DEBUG, - 200 => Google::Logging::Type::LogSeverity::INFO, - 300 => Google::Logging::Type::LogSeverity::NOTICE, - 400 => Google::Logging::Type::LogSeverity::WARNING, - 500 => Google::Logging::Type::LogSeverity::ERROR, - 600 => Google::Logging::Type::LogSeverity::CRITICAL, - 700 => Google::Logging::Type::LogSeverity::ALERT, - 800 => Google::Logging::Type::LogSeverity::EMERGENCY + 'DEFAULT' => Google::Cloud::Logging::Type::LogSeverity::DEFAULT, + 'DEBUG' => Google::Cloud::Logging::Type::LogSeverity::DEBUG, + 'INFO' => Google::Cloud::Logging::Type::LogSeverity::INFO, + 'NOTICE' => Google::Cloud::Logging::Type::LogSeverity::NOTICE, + 'WARNING' => Google::Cloud::Logging::Type::LogSeverity::WARNING, + 'ERROR' => Google::Cloud::Logging::Type::LogSeverity::ERROR, + 'CRITICAL' => Google::Cloud::Logging::Type::LogSeverity::CRITICAL, + 'ALERT' => Google::Cloud::Logging::Type::LogSeverity::ALERT, + 'EMERGENCY' => Google::Cloud::Logging::Type::LogSeverity::EMERGENCY, + 0 => Google::Cloud::Logging::Type::LogSeverity::DEFAULT, + 100 => Google::Cloud::Logging::Type::LogSeverity::DEBUG, + 200 => Google::Cloud::Logging::Type::LogSeverity::INFO, + 300 => Google::Cloud::Logging::Type::LogSeverity::NOTICE, + 400 => Google::Cloud::Logging::Type::LogSeverity::WARNING, + 500 => Google::Cloud::Logging::Type::LogSeverity::ERROR, + 600 => Google::Cloud::Logging::Type::LogSeverity::CRITICAL, + 700 => Google::Cloud::Logging::Type::LogSeverity::ALERT, + 800 => Google::Cloud::Logging::Type::LogSeverity::EMERGENCY }.freeze def grpc_severity(severity) # TODO: find out why this doesn't work. # if severity.is_a? String - # return Google::Logging::Type::LogSeverity.resolve(severity) + # return Google::Cloud::Logging::Type::LogSeverity.resolve(severity) # end return GRPC_SEVERITY_MAPPING[severity] if GRPC_SEVERITY_MAPPING.key?(severity) @@ -1700,9 +1700,9 @@ def delete_and_extract_labels(hash, label_map) label_map.each_with_object({}) \ do |(original_label, new_label), extracted_labels| - value = hash.delete(original_label) - extracted_labels[new_label] = convert_to_utf8(value.to_s) if value - end + value = hash.delete(original_label) + extracted_labels[new_label] = convert_to_utf8(value.to_s) if value + end end def value_from_ruby(value) @@ -1834,11 +1834,11 @@ def init_api_client "#{Google::Apis::OS_VERSION}" channel_args = { 'grpc.primary_user_agent' => user_agent } .merge!(compression_channel_args) - @client = Google::Cloud::Logging::V2::LoggingServiceV2Client.new( - credentials: GRPC::Core::Channel.new( + @client = Google::Cloud::Logging::V2::LoggingService::Client.new do |config| + config.credentials = GRPC::Core::Channel.new( "#{host}#{port}", channel_args, creds ) - ) + end else # TODO: Use a non-default ClientOptions object. Google::Apis::ClientOptions.default.application_name = PLUGIN_NAME @@ -2004,11 +2004,10 @@ def construct_error_details_map(error) # are a list of indexes of log entries that failed due to this error. # # A sample error looks like: - # d hr %02d min %02d sec', hours: uptime / 3600, minutes: (uptime / 60) % 60, seconds: uptime % 60) - ERB.new(STATUSZ_TMPL).result(binding) + ERB.new(STATUSZ_TMPL).result_with_hash( + plugin: plugin, + uptime_str: uptime_str + ) end end diff --git a/test/plugin/base_test.rb b/test/plugin/base_test.rb index 47bebc0e..5df3fca3 100644 --- a/test/plugin/base_test.rb +++ b/test/plugin/base_test.rb @@ -29,8 +29,10 @@ module Monitoring # Prevent OpenCensus from writing to the network. - class OpenCensusMonitoringRegistry - def export + OpenCensusMonitoringRegistry.class_eval do + # Suppress redefine warning (https://bugs.ruby-lang.org/issues/17055). + alias_method :export, :export + define_method(:export) do nil end end @@ -55,6 +57,7 @@ def setup registry.unregister(:stackdriver_retried_entries_count) setup_auth_stubs('https://www.googleapis.com/oauth2/v4/token') + setup_auth_stubs('https://oauth2.googleapis.com/token') @logs_sent = [] end diff --git a/test/plugin/constants.rb b/test/plugin/constants.rb index 494da76d..d2c5bf46 100644 --- a/test/plugin/constants.rb +++ b/test/plugin/constants.rb @@ -12,20 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +require 'google/rpc/error_details_pb' + # Add some helper methods to standard classes. -module Google - module Protobuf - Any.class_eval do - # TODO(igorpeshansky): Remove this once - # https://github.com/google/protobuf/pull/4719 gets released. - def self.pack(msg, type_url_prefix = 'type.googleapis.com/') - any = Google::Protobuf::Any.new - any.pack(msg, type_url_prefix) - any - end - end - end -end String.class_eval do def inspect_octal specials = { @@ -1018,7 +1007,7 @@ module Constants }.freeze PARTIAL_SUCCESS_GRPC_METADATA = begin - partial_errors = Google::Logging::V2::WriteLogEntriesPartialErrors.new( + partial_errors = Google::Cloud::Logging::V2::WriteLogEntriesPartialErrors.new( log_entry_errors: { 0 => Google::Rpc::Status.new( code: GRPC::Core::StatusCodes::PERMISSION_DENIED, diff --git a/test/plugin/test_out_google_cloud.rb b/test/plugin/test_out_google_cloud.rb index e0929ac0..b7e60f15 100644 --- a/test/plugin/test_out_google_cloud.rb +++ b/test/plugin/test_out_google_cloud.rb @@ -198,10 +198,10 @@ def test_parse_severity # known severities should translate to themselves, regardless of case %w[DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY].each \ do |severity| - assert_equal(severity, test_obj.parse_severity(severity)) - assert_equal(severity, test_obj.parse_severity(severity.downcase)) - assert_equal(severity, test_obj.parse_severity(severity.capitalize)) - end + assert_equal(severity, test_obj.parse_severity(severity)) + assert_equal(severity, test_obj.parse_severity(severity.downcase)) + assert_equal(severity, test_obj.parse_severity(severity.capitalize)) + end # numeric levels assert_equal(0, test_obj.parse_severity('0')) diff --git a/test/plugin/test_out_google_cloud_grpc.rb b/test/plugin/test_out_google_cloud_grpc.rb index 537cc459..1c5c8e0b 100644 --- a/test/plugin/test_out_google_cloud_grpc.rb +++ b/test/plugin/test_out_google_cloud_grpc.rb @@ -34,6 +34,8 @@ def test_user_agent # Record user agent when creating a GRPC::Core::Channel. GRPC::Core::Channel.class_eval do old_initialize = instance_method(:initialize) + # Suppress redefine warning (https://bugs.ruby-lang.org/issues/17055). + alias_method :initialize, :initialize define_method(:initialize) do |url, args, creds| user_agent = args['grpc.primary_user_agent'] old_initialize.bind(self).call(url, args, creds) @@ -270,8 +272,8 @@ def test_non_integer_timestamp private - WriteLogEntriesRequest = Google::Logging::V2::WriteLogEntriesRequest - WriteLogEntriesResponse = Google::Logging::V2::WriteLogEntriesResponse + WriteLogEntriesRequest = Google::Cloud::Logging::V2::WriteLogEntriesRequest + WriteLogEntriesResponse = Google::Cloud::Logging::V2::WriteLogEntriesResponse USE_GRPC_CONFIG = %( use_grpc true @@ -332,13 +334,13 @@ def api_client # GRPC logging mock that successfully logs the records. class GRPCLoggingMockService < - Google::Cloud::Logging::V2::LoggingServiceV2Client + Google::Cloud::Logging::V2::LoggingService::Client def initialize(requests_received) super() @requests_received = requests_received end - def write_log_entries(entries, + def write_log_entries(entries:, log_name: nil, resource: nil, labels: nil, @@ -357,7 +359,7 @@ def write_log_entries(entries, # GRPC logging mock that fails and returns server side or client side errors. class GRPCLoggingMockFailingService < - Google::Cloud::Logging::V2::LoggingServiceV2Client + Google::Cloud::Logging::V2::LoggingService::Client def initialize(error, failed_attempts) super() @error = error @@ -365,7 +367,7 @@ def initialize(error, failed_attempts) end # rubocop:disable Lint/UnusedMethodArgument - def write_log_entries(entries, + def write_log_entries(entries:, log_name: nil, resource: nil, labels: nil, @@ -374,8 +376,8 @@ def write_log_entries(entries, begin raise @error rescue StandardError - # Google::Gax::GaxError will wrap the latest thrown exception as @cause. - raise Google::Gax::GaxError, 'This test message does not matter.' + # Google::Cloud::Error will wrap the latest thrown exception as @cause. + raise Google::Cloud::Error, 'This test message does not matter.' end end # rubocop:enable Lint/UnusedMethodArgument diff --git a/test/plugin/utils.rb b/test/plugin/utils.rb index adb01330..ac4f69df 100644 --- a/test/plugin/utils.rb +++ b/test/plugin/utils.rb @@ -60,8 +60,7 @@ def setup_gce_metadata_stubs "attribute1\nattribute2\nattribute3") # Used by 'googleauth' to fetch the default service account credentials. - stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \ - 'instance/service-accounts/default/token') + stub_request(:get, %r{http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token(?:\?.*)}) .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}), status: 200, headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,