diff --git a/lib/fluent/config/types.rb b/lib/fluent/config/types.rb index 66ef37ad39..98e7d0d9d8 100644 --- a/lib/fluent/config/types.rb +++ b/lib/fluent/config/types.rb @@ -20,6 +20,10 @@ module Fluent module Config + def self.reformatted_value(type, val, opts = {}, name = nil) + REFORMAT_VALUE.call(type, val, opts, name) + end + def self.size_value(str, opts = {}, name = nil) return nil if str.nil? @@ -104,6 +108,16 @@ def self.string_value(val, opts = {}, name = nil) Config.string_value(val, opts, name) } + def self.symbol_value(val, opts = {}, name = nil) + return nil if val.nil? || val.empty? + + val.delete_prefix(":").to_sym + end + + SYMBOL_TYPE = Proc.new { |val, opts = {}, name = nil| + Config.symbol_value(val, opts, name) + } + def self.enum_value(val, opts = {}, name = nil) return nil if val.nil? @@ -176,6 +190,7 @@ def self.enum_value(val, opts = {}, name = nil) when :bool then Config.bool_value(value, opts, name) when :time then Config.time_value(value, opts, name) when :regexp then Config.regexp_value(value, opts, name) + when :symbol then Config.symbol_value(value, opts, name) else raise "unknown type in REFORMAT: #{type}" end diff --git a/lib/fluent/env.rb b/lib/fluent/env.rb index 2b0bf5c8d7..5ddf505bc2 100644 --- a/lib/fluent/env.rb +++ b/lib/fluent/env.rb @@ -15,13 +15,14 @@ # require 'serverengine/utils' +require 'fluent/oj_options' module Fluent DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf' DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin' DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock' DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent' - DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true} + DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env DEFAULT_DIR_PERMISSION = 0755 DEFAULT_FILE_PERMISSION = 0644 diff --git a/lib/fluent/oj_options.rb b/lib/fluent/oj_options.rb new file mode 100644 index 0000000000..f1c274c119 --- /dev/null +++ b/lib/fluent/oj_options.rb @@ -0,0 +1,62 @@ +require 'fluent/config/types' + +module Fluent + class OjOptions + OPTIONS = { + 'bigdecimal_load': :symbol, + 'max_nesting': :integer, + 'mode': :symbol, + 'use_to_json': :bool + } + + ALLOWED_VALUES = { + 'bigdecimal_load': %i[bigdecimal float auto], + 'mode': %i[strict null compat json rails object custom] + } + + DEFAULTS = { + 'bigdecimal_load': :float, + 'mode': :compat, + 'use_to_json': true + } + + @@available = false + + def self.available? + @@available + end + + def self.load_env + options = self.get_options + begin + require 'oj' + Oj.default_options = options + @@available = true + rescue LoadError + @@available = false + end + options + end + + private + + def self.get_options + options = {} + DEFAULTS.each { |key, value| options[key] = value } + + OPTIONS.each do |key, type| + env_value = ENV["FLUENT_OJ_OPTION_#{key.upcase}"] + next if env_value.nil? + + cast_value = Fluent::Config.reformatted_value(OPTIONS[key], env_value, { strict: true }) + next if cast_value.nil? + + next if ALLOWED_VALUES[key] && !ALLOWED_VALUES[key].include?(cast_value) + + options[key.to_sym] = cast_value + end + + options + end + end +end diff --git a/lib/fluent/plugin/formatter_json.rb b/lib/fluent/plugin/formatter_json.rb index 180d23eeca..8d850f28fb 100644 --- a/lib/fluent/plugin/formatter_json.rb +++ b/lib/fluent/plugin/formatter_json.rb @@ -15,7 +15,7 @@ # require 'fluent/plugin/formatter' -require 'fluent/env' +require 'fluent/oj_options' module Fluent module Plugin @@ -30,12 +30,14 @@ class JSONFormatter < Formatter def configure(conf) super - begin - raise LoadError unless @json_parser == 'oj' - require 'oj' - Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS - @dump_proc = Oj.method(:dump) - rescue LoadError + if @json_parser == 'oj' + if Fluent::OjOptions.available? + @dump_proc = Oj.method(:dump) + else + log.info "Oj isn't installed, fallback to Yajl as json parser" + @dump_proc = Yajl.method(:dump) + end + else @dump_proc = Yajl.method(:dump) end diff --git a/lib/fluent/plugin/parser_json.rb b/lib/fluent/plugin/parser_json.rb index ea6fa657a3..840f6a963c 100644 --- a/lib/fluent/plugin/parser_json.rb +++ b/lib/fluent/plugin/parser_json.rb @@ -15,8 +15,8 @@ # require 'fluent/plugin/parser' -require 'fluent/env' require 'fluent/time' +require 'fluent/oj_options' require 'yajl' require 'json' @@ -50,8 +50,7 @@ def configure(conf) def configure_json_parser(name) case name when :oj - require 'oj' - Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS + raise LoadError unless Fluent::OjOptions.available? [Oj.method(:load), Oj::ParseError] when :json then [JSON.method(:load), JSON::ParserError] when :yajl then [Yajl.method(:load), Yajl::ParseError] diff --git a/test/config/test_types.rb b/test/config/test_types.rb index 5217204030..9d7b9825a9 100644 --- a/test/config/test_types.rb +++ b/test/config/test_types.rb @@ -180,6 +180,13 @@ class TestConfigTypes < ::Test::Unit::TestCase assert_equal Encoding::UTF_8, actual.encoding end + data('starts_with_semicolon' => [:conor, ':conor'], + 'simple_string' => [:conor, 'conor'], + 'empty_string' => [nil, '']) + test 'symbol' do |(expected, val)| + assert_equal Config::SYMBOL_TYPE.call(val, {}), expected + end + data("val" => [:val, 'val'], "v" => [:v, 'v'], "value" => [:value, 'value']) diff --git a/test/test_event_time.rb b/test/test_event_time.rb index 581a276e57..6ce82f5cc8 100644 --- a/test/test_event_time.rb +++ b/test/test_event_time.rb @@ -64,8 +64,8 @@ class EventTimeTest < Test::Unit::TestCase test 'Oj.dump' do time = Fluent::EventTime.new(100) - require 'fluent/env' - Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS + require 'fluent/oj_options' + Fluent::OjOptions.load_env assert_equal('{"time":100}', Oj.dump({'time' => time})) assert_equal('["tag",100,{"key":"value"}]', Oj.dump(["tag", time, {"key" => "value"}], mode: :compat)) end diff --git a/test/test_oj_options.rb b/test/test_oj_options.rb new file mode 100644 index 0000000000..cf91612d0c --- /dev/null +++ b/test/test_oj_options.rb @@ -0,0 +1,55 @@ +require_relative 'helper' +require 'fluent/test' +require 'fluent/oj_options' + +class OjOptionsTest < ::Test::Unit::TestCase + begin + require 'oj' + @@oj_is_avaibale = true + rescue LoadError + @@oj_is_avaibale = false + end + + setup do + @orig_env = {} + ENV.each do |key, value| + @orig_env[key] = value if key.start_with?("FLUENT_OJ_OPTION_") + end + end + + teardown do + ENV.delete_if { |key| key.start_with?("FLUENT_OJ_OPTION_") } + @orig_env.each { |key, value| ENV[key] = value } + end + + test "available?" do + assert_equal(@@oj_is_avaibale, Fluent::OjOptions.available?) + end + + sub_test_case "set by environment variable" do + test "when no env vars set, returns default options" do + ENV.delete_if { |key| key.start_with?("FLUENT_OJ_OPTION_") } + defaults = Fluent::OjOptions::DEFAULTS + assert_equal(defaults, Fluent::OjOptions.load_env) + assert_equal(defaults, Oj.default_options.slice(*defaults.keys)) if @@oj_is_avaibale + end + + test "valid env var passed with valid value, default is overridden" do + ENV["FLUENT_OJ_OPTION_BIGDECIMAL_LOAD"] = ":bigdecimal" + assert_equal(:bigdecimal, Fluent::OjOptions.load_env[:bigdecimal_load]) + assert_equal(:bigdecimal, Oj.default_options[:bigdecimal_load]) if @@oj_is_avaibale + end + + test "valid env var passed with invalid value, default is not overriden" do + ENV["FLUENT_OJ_OPTION_BIGDECIMAL_LOAD"] = ":conor" + assert_equal(:float, Fluent::OjOptions.load_env[:bigdecimal_load]) + assert_equal(:float, Oj.default_options[:bigdecimal_load]) if @@oj_is_avaibale + end + + test "invalid env var passed, nothing done with it" do + ENV["FLUENT_OJ_OPTION_CONOR"] = ":conor" + assert_equal(nil, Fluent::OjOptions.load_env[:conor]) + assert_equal(nil, Oj.default_options[:conor]) if @@oj_is_avaibale + end + end +end