@@ -88,8 +88,8 @@ gemspec_ = ::Gem::Specification.new do |s_| s_.has_rdoc = true s_.test_files = ::FileList['tests/tc_*.rb'] s_.platform = ::Gem::Platform::RUBY - s_.add_dependency('blockenspiel', '>= 0.2.2') - s_.add_dependency('versionomy', '>= 0.1.2') + s_.add_dependency('blockenspiel', '>= 0.3.0') + s_.add_dependency('versionomy', '>= 0.2.0') end ::Rake::GemPackageTask.new(gemspec_) do |task_| task_.need_zip = false Rakefile @@ -264,13 +264,13 @@ module Sawmill # Additionally, these options are recognized: # # <tt>:encoding</tt>:: - # Specify an encoding name for file data. (Ruby 1.9 only) + # Specify an encoding for file data. (Ruby 1.9 only.) + # You may specify an encoding name or an encoding object. # If not specified, uses the default external encoding. - # BUG: does not have any effect when opening a gzipped file. # <tt>:internal_encoding</tt>:: - # Specify an encoding name to transcode to. (Ruby 1.9 only) + # Specify an encoding to transcode to. (Ruby 1.9 only.) + # You may specify an encoding name or an encoding object. # If not specified, uses the default internal encoding. - # BUG: does not have any effect when opening a gzipped file. def open_entries(globs_, opts_={}, &block_) processor_ = EntryProcessor.build(&block_) @@ -295,13 +295,13 @@ module Sawmill # Additionally, these options are recognized: # # <tt>:encoding</tt>:: - # Specify an encoding name for file data. (Ruby 1.9 only) + # Specify an encoding for file data. (Ruby 1.9 only.) + # You may specify an encoding name or an encoding object. # If not specified, uses the default external encoding. - # BUG: does not have any effect when opening a gzipped file. # <tt>:internal_encoding</tt>:: - # Specify an encoding name to transcode to. (Ruby 1.9 only) + # Specify an encoding to transcode to. (Ruby 1.9 only.) + # You may specify an encoding name or an encoding object. # If not specified, uses the default internal encoding. - # BUG: does not have any effect when opening a gzipped file. def open_records(globs_, opts_={}, &block_) processor_ = RecordProcessor.build(&block_) @@ -329,42 +329,66 @@ module Sawmill # after all files have been parsed, and the return value is returned. # Otherwise, the processor is left open and nil is returned. # <tt>:encoding</tt>:: - # Specify an encoding name for file data. (Ruby 1.9 only) + # Specify an encoding for file data. (Ruby 1.9 only.) + # You may specify an encoding name or an encoding object. # If not specified, uses the default external encoding. - # BUG: does not have any effect when opening a gzipped file. # <tt>:internal_encoding</tt>:: - # Specify an encoding name to transcode to. (Ruby 1.9 only) + # Specify an encoding to transcode to. (Ruby 1.9 only.) + # You may specify an encoding name or an encoding object. # If not specified, uses the default internal encoding. - # BUG: does not have any effect when opening a gzipped file. def open_files(globs_, processor_, opts_={}) - io_array_ = [] + finish_opt_ = opts_.delete(:finish) + encoding_ = opts_.delete(:encoding) + internal_encoding_ = opts_.delete(:internal_encoding) mode_ = 'r' - if defined?(::Encoding) && (encoding_ = opts_[:encoding]) - mode_ << ":#{encoding_}" - if (encoding_ = opts_[:internal_encoding]) - mode_ << ":#{encoding_}" + if defined?(::Encoding) + if encoding_ + encoding_ = ::Encoding.find(encoding_) unless encoding_.respond_to?(:name) + encoding_ = nil if encoding_ == ::Encoding.default_external end + if internal_encoding_ + internal_encoding_ = ::Encoding.find(internal_encoding_) unless internal_encoding_.respond_to?(:name) + internal_encoding_ = nil if internal_encoding_ == ::Encoding.default_internal + end + if encoding_ + mode_ << ":#{encoding_.name}" + elsif internal_encoding_ + mode_ << ":#{::Encoding.default_external.name}" + end + mode_ << ":#{internal_encoding_.name}" if internal_encoding_ + else + encoding_ = nil + internal_encoding_ = nil end globs_ = [globs_] unless globs_.kind_of?(::Array) + io_array_ = [] + encoding_array_ = [] + internal_encoding_array_ = [] begin globs_.each do |glob_| ::Dir.glob(glob_).each do |path_| if path_ =~ /\.gz$/ io_ = ::File.open(path_, 'rb') io_array_ << ::Zlib::GzipReader.new(io_) + encoding_array_ << encoding_ + internal_encoding_array_ << internal_encoding_ else io_array_ << ::File.open(path_, mode_) + encoding_array_ << nil + internal_encoding_array_ << nil end end end + opts_[:encoding_array] = encoding_array_ if encoding_array_.find{ |elem_| elem_ } + opts_[:internal_encoding_array] = internal_encoding_array_ if internal_encoding_array_.find{ |elem_| elem_ } MultiParser.new(io_array_, processor_, opts_).parse_all ensure io_array_.each do |io_| io_.close rescue nil end end - opts_[:finish] ? processor_.finish : nil + finish_opt_ ? processor_.finish : nil end lib/sawmill/interface.rb @@ -64,7 +64,14 @@ module Sawmill @emit_incomplete_records_at_eof = opts_.delete(:emit_incomplete_records_at_eof) @heap = Util::Heap.new{ |a_, b_| a_[1].timestamp <=> b_[1].timestamp } @queue = Util::Queue.new - io_array_.each{ |io_| _enqueue(Parser.new(io_, nil, opts_)) } + encoding_array_ = opts_.delete(:encoding_array) + internal_encoding_array_ = opts_.delete(:internal_encoding_array) + io_array_.each_with_index do |io_, index_| + opts2_ = opts_.dup + opts2_[:encoding] = encoding_array_[index_] if encoding_array_ + opts2_[:internal_encoding] = internal_encoding_array_[index_] if internal_encoding_array_ + _enqueue(Parser.new(io_, nil, opts2_)) + end @processor = nil if processor_.respond_to?(:record) && processor_.respond_to?(:extra_entry) @processor = RecordBuilder.new(processor_) lib/sawmill/multi_parser.rb @@ -78,6 +78,12 @@ module Sawmill @emit_incomplete_records_at_eof = opts_[:emit_incomplete_records_at_eof] @current_record_id = nil @parser_directives = {} + @encoding = opts_[:encoding] + @internal_encoding = opts_[:internal_encoding] + if defined?(::Encoding) + @encoding = ::Encoding.find(@encoding) if @encoding && !@encoding.kind_of?(::Encoding) + @internal_encoding = ::Encoding.find(@internal_encoding) if @internal_encoding && !@internal_encoding.kind_of?(::Encoding) + end end @@ -86,7 +92,7 @@ module Sawmill # Returns nil if EOF has been reached. def parse_one_entry - str_ = @io.gets + str_ = _get_next_line entry_ = nil if str_ match_ = LINE_REGEXP.match(str_) @@ -113,7 +119,7 @@ module Sawmill count_ = $1.length str_ = $` + "\\"*(count_/2) while count_ % 2 == 1 - str2_ = @io.gets + str2_ = _get_next_line if str2_ && str2_ =~ /(\\*)\n?$/ count_ = $1.length str_ << "\n" << $` << "\\"*(count_/2) @@ -174,6 +180,19 @@ module Sawmill end + private + + + def _get_next_line # :nodoc: + str_ = @io.gets + if str_ + str_.force_encoding(@encoding) if @encoding + str_.encode!(@internal_encoding) if @internal_encoding + end + str_ + end + + end lib/sawmill/parser.rb 8765c8681af266fffe79752d08810e918ea8d6d6 Daniel Azuma dazuma@gmail.com http://github.com/dazuma/sawmill/commit/04a0ca48df6ea0410705dc93ea9350d581bc9f62 04a0ca48df6ea0410705dc93ea9350d581bc9f62 2009-11-06T13:56:04-08:00 2009-11-06T13:56:04-08:00 Workaround to allow setting encoding on reading gzip files 8158dc76b9079c7a56bd43b94bb47ce220ff0809 Daniel Azuma dazuma@gmail.com