Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Added FlatPack and SQL streamers

These streamers use their respective parsers
to automatically recordize lines.
  • Loading branch information...
commit d828af7a68f6540679ecfa4d5e4df89bae3c5b81 1 parent c942834
@dieterichlawson dieterichlawson authored
View
2  lib/wukong/streamer.rb
@@ -8,6 +8,8 @@ module Streamer
autoload :StructStreamer, 'wukong/streamer/struct_streamer'
autoload :StructRecordizer, 'wukong/streamer/struct_streamer'
autoload :InstanceStreamer, 'wukong/streamer/instance_streamer'
+ autoload :FlatPackStreamer, 'wukong/streamer/flatpack_streamer'
+ autoload :SQLStreamer, 'wukong/streamer/sql_streamer'
#
autoload :Filter, 'wukong/streamer/filter'
autoload :EncodingCleaner, 'wukong/streamer/encoding_cleaner'
View
16 lib/wukong/streamer/flatpack_streamer.rb
@@ -0,0 +1,16 @@
+require 'wukong/parser/flatpack_parser'
+
+module Wukong
+ module Streamer
+ class FlatPackStreamer < Wukong::Streamer::Base
+
+ def self.format format
+ @@parser = Wukong::Parser::FlatPack.create_parser format
+ end
+
+ def recordize line
+ @@parser.parse line
+ end
+ end
+ end
+end
View
31 lib/wukong/streamer/sql_streamer.rb
@@ -0,0 +1,31 @@
+require 'wukong/parser/sql_parser'
+
+module Wukong
+ module Streamer
+ class SQLStreamer < Wukong::Streamer::Base
+
+ def self.columns columns
+ @@sql_parser = Wukong::Parser::SQLParser.new columns
+ end
+
+ def stream
+ Log.info("Streaming on:\t%s" % [Script.input_file]) unless Script.input_file.blank?
+ before_stream
+ each_record do |line|
+ recordize(line.chomp) do |record|
+ next if record.nil?
+ process(*record) do |output_record|
+ emit output_record
+ end
+ track(record)
+ end
+ end
+ after_stream
+ end
+
+ def recordize line, &blk
+ @@sql_parser.parse line, &blk
+ end
+ end
+ end
+end
Please sign in to comment.
Something went wrong with that request. Please try again.