Skip to content
Browse files

migrating to gorillib and gorillib models from the old wukong core ex…

…tensions
  • Loading branch information...
1 parent 6223d52 commit fb3645a676134258c8cf32d179b4881cf454793b Philip (flip) Kromer committed Aug 18, 2012
View
2 VERSION
@@ -1 +1 @@
-2.1.0
+2.1.1
View
2 lib/wukong.rb
@@ -3,8 +3,10 @@
require 'wukong/datatypes'
require 'wukong/periodic_monitor'
require 'wukong/logger'
+
autoload :BadRecord, 'wukong/bad_record'
autoload :TypedStruct, 'wukong/typed_struct'
+
module Wukong
autoload :Script, 'wukong/script'
autoload :Streamer, 'wukong/streamer'
View
9 lib/wukong/bad_record.rb
@@ -8,10 +8,11 @@
# return BadRecord.new("do_stuff_to-failed", record)
# end
#
-class BadRecord < Struct.new(
- :errors,
- :record
- )
+class BadRecord
+ include Gorillib::Model
+ field :errors, String, position: 0
+ field :record, Whatever, position: 1
+
def initialize errors='', *record_fields
super errors, record_fields
end
View
32 lib/wukong/encoding.rb
@@ -1,22 +1,22 @@
require 'htmlentities'
require 'addressable/uri'
-# Fix a bug (?) in the HTMLEntities encoder class with $KCODE='NONE'
-HTMLEntities::Encoder.class_eval do
-private
- def extended_entity_regexp
- @extended_entity_regexp ||= (
- if encoding_aware?
- regexp = '[^\u{20}-\u{7E}]'
- else
- # regexp = '[^\x20-\x7E]'
- regexp = '[\x00-\x1f]|[\xc0-\xfd][\x80-\xbf]+'
- end
- regexp += "|'" if @flavor == 'html4'
- Regexp.new(regexp)
- )
- end
-end
+# # Fix a bug (?) in the HTMLEntities encoder class with $KCODE='NONE'
+# HTMLEntities::Encoder.class_eval do
+# private
+# def extended_entity_regexp
+# @extended_entity_regexp ||= (
+# if encoding_aware?
+# regexp = '[^\u{20}-\u{7E}]'
+# else
+# # regexp = '[^\x20-\x7E]'
+# regexp = '[\x00-\x1f]|[\xc0-\xfd][\x80-\xbf]+'
+# end
+# regexp += "|'" if @flavor == 'html4'
+# Regexp.new(regexp)
+# )
+# end
+# end
module Wukong
#
View
14 lib/wukong/extensions.rb
@@ -2,15 +2,5 @@
# These pull in the minimal functionality of the extlib|activesupport family of
# gems.
#
-require 'extlib/blank'
-require 'extlib/class'
-require 'wukong/extensions/enumerable'
-require 'wukong/extensions/symbol'
-require 'wukong/extensions/hash'
-require 'wukong/extensions/hash_like'
-require 'wukong/extensions/array'
-require 'wukong/extensions/struct'
-require 'wukong/extensions/module'
-require 'wukong/extensions/string'
-require 'wukong/extensions/date_time'
-require 'wukong/extensions/emittable'
+require 'gorillib'
+require 'gorillib/model'
View
10 lib/wukong/script.rb
@@ -1,6 +1,12 @@
-require 'pathname'
-require 'wukong/extensions'
require 'configliere' ; Settings.use(:commandline, :env_var, :define)
+require 'gorillib'
+require 'gorillib/model'
+require 'gorillib/model/positional_fields'
+require 'gorillib/model/serialization'
+require 'gorillib/hash/deep_merge'
+require 'gorillib/type/extended'
+require 'gorillib/datetime/to_flat'
+#
require 'wukong'
require 'wukong/script/hadoop_command'
require 'wukong/script/local_command'
View
1 lib/wukong/streamer.rb
@@ -3,6 +3,7 @@ module Streamer
autoload :Base, 'wukong/streamer/base'
autoload :LineStreamer, 'wukong/streamer/line_streamer'
autoload :RecordStreamer, 'wukong/streamer/record_streamer'
+ autoload :ModelStreamer, 'wukong/streamer/model_streamer'
autoload :JsonStreamer, 'wukong/streamer/json_streamer'
autoload :StructStreamer, 'wukong/streamer/struct_streamer'
autoload :StructRecordizer, 'wukong/streamer/struct_streamer'
View
3 lib/wukong/streamer/base.rb
@@ -66,7 +66,8 @@ def recordize line
# yours if you override this method.
#
def emit record
- puts record.to_flat.join("\t")
+ return if record.nil?
+ puts record.to_tsv
end
#
View
1 spec/wukong/encoding_spec.rb
@@ -32,5 +32,4 @@
end
end
-
end
View
4 wukong.gemspec
@@ -5,11 +5,11 @@
Gem::Specification.new do |s|
s.name = "wukong"
- s.version = "2.1.0"
+ s.version = "2.1.1"
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
s.authors = ["Infochimps"]
- s.date = "2012-08-13"
+ s.date = "2012-08-17"
s.description = " Treat your dataset like a:\n\n * stream of lines when it's efficient to process by lines\n * stream of field arrays when it's efficient to deal directly with fields\n * stream of lightweight objects when it's efficient to deal with objects\n\n Wukong is friends with Hadoop the elephant, Pig the query language, and the cat on your command line.\n"
s.email = "coders@infochimps.org"
s.executables = ["wu-lign", "hdp-parts_to_keys.rb"]

0 comments on commit fb3645a

Please sign in to comment.
Something went wrong with that request. Please try again.