removing tuples, using static method on scalar scanner

lib/psych/scalar_scanner.rb

@@ -12,25 +12,29 @@ def initialize string
     end
 
     def tokenize
-      return [:NULL, nil] if @string.empty?
+      self.class.tokenize @string
+    end
+
+    def self.tokenize string
+      return nil if string.empty?
 
-      case @string
+      case string
       when /^[A-Za-z~]/
-        return [:SCALAR, @string] if @string.length > 5
-        case @string
+        return string if string.length > 5
+        case string
         when /^[^ytonf~]/i
-          [:SCALAR, @string]
+          string
         when '~', /^null$/i
-          [:NULL, nil]
+          nil
         when /^(yes|true|on)$/i
-          [:BOOLEAN, true]
+          true
         when /^(no|false|off)$/i
-          [:BOOLEAN, false]
+          false
         else
-          [:SCALAR, @string]
+          string
         end
       when TIME
-        date, time = *(@string.split(/[ tT]/, 2))
+        date, time = *(string.split(/[ tT]/, 2))
         (yy, m, dd) = date.split('-').map { |x| x.to_i }
         md = time.match(/(\d+:\d+:\d+)(\.\d*)?\s*(Z|[-+]\d+(:\d\d)?)?/)
 
@@ -39,44 +43,41 @@ def tokenize
 
         time = Time.utc(yy, m, dd, hh, mm, ss, us)
 
-        return [:TIME, time] if 'Z' == md[3]
+        return time if 'Z' == md[3]
 
         tz = md[3] ? Integer(md[3].split(':').first) : 0
-        [:TIME, Time.at((time - (tz * 3600)).to_i, us)]
+        Time.at((time - (tz * 3600)).to_i, us)
       when /^\d{4}-\d{1,2}-\d{1,2}$/
         require 'date'
-        [:DATE, Date.strptime(@string, '%Y-%m-%d')]
+        Date.strptime(string, '%Y-%m-%d')
       when /^\.inf$/i
-        [:POSITIVE_INFINITY, 1 / 0.0]
+        1 / 0.0
       when /^-\.inf$/i
-        [:NEGATIVE_INFINITY, -1 / 0.0]
+        -1 / 0.0
       when /^\.nan$/i
-        [:NAN, 0.0 / 0.0]
+        0.0 / 0.0
       when /^:./
-        if @string =~ /^:(["'])(.*)\1/
-          [:SYMBOL, $2.sub(/^:/, '').to_sym]
+        if string =~ /^:(["'])(.*)\1/
+          $2.sub(/^:/, '').to_sym
         else
-          [:SYMBOL, @string.sub(/^:/, '').to_sym]
+          string.sub(/^:/, '').to_sym
         end
       when /^[-+]?[1-9][0-9_]*(:[0-5]?[0-9])+$/
         i = 0
-        @string.split(':').each_with_index do |n,e|
+        string.split(':').each_with_index do |n,e|
           i += (n.to_i * 60 ** (e - 2).abs)
         end
-
-        [:INTEGER, i]
+        i
       when /^[-+]?[0-9][0-9_]*(:[0-5]?[0-9])+\.[0-9_]*$/
         i = 0
-        @string.split(':').each_with_index do |n,e|
+        string.split(':').each_with_index do |n,e|
           i += (n.to_f * 60 ** (e - 2).abs)
         end
-
-        [:FLOAT, i]
+        i
       else
-        return [:INTEGER, Integer(@string.gsub(/[,_]/, ''))] rescue ArgumentError
-        return [:FLOAT, Float(@string.gsub(/[,_]/, ''))] rescue ArgumentError
-
-        [:SCALAR, @string]
+        return Integer(string.gsub(/[,_]/, '')) rescue ArgumentError
+        return Float(string.gsub(/[,_]/, '')) rescue ArgumentError
+        string
       end
     end
   end

lib/psych/visitors/to_ruby.rb

@@ -28,7 +28,7 @@ def visit_Psych_Nodes_Scalar o
         @st[o.anchor] = o.value if o.anchor
 
         return o.value if o.quoted
-        return resolve_unknown(o) unless o.tag
+        return ScalarScanner.tokenize(o.value) unless o.tag
 
         case o.tag
         when '!binary', 'tag:yaml.org,2002:binary'
@@ -40,7 +40,7 @@ def visit_Psych_Nodes_Scalar o
         when "!ruby/object:Rational"
           Rational(o.value)
         when "tag:yaml.org,2002:float", "!float"
-          Float(ScalarScanner.new(o.value).tokenize.last)
+          Float(ScalarScanner.tokenize(o.value))
         when "!ruby/regexp"
           o.value =~ /^\/(.*)\/([mix]*)$/
           source  = $1
@@ -62,7 +62,7 @@ def visit_Psych_Nodes_Scalar o
           args.push(args.delete_at(1) == '...')
           Range.new(*args)
         else
-          resolve_unknown o
+          ScalarScanner.tokenize o.value
         end
       end
 
@@ -199,10 +199,6 @@ def resolve_class klassname
           raise ex
         end
       end
-
-      def resolve_unknown o
-        ScalarScanner.new(o.value).tokenize.last
-      end
     end
   end
 end

lib/psych/visitors/yaml_tree.rb

@@ -149,7 +149,7 @@ def visit_String o
         else
           str   = o
           tag   = nil
-          quote = ScalarScanner.new(o).tokenize.first != :SCALAR
+          quote = !(String === ScalarScanner.new(o).tokenize)
           plain = !quote
         end
 

test/test_scalar_scanner.rb

@@ -9,69 +9,67 @@ def test_scan_time
       '2001-12-15 2:59:43.10',
     ].each do |time|
       ss = Psych::ScalarScanner.new time
-      assert_equal :TIME, ss.tokenize.first
+      assert_instance_of Time, ss.tokenize
     end
   end
 
   def test_scan_date
     date = '1980-12-16'
     ss = Psych::ScalarScanner.new date
-    type, token = ss.tokenize
-    assert_equal :DATE, type
+    token = ss.tokenize
     assert_equal 1980, token.year
     assert_equal 12, token.month
     assert_equal 16, token.day
   end
 
   def test_scan_inf
     ss = Psych::ScalarScanner.new('.inf')
-    assert_equal [:POSITIVE_INFINITY, 1 / 0.0], ss.tokenize
+    assert_equal 1 / 0.0, ss.tokenize
   end
 
   def test_scan_minus_inf
     ss = Psych::ScalarScanner.new('-.inf')
-    assert_equal [:NEGATIVE_INFINITY, -1 / 0.0], ss.tokenize
+    assert_equal -1 / 0.0, ss.tokenize
   end
 
   def test_scan_nan
     ss = Psych::ScalarScanner.new('.nan')
-    assert_equal :NAN, ss.tokenize.first
-    assert ss.tokenize.last.nan?
+    assert ss.tokenize.nan?
   end
 
   def test_scan_null
     ss = Psych::ScalarScanner.new('null')
-    assert_equal [:NULL, nil], ss.tokenize
+    assert_equal nil, ss.tokenize
 
     ss = Psych::ScalarScanner.new('~')
-    assert_equal [:NULL, nil], ss.tokenize
+    assert_equal nil, ss.tokenize
 
     ss = Psych::ScalarScanner.new('')
-    assert_equal [:NULL, nil], ss.tokenize
+    assert_equal nil, ss.tokenize
   end
 
   def test_scan_symbol
     ss = Psych::ScalarScanner.new(':foo')
-    assert_equal [:SYMBOL, :foo], ss.tokenize
+    assert_equal :foo, ss.tokenize
   end
 
   def test_scan_sexagesimal_float
     ss = Psych::ScalarScanner.new('190:20:30.15')
-    assert_equal [:FLOAT, 685230.15], ss.tokenize
+    assert_equal 685230.15, ss.tokenize
   end
 
   def test_scan_sexagesimal_int
     ss = Psych::ScalarScanner.new('190:20:30')
-    assert_equal [:INTEGER, 685230], ss.tokenize
+    assert_equal 685230, ss.tokenize
   end
 
   def test_scan_float
     ss = Psych::ScalarScanner.new('1.2')
-    assert_equal [:FLOAT, 1.2], ss.tokenize
+    assert_equal 1.2, ss.tokenize
   end
 
   def test_scan_true
     ss = Psych::ScalarScanner.new('true')
-    assert_equal [:BOOLEAN, true], ss.tokenize
+    assert_equal true, ss.tokenize
   end
 end