testing scalar encoding supprt

ruby · Jan 5, 2010 · 965de3c · 965de3c
1 parent 37e694a
commit 965de3c
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 9 deletions.
diff --git a/ext/psych/parser.c b/ext/psych/parser.c
@@ -45,6 +45,7 @@ static VALUE parse(VALUE self, VALUE yaml)
   }
 
   int done = 0;
+  int encoding = YAML_ANY_ENCODING;
 
   VALUE handler = rb_iv_get(self, "@handler");
 
@@ -60,6 +61,8 @@ static VALUE parse(VALUE self, VALUE yaml)
 
     switch(event.type) {
       case YAML_STREAM_START_EVENT:
+        encoding = event.data.stream_start.encoding;
+
         rb_funcall(handler, rb_intern("start_stream"), 1,
             INT2NUM((long)event.data.stream_start.encoding)
         );
@@ -114,6 +117,8 @@ static VALUE parse(VALUE self, VALUE yaml)
               (long)event.data.scalar.length
           );
 
+          PSYCH_ASSOCIATE_ENCODING(val, encoding);
+
           VALUE anchor = event.data.scalar.anchor ?
             rb_str_new2((const char *)event.data.scalar.anchor) :
             Qnil;

diff --git a/ext/psych/psych.h b/ext/psych/psych.h
@@ -4,11 +4,31 @@
 #include <ruby.h>
 #include <yaml.h>
 
+#define PSYCH_ASSOCIATE_ENCODING(_value, _encoding) \
+  ({ \
+    switch(_encoding) { \
+      case YAML_ANY_ENCODING: \
+        break; \
+      case YAML_UTF8_ENCODING: \
+        rb_enc_associate_index(_value, rb_enc_find_index("UTF-8"));\
+        break; \
+      case YAML_UTF16LE_ENCODING: \
+        rb_enc_associate_index(_value, rb_enc_find_index("UTF-16LE"));\
+        break; \
+      case YAML_UTF16BE_ENCODING: \
+        rb_enc_associate_index(_value, rb_enc_find_index("UTF-16BE"));\
+        break; \
+      default:\
+        break; \
+    }\
+   })
+
 #include <parser.h>
 #include <emitter.h>
 #include <to_ruby.h>
 #include <yaml_tree.h>
 
 extern VALUE mPsych;
 
+
 #endif
diff --git a/lib/psych.rb b/lib/psych.rb
@@ -1,15 +1,7 @@
 require 'psych/visitable'
 
-require 'psych/nodes/node'
-require 'psych/nodes/stream'
-require 'psych/nodes/document'
-require 'psych/nodes/sequence'
-require 'psych/nodes/scalar'
-require 'psych/nodes/mapping'
-require 'psych/nodes/alias'
-
+require 'psych/nodes'
 require 'psych/visitors'
-
 require 'psych/handler'
 require 'psych/tree_builder'
 require 'psych/parser'

diff --git a/lib/psych/nodes.rb b/lib/psych/nodes.rb
@@ -0,0 +1,12 @@
+require 'psych/nodes/node'
+require 'psych/nodes/stream'
+require 'psych/nodes/document'
+require 'psych/nodes/sequence'
+require 'psych/nodes/scalar'
+require 'psych/nodes/mapping'
+require 'psych/nodes/alias'
+
+module Psych
+  module Nodes
+  end
+end
diff --git a/test/psych/test_scalar.rb b/test/psych/test_scalar.rb
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+
+require 'minitest/autorun'
+require 'psych'
+
+module Psych
+  class TestScalar < MiniTest::Unit::TestCase
+    def test_utf_8
+      assert_equal "日本語", Psych.load("--- 日本語")
+    end
+  end
+end