fix builtins implemented in C++ to return UTF-8 Strings

added rb_utf8_str_new() helpers for creating UTF-8 Strings

src/binary/Builtin.cc

@@ -31,6 +31,7 @@ extern "C" {
 #include "Y2RubyTypeConv.h"
 #include "RubyLogger.h"
 #include "YRuby.h"
+#include "Y2RubyUtils.h"
 
 static VALUE rb_mSCR;
 static VALUE rb_mWFM;
@@ -169,7 +170,7 @@ extern "C" {
     std::string utf_res;
     if (!recode(res,utf_res))
       return Qnil;
-    return rb_str_new2(utf_res.c_str());
+    return rb_utf8_str_new(utf_res);
   }
 
   // crypt part taken from y2crypt from yast core
@@ -310,7 +311,7 @@ extern "C" {
     char * res = crypt_pass(source, type);
     if (!res)
       return Qnil;
-    VALUE ret = rb_str_new2(res);
+    VALUE ret = rb_utf8_str_new(res);
     delete res;
     return ret;
   }
@@ -537,7 +538,7 @@ extern "C" {
     }
 
     if (result.solved)
-      return rb_str_new2 (result.result_str.c_str ());
+      return rb_utf8_str_new(result.result_str);
 
     return Qnil;
   }
@@ -598,7 +599,7 @@ extern "C" {
     if (result.solved) {
       for (int i = 1; i <= result.match_nb; i++)
       {
-          rb_ary_push(list, rb_str_new2 (result.match_str[i].c_str()));
+          rb_ary_push(list, rb_utf8_str_new(result.match_str[i]));
       }
     }
 

src/binary/Y2RubyUtils.cc

@@ -1,11 +1,17 @@
 #include <vector>
 #include <string>
 
+#include <ruby.h>
+#include <ruby/encoding.h>
+
 #include "y2util/stringutil.h"
 #include "Y2RubyUtils.h"
 
 using namespace std;
 
+// cache the UTF-8 encoding object
+static rb_encoding *utf8;
+
 static VALUE const_get_wrapper(VALUE input)
 {
   VALUE *data = (VALUE*) input;
@@ -32,3 +38,18 @@ VALUE y2ruby_nested_const_get(const std::string &name)
   }
   return module;
 }
+
+VALUE rb_utf8_str_new(const std::string &str) {
+  if (!utf8)
+    utf8 = rb_enc_find("UTF-8");
+
+  return rb_enc_str_new(str.c_str(), str.size(), utf8);
+}
+
+VALUE rb_utf8_str_new(const char *str) {
+  if (!utf8)
+    utf8 = rb_enc_find("UTF-8");
+
+  return rb_enc_str_new(str, strlen(str), utf8);
+}
+

src/binary/Y2RubyUtils.h

@@ -31,4 +31,11 @@ as published by the Free Software Foundation; either version
  */
 VALUE y2ruby_nested_const_get(const std::string &name);
 
+/**
+ * Create Ruby String object from a C++ string
+ * The resulting string has UTF-8 encoding
+ */
+VALUE rb_utf8_str_new(const std::string &str);
+VALUE rb_utf8_str_new(const char *str);
+
 #endif

src/binary/Y2YCPTypeConv.cc

@@ -41,9 +41,8 @@ as published by the Free Software Foundation; either version
 
 #include "YRuby.h"
 
-#include <ruby/encoding.h>
-
 #include "Y2YCPTypeConv.h"
+#include "Y2RubyUtils.h"
 
 //must match same magic id as in vica versa conversion
 #define YCP_EXTERNAL_MAGIC "Ruby object"
@@ -58,7 +57,7 @@ ycp_path_to_rb_path( YCPPath ycppath )
 
   VALUE ycp = rb_define_module("YCP");
   VALUE cls = rb_const_get(ycp, rb_intern("Path"));
-  VALUE value = rb_str_new2(ycppath->asString()->value().c_str());
+  VALUE value = rb_utf8_str_new(ycppath->asString()->value());
   return rb_class_new_instance(1,&value,cls);
 }
 
@@ -117,7 +116,7 @@ ycp_ext_to_rb_ext( YCPExternal ext )
   VALUE cls = rb_const_get(ycp, rb_intern("External"));
   // FIXME marking and deallocation
   VALUE tdata = Data_Wrap_Struct(cls, 0, NULL, new YCPExternal(ext));
-  VALUE argv[] = {rb_str_new2(ext->magic().c_str())};
+  VALUE argv[] = {rb_utf8_str_new(ext->magic())};
   rb_obj_call_init(tdata, 1, argv);
   return tdata;
 
@@ -134,9 +133,6 @@ ycp_ext_to_rb_ext( YCPExternal ext )
 extern "C" VALUE
 ycpvalue_2_rbvalue( YCPValue ycpval )
 {
-  // cache the UTF-8 encoding object
-  static rb_encoding *utf8;
-
   // TODO
   // YT_BYTEBLOCK YT_CODE YT_RETURN YT_BREAK YT_ENTRY YT_ERROR  YT_REFERENCE YT_EXTERNA
   if (ycpval.isNull() || ycpval->isVoid())
@@ -149,13 +145,8 @@ ycpvalue_2_rbvalue( YCPValue ycpval )
   }
   else if (ycpval->isString())
   {
-    if (!utf8)
-    {
-      utf8 = rb_enc_find("UTF-8");
-    }
-
     // always use UTF-8 encoding
-    return rb_enc_str_new(ycpval->asString()->value().c_str(), ycpval->asString()->value().size(), utf8);
+    return rb_utf8_str_new(ycpval->asString()->value());
   }
   else if (ycpval->isPath())
   {

src/binary/YCP.cc

@@ -40,6 +40,7 @@ as published by the Free Software Foundation; either version
 
 #include "Y2YCPTypeConv.h"
 #include "Y2RubyTypeConv.h"
+#include "Y2RubyUtils.h"
 
 /*
  * Ruby module anchors
@@ -229,7 +230,7 @@ ycp_module_symbols(VALUE self, VALUE namespace_name)
   for (unsigned int i=0; i < ns->symbolCount(); ++i)
   {
     SymbolEntryPtr s = ns->symbolEntry(i);
-    VALUE name = rb_str_new2(s->name());
+    VALUE name = rb_utf8_str_new(s->name());
     VALUE type = ID2SYM(rb_intern(s->catString().c_str()));
     rb_hash_aset(res,name,type);
   }

src/binary/YRuby.cc

@@ -47,7 +47,7 @@ as published by the Free Software Foundation; either version
 
 void set_last_exception(VALUE& module,const string& message)
 {
-  rb_ivar_set(module,rb_intern("@__last_exception"),rb_str_new2(message.c_str()));
+  rb_ivar_set(module,rb_intern("@__last_exception"),rb_utf8_str_new(message));
 }
 
 YRuby * YRuby::_yRuby = 0;

tests/ruby/builtins_test.rb

@@ -141,13 +141,19 @@ def test_regexpsub
     assert_equal "lest test\tsrst", YCP::Builtins.regexpsub(" lest test\tsrst\t", "^[ \t]*(([^ \t]*[ \t]*[^ \t]+)*)[ \t]*$", "\\1")
     assert_equal "", YCP::Builtins.regexpsub("", "^[ \t]*(([^ \t]*[ \t]*[^ \t]+)*)[ \t]*$", "\\1")
     assert_equal "", YCP::Builtins.regexpsub("  \t  ", "^[ \t]*(([^ \t]*[ \t]*[^ \t]+)*)[ \t]*$", "\\1")
+
+    # the result must be UTF-8 string
+    assert_equal Encoding::UTF_8, YCP::Builtins.regexpsub("aaabbb", "(.*ab)", "s_\\1_e").encoding
   end
 
   def test_regexptokenize
     assert_equal ["aaabbB"], YCP::Builtins.regexptokenize("aaabbBb", "(.*[A-Z]).*")
     assert_equal ["aaab", "bb"], YCP::Builtins.regexptokenize("aaabbb", "(.*ab)(.*)")
     assert_equal [], YCP::Builtins.regexptokenize("aaabbb", "(.*ba).*")
     assert_equal nil, YCP::Builtins.regexptokenize("aaabbb", "(.*ba).*(");
+
+    # the result must be UTF-8 string
+    assert_equal Encoding::UTF_8, YCP::Builtins.regexptokenize("aaabbBb", "(.*[A-Z]).*").first.encoding
   end
 
   def test_tohexstring
@@ -794,7 +800,9 @@ def test_findlastnotof
   def test_float_tolstring
     old_lang = ENV["LANG"]
     ENV["LANG"] = "cs_CZ.utf-8"
-    assert_equal "0,5", YCP::Builtins::Float.tolstring(0.52,1)
+    ret = YCP::Builtins::Float.tolstring(0.52,1)
+    assert_equal "0,5", ret
+    assert_equal Encoding::UTF_8, ret.encoding
     ENV["LANG"] = old_lang
   end