Skip to content

add encodings support to unpack (compatible with msgpack-jruby) #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
technorama wants to merge 1 commit into from
Closed

add encodings support to unpack (compatible with msgpack-jruby) #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doclib/msgpack/unpacker.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class Unpacker
# Supported options:
#
# * *:symbolize_keys* deserialize keys of Hash objects as Symbol instead of String
# * *:encoding* set the default encoding for unpacked Strings
#
# See also Buffer#initialize for other options.
#
Expand Down
2 changes: 1 addition & 1 deletion ext/msgpack/buffer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
#include "rmem.h"

#ifdef COMPAT_HAVE_ENCODING /* see compat.h*/
int s_enc_ascii8bit;
static int s_enc_ascii8bit;
#endif

#ifndef HAVE_RB_STR_REPLACE
Expand Down
14 changes: 9 additions & 5 deletions ext/msgpack/unpacker.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@
#include "unpacker.h"
#include "rmem.h"

#ifdef COMPAT_HAVE_ENCODING
static int s_enc_utf8;
#endif

#if !defined(DISABLE_RMEM) && !defined(DISABLE_UNPACKER_STACK_RMEM) && \
MSGPACK_UNPACKER_STACK_CAPACITY * MSGPACK_UNPACKER_STACK_SIZE <= MSGPACK_RMEM_PAGE_SIZE
#define UNPACKER_STACK_RMEM
Expand All @@ -28,10 +32,6 @@
static msgpack_rmem_t s_stack_rmem;
#endif

#ifdef COMPAT_HAVE_ENCODING /* see compat.h*/
static int s_enc_utf8;
#endif

void msgpack_unpacker_static_init()
{
#ifdef UNPACKER_STACK_RMEM
Expand Down Expand Up @@ -71,6 +71,10 @@ void msgpack_unpacker_init(msgpack_unpacker_t* uk)
uk->stack = malloc(MSGPACK_UNPACKER_STACK_CAPACITY * sizeof(msgpack_unpacker_stack_t));
#endif
uk->stack_capacity = MSGPACK_UNPACKER_STACK_CAPACITY;

#ifdef COMPAT_HAVE_ENCODING
uk->encoding_index = s_enc_utf8;
#endif
}

void msgpack_unpacker_destroy(msgpack_unpacker_t* uk)
Expand Down Expand Up @@ -151,7 +155,7 @@ static inline int object_complete_string(msgpack_unpacker_t* uk, VALUE str)
{
#ifdef COMPAT_HAVE_ENCODING
// TODO ruby 2.0 has String#b method
ENCODING_SET(str, s_enc_utf8);
ENCODING_SET(str, uk->encoding_index);
#endif
return object_complete(uk, str);
}
Expand Down
6 changes: 6 additions & 0 deletions ext/msgpack/unpacker.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ struct msgpack_unpacker_t {

/* options */
bool symbolize_keys;
int encoding_index;
};

#define UNPACKER_BUFFER_(uk) (&(uk)->buffer)
Expand Down Expand Up @@ -91,6 +92,11 @@ static inline void msgpack_unpacker_set_symbolized_keys(msgpack_unpacker_t* uk,
uk->symbolize_keys = enable;
}

static inline void msgpack_unpacker_set_encoding_index(msgpack_unpacker_t* uk, int encoding_index)
{
uk->encoding_index = encoding_index;
}


/* error codes */
#define PRIMITIVE_CONTAINER_START 1
Expand Down
11 changes: 11 additions & 0 deletions ext/msgpack/unpacker_class.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,17 @@ void MessagePack_Unpacker_initialize(msgpack_unpacker_t* uk, VALUE io, VALUE opt
if(options != Qnil) {
VALUE v;

v = rb_hash_aref(options, ID2SYM(rb_intern("encoding")));
if (v != Qnil) {
const char *s = StringValueCStr(v);
int encoding_index = rb_enc_find_index(s);
if (encoding_index == -1) {
rb_raise(rb_eArgError, "unknown encoding name - %s", s);
}

msgpack_unpacker_set_encoding_index(uk, encoding_index);
}

v = rb_hash_aref(options, ID2SYM(rb_intern("symbolize_keys")));
msgpack_unpacker_set_symbolized_keys(uk, RTEST(v));
}
Expand Down
19 changes: 19 additions & 0 deletions spec/unpacker_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,26 @@
symbolized_hash = {:a => 'b', :c => 'd'}
unpacker.feed(MessagePack.pack(symbolized_hash)).read.should == symbolized_hash
end
it 'MessagePack.unpack encoding' do
pack = MessagePack.pack 'a'

MessagePack.load(pack).encoding.should == Encoding::UTF_8
MessagePack.load(pack, :encoding => 'utf-8').encoding.should == Encoding::UTF_8
MessagePack.load(pack, :encoding => 'euc-jp').encoding.should == Encoding::EUC_JP
end

it 'Unpacker#unpack encoding' do
pack = MessagePack.pack 'a'

unpacker = Unpacker.new
unpacker.feed(pack).read.encoding.should == Encoding::UTF_8

unpacker = Unpacker.new(:encoding => 'utf-8')
unpacker.feed(pack).read.encoding.should == Encoding::UTF_8

unpacker = Unpacker.new(:encoding => 'euc-jp')
unpacker.feed(pack).read.encoding.should == Encoding::EUC_JP
end
it "msgpack str 8 type" do
MessagePack.unpack([0xd9, 0x00].pack('C*')).should == ""
MessagePack.unpack([0xd9, 0x01].pack('C*') + 'a').should == "a"
Expand Down