From a08b4e3f44c02b0950554ab921c5e64323d6a67b Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 10 Apr 2013 18:01:03 +0400 Subject: [PATCH 1/5] Pack floats table values in network and not native order --- codegen/codegen_helpers.py | 10 +++--- codegen/protocol.rb.pytemplate | 8 +++-- lib/amq/endianness.rb | 15 ++++++++ lib/amq/hacks.rb | 46 ++++++++++++++++++------- lib/amq/protocol/client.rb | 26 +++++++------- lib/amq/protocol/frame.rb | 2 +- lib/amq/protocol/table_value_decoder.rb | 15 ++++---- lib/amq/protocol/type_constants.rb | 16 ++++----- spec/amq/hacks_spec.rb | 20 +++++------ spec/amq/protocol/table_spec.rb | 16 ++++----- 10 files changed, 103 insertions(+), 71 deletions(-) create mode 100644 lib/amq/endianness.rb diff --git a/codegen/codegen_helpers.py b/codegen/codegen_helpers.py index a9081ec..d200340 100644 --- a/codegen/codegen_helpers.py +++ b/codegen/codegen_helpers.py @@ -16,9 +16,9 @@ def genSingleEncode(spec, cValue, unresolved_domain): elif type == 'long': buffer.append("buffer << [%s].pack(PACK_UINT32)" % (cValue,)) elif type == 'longlong': - buffer.append("buffer << AMQ::Hacks.pack_64_big_endian(%s)" % (cValue,)) + buffer.append("buffer << AMQ::Hacks.pack_uint64_big_endian(%s)" % (cValue,)) elif type == 'timestamp': - buffer.append("buffer << AMQ::Hacks.pack_64_big_endian(%s)" % (cValue,)) + buffer.append("buffer << AMQ::Hacks.pack_uint64_big_endian(%s)" % (cValue,)) elif type == 'bit': raise "Can't encode bit in genSingleEncode" elif type == 'table': @@ -58,7 +58,7 @@ def genSingleDecode(spec, field): buffer.append("%s = data[offset, 4].unpack(PACK_UINT32).first" % (cLvalue,)) buffer.append("offset += 4") elif type == 'longlong': - buffer.append("%s = AMQ::Hacks.unpack_64_big_endian(data[offset, 8]).first" % (cLvalue,)) + buffer.append("%s = AMQ::Hacks.unpack_uint64_big_endian(data[offset, 8]).first" % (cLvalue,)) buffer.append("offset += 8") elif type == 'timestamp': buffer.append("%s = data[offset, 8].unpack(PACK_UINT32_X2).first" % (cLvalue,)) @@ -91,13 +91,13 @@ def genSingleSimpleDecode(spec, field): elif type == 'longstr': buffer.append("data.to_s") elif type == 'octet': - buffer.append("data.unpack(PACK_CHAR).first") + buffer.append("data.unpack(PACK_INT8).first") elif type == 'short': buffer.append("data.unpack(PACK_UINT16).first") elif type == 'long': buffer.append("data.unpack(PACK_UINT32).first") elif type == 'longlong': - buffer.append("AMQ::Hacks.unpack_64_big_endian(data).first") + buffer.append("AMQ::Hacks.unpack_uint64_big_endian(data).first") elif type == 'timestamp': buffer.append("Time.at(data.unpack(PACK_UINT32_X2).last)") elif type == 'bit': diff --git a/codegen/protocol.rb.pytemplate b/codegen/protocol.rb.pytemplate index 40e0ce6..53d7051 100644 --- a/codegen/protocol.rb.pytemplate +++ b/codegen/protocol.rb.pytemplate @@ -20,6 +20,7 @@ module AMQ # caching EMPTY_STRING = "".freeze + PACK_INT8 = 'c'.freeze PACK_CHAR = 'C'.freeze PACK_UINT16 = 'n'.freeze PACK_UINT16_X2 = 'n2'.freeze @@ -30,7 +31,7 @@ module AMQ PACK_CHAR_UINT16_UINT32 = 'cnN'.freeze PACK_32BIT_FLOAT = 'f'.freeze - PACK_64BIT_FLOAT = 'd'.freeze + PACK_64BIT_FLOAT = 'G'.freeze @@ -262,7 +263,7 @@ module AMQ # result = [${klass.index}, 0, body_size, flags].pack('n2Qn') result = [${klass.index}, 0].pack(PACK_UINT16_X2) - result += AMQ::Hacks.pack_64_big_endian(body_size) + result += AMQ::Hacks.pack_uint64_big_endian(body_size) result += [flags].pack(PACK_UINT16) result + pieces.join(EMPTY_STRING) end @@ -392,7 +393,8 @@ module AMQ frames << HeaderFrame.new(properties_payload, channel) % endif % if "payload" in method.args(): - frames + self.encode_body(payload, channel, frame_size) + frames += self.encode_body(payload, channel, frame_size) + frames % endif % else: MethodFrame.new(buffer, channel) diff --git a/lib/amq/endianness.rb b/lib/amq/endianness.rb new file mode 100644 index 0000000..3530991 --- /dev/null +++ b/lib/amq/endianness.rb @@ -0,0 +1,15 @@ +module AMQ + module Endianness + BIG_ENDIAN = ([1].pack("s") == "\x00\x01") + + def big_endian? + BIG_ENDIAN + end + + def little_endian? + !BIG_ENDIAN + end + + extend self + end +end diff --git a/lib/amq/hacks.rb b/lib/amq/hacks.rb index eae2ea1..86c63d6 100644 --- a/lib/amq/hacks.rb +++ b/lib/amq/hacks.rb @@ -1,33 +1,53 @@ # encoding: binary +require 'amq/endianness' + # Ruby doesn't support pack to/unpack from # 64bit string in network byte order. module AMQ module Hacks - BIG_ENDIAN = ([1].pack("s") == "\x00\x01") - Q = "Q".freeze + UINT64 = "Q".freeze + INT16 = "c".freeze + + if Endianness.big_endian? + def self.pack_uint64_big_endian(long_long) + [long_long].pack(UINT64) + end - if BIG_ENDIAN - def self.pack_64_big_endian(long_long) - [long_long].pack(Q) + def self.unpack_uint64_big_endian(data) + data.unpack(UINT64) end - def self.unpack_64_big_endian(data) - data.unpack(Q) + def self.pack_int16_big_endian(short) + [long_long].pack(INT16) + end + + def self.unpack_int16_big_endian(data) + data.unpack(INT16) end else - def self.pack_64_big_endian(long_long) - result = [long_long].pack(Q) + def self.pack_uint64_big_endian(long_long) + result = [long_long].pack(UINT64) + result.bytes.to_a.reverse.map(&:chr).join + end + + def self.unpack_uint64_big_endian(data) + data = data.bytes.to_a.reverse.map(&:chr).join + data.unpack(UINT64) + end + + def self.pack_int16_big_endian(short) + result = [long_long].pack(INT16) result.bytes.to_a.reverse.map(&:chr).join end - def self.unpack_64_big_endian(data) + def self.unpack_int16_big_endian(data) data = data.bytes.to_a.reverse.map(&:chr).join - data.unpack(Q) + data.unpack(INT16) end end end end -# AMQ::Hacks.pack_64_big_endian(17) -# AMQ::Hacks.unpack_64_big_endian("\x00\x00\x00\x00\x00\x00\x00\x11") +# AMQ::Hacks.pack_uint64_big_endian(17) +# AMQ::Hacks.unpack_uint64_big_endian("\x00\x00\x00\x00\x00\x00\x00\x11") diff --git a/lib/amq/protocol/client.rb b/lib/amq/protocol/client.rb index 4dd1feb..3217666 100644 --- a/lib/amq/protocol/client.rb +++ b/lib/amq/protocol/client.rb @@ -19,6 +19,7 @@ module Protocol # caching EMPTY_STRING = "".freeze + PACK_INT8 = 'c'.freeze PACK_CHAR = 'C'.freeze PACK_UINT16 = 'n'.freeze PACK_UINT16_X2 = 'n2'.freeze @@ -29,10 +30,7 @@ module Protocol PACK_CHAR_UINT16_UINT32 = 'cnN'.freeze PACK_32BIT_FLOAT = 'f'.freeze - PACK_64BIT_FLOAT = 'd'.freeze - - PACK_SIGNED_8BIT = 'c'.freeze - PACK_SIGNED_16BIT = 's'.freeze + PACK_64BIT_FLOAT = 'G'.freeze @@ -268,6 +266,7 @@ def self.encode_body(body, channel, frame_size) array = Array.new while body payload, body = body[0, limit], body[limit, body.length - limit] + # array << [0x03, payload] array << BodyFrame.new(payload, channel) end @@ -1438,7 +1437,7 @@ def self.encode_message_id(value) # 1 << 6 def self.encode_timestamp(value) buffer = '' - buffer << AMQ::Hacks.pack_64_big_endian(value) + buffer << AMQ::Hacks.pack_uint64_big_endian(value) [9, 0x0040, buffer] end @@ -1487,7 +1486,7 @@ def self.encode_properties(body_size, properties) # result = [60, 0, body_size, flags].pack('n2Qn') result = [60, 0].pack(PACK_UINT16_X2) - result += AMQ::Hacks.pack_64_big_endian(body_size) + result += AMQ::Hacks.pack_uint64_big_endian(body_size) result += [flags].pack(PACK_UINT16) result + pieces.join(EMPTY_STRING) end @@ -1796,7 +1795,6 @@ def self.encode(channel, payload, user_headers, exchange, routing_key, mandatory properties_payload = Basic.encode_properties(payload.bytesize, properties) frames << HeaderFrame.new(properties_payload, channel) frames += self.encode_body(payload, channel, frame_size) - frames end @@ -1856,7 +1854,7 @@ def self.decode(data) offset += 1 consumer_tag = data[offset, length] offset += length - delivery_tag = AMQ::Hacks.unpack_64_big_endian(data[offset, 8]).first + delivery_tag = AMQ::Hacks.unpack_uint64_big_endian(data[offset, 8]).first offset += 8 bit_buffer = data[offset, 1].unpack(PACK_CHAR).first offset += 1 @@ -1925,7 +1923,7 @@ class GetOk < Protocol::Method # @return def self.decode(data) offset = 0 - delivery_tag = AMQ::Hacks.unpack_64_big_endian(data[offset, 8]).first + delivery_tag = AMQ::Hacks.unpack_uint64_big_endian(data[offset, 8]).first offset += 8 bit_buffer = data[offset, 1].unpack(PACK_CHAR).first offset += 1 @@ -1996,7 +1994,7 @@ class Ack < Protocol::Method # @return def self.decode(data) offset = 0 - delivery_tag = AMQ::Hacks.unpack_64_big_endian(data[offset, 8]).first + delivery_tag = AMQ::Hacks.unpack_uint64_big_endian(data[offset, 8]).first offset += 8 bit_buffer = data[offset, 1].unpack(PACK_CHAR).first offset += 1 @@ -2019,7 +2017,7 @@ def self.has_content? def self.encode(channel, delivery_tag, multiple) buffer = '' buffer << @packed_indexes - buffer << AMQ::Hacks.pack_64_big_endian(delivery_tag) + buffer << AMQ::Hacks.pack_uint64_big_endian(delivery_tag) bit_buffer = 0 bit_buffer = bit_buffer | (1 << 0) if multiple buffer << [bit_buffer].pack(PACK_CHAR) @@ -2044,7 +2042,7 @@ def self.has_content? def self.encode(channel, delivery_tag, requeue) buffer = '' buffer << @packed_indexes - buffer << AMQ::Hacks.pack_64_big_endian(delivery_tag) + buffer << AMQ::Hacks.pack_uint64_big_endian(delivery_tag) bit_buffer = 0 bit_buffer = bit_buffer | (1 << 0) if requeue buffer << [bit_buffer].pack(PACK_CHAR) @@ -2132,7 +2130,7 @@ class Nack < Protocol::Method # @return def self.decode(data) offset = 0 - delivery_tag = AMQ::Hacks.unpack_64_big_endian(data[offset, 8]).first + delivery_tag = AMQ::Hacks.unpack_uint64_big_endian(data[offset, 8]).first offset += 8 bit_buffer = data[offset, 1].unpack(PACK_CHAR).first offset += 1 @@ -2157,7 +2155,7 @@ def self.has_content? def self.encode(channel, delivery_tag, multiple, requeue) buffer = '' buffer << @packed_indexes - buffer << AMQ::Hacks.pack_64_big_endian(delivery_tag) + buffer << AMQ::Hacks.pack_uint64_big_endian(delivery_tag) bit_buffer = 0 bit_buffer = bit_buffer | (1 << 0) if multiple bit_buffer = bit_buffer | (1 << 1) if requeue diff --git a/lib/amq/protocol/frame.rb b/lib/amq/protocol/frame.rb index a39d81b..cba1c7c 100644 --- a/lib/amq/protocol/frame.rb +++ b/lib/amq/protocol/frame.rb @@ -161,7 +161,7 @@ def decode_payload # the total size of the content body, that is, the sum of the body sizes for the # following content body frames. Zero indicates that there are no content body frames. # So this is NOT related to this very header frame! - @body_size = AMQ::Hacks.unpack_64_big_endian(@payload[4..11]).first + @body_size = AMQ::Hacks.unpack_uint64_big_endian(@payload[4..11]).first @data = @payload[12..-1] @properties = Basic.decode_properties(@data) end diff --git a/lib/amq/protocol/table_value_decoder.rb b/lib/amq/protocol/table_value_decoder.rb index e424dfe..3180e40 100644 --- a/lib/amq/protocol/table_value_decoder.rb +++ b/lib/amq/protocol/table_value_decoder.rb @@ -1,5 +1,6 @@ # encoding: binary +require "amq/endianness" require "amq/protocol/client" require "amq/protocol/type_constants" require "amq/protocol/table" @@ -20,10 +21,6 @@ class TableValueDecoder # API # - BIG_ENDIAN = ([1].pack("s") == "\x00\x01") - Q = "q".freeze - - def self.decode_array(data, initial_offset) array_length = data.slice(initial_offset, 4).unpack(PACK_UINT32).first @@ -102,9 +99,9 @@ def self.decode_integer(data, offset) end # self.decode_integer(data, offset) - if BIG_ENDIAN + if AMQ::Endianness.big_endian? def self.decode_long(data, offset) - v = data.slice(offset, 8).unpack(Q) + v = data.slice(offset, 8).unpack(PACK_INT64) offset += 8 [v, offset] @@ -112,7 +109,7 @@ def self.decode_long(data, offset) else def self.decode_long(data, offset) slice = data.slice(offset, 8).bytes.to_a.reverse.map(&:chr).join - v = slice.unpack(Q).first + v = slice.unpack(PACK_INT64).first offset += 8 [v, offset] @@ -177,13 +174,13 @@ def self.decode_hash(data, offset) def self.decode_short_short(data, offset) - v = data.slice(offset, 1).unpack(PACK_SIGNED_8BIT).first + v = data.slice(offset, 1).unpack(PACK_INT8).first offset += 1 [v, offset] end def self.decode_short(data, offset) - v = data.slice(offset, 2).unpack(PACK_SIGNED_16BIT).first + v = AMQ::Hacks.unpack_int16_big_endian(data.slice(offset, 2)).first offset += 2 [v, offset] end diff --git a/lib/amq/protocol/type_constants.rb b/lib/amq/protocol/type_constants.rb index 4fd338b..c0bf388 100644 --- a/lib/amq/protocol/type_constants.rb +++ b/lib/amq/protocol/type_constants.rb @@ -5,18 +5,18 @@ module Protocol module TypeConstants TYPE_STRING = 'S'.freeze TYPE_INTEGER = 'I'.freeze - TYPE_HASH = 'F'.freeze TYPE_TIME = 'T'.freeze TYPE_DECIMAL = 'D'.freeze - TYPE_BOOLEAN = 't'.freeze - TYPE_SIGNED_8BIT = 'c'.freeze - TYPE_SIGNED_16BIT = 's'.freeze - TYPE_SIGNED_64BIT = 'l'.freeze - TYPE_32BIT_FLOAT = 'f'.freeze + TYPE_HASH = 'F'.freeze + TYPE_ARRAY = 'A'.freeze + TYPE_SIGNED_8BIT = 'b'.freeze TYPE_64BIT_FLOAT = 'd'.freeze - TYPE_VOID = 'V'.freeze + TYPE_32BIT_FLOAT = 'f'.freeze + TYPE_SIGNED_64BIT = 'l'.freeze + TYPE_SIGNED_16BIT = 's'.freeze + TYPE_BOOLEAN = 't'.freeze TYPE_BYTE_ARRAY = 'x'.freeze - TYPE_ARRAY = 'A'.freeze + TYPE_VOID = 'V'.freeze TEN = '10'.freeze BOOLEAN_TRUE = "\x01".freeze diff --git a/spec/amq/hacks_spec.rb b/spec/amq/hacks_spec.rb index f9c42bc..04a7521 100644 --- a/spec/amq/hacks_spec.rb +++ b/spec/amq/hacks_spec.rb @@ -21,40 +21,40 @@ module AMQ it "packs integers into big-endian string" do examples.each do |key, value| - described_class.pack_64_big_endian(key).should == value + described_class.pack_uint64_big_endian(key).should == value end end it "should unpack string representation into integer" do examples.each do |key, value| - described_class.unpack_64_big_endian(value)[0].should == key + described_class.unpack_uint64_big_endian(value)[0].should == key end end - + if RUBY_VERSION < '1.9' describe "with utf encoding" do before do $KCODE = 'u' end - - after do + + after do $KCODE = 'NONE' end - + it "packs integers into big-endian string" do examples.each do |key, value| - described_class.pack_64_big_endian(key).should == value + described_class.pack_uint64_big_endian(key).should == value end end it "should unpack string representation into integer" do examples.each do |key, value| - described_class.unpack_64_big_endian(value)[0].should == key + described_class.unpack_uint64_big_endian(value)[0].should == key end end end end - + end end -end \ No newline at end of file +end diff --git a/spec/amq/protocol/table_spec.rb b/spec/amq/protocol/table_spec.rb index a3edf3e..0612cd5 100644 --- a/spec/amq/protocol/table_spec.rb +++ b/spec/amq/protocol/table_spec.rb @@ -29,7 +29,7 @@ module Protocol { {} => "\x00\x00\x00\x00", {"test" => 1} => "\x00\x00\x00\n\x04testI\x00\x00\x00\x01", - {"float" => 1.92} => "\x00\x00\x00\x0F\x05floatd\xB8\x1E\x85\xEBQ\xB8\xFE?", + {"float" => 1.92} => "\x00\x00\x00\x0F\x05floatd?\xFE\xB8Q\xEB\x85\x1E\xB8", {"test" => "string"} => "\x00\x00\x00\x10\x04testS\x00\x00\x00\x06string", {"test" => {}} => "\x00\x00\x00\n\x04testF\x00\x00\x00\x00", {"test" => bigdecimal_1} => "\x00\x00\x00\v\x04testD\x00\x00\x00\x00\x01", @@ -49,20 +49,20 @@ module Protocol Table.encode(nil).should eql(encoded_value) end - it "should return \x00\x00\x00\a\x04testt\x01 for { :test => true }" do + it "should serialize { :test => true }" do Table.encode(:test => true).should eql("\x00\x00\x00\a\x04testt\x01") end - it "should return \x00\x00\x00\a\x04testt\x00 for { :test => false }" do + it "should serialize { :test => false }" do Table.encode(:test => false).should eql("\x00\x00\x00\a\x04testt\x00") end - it "should return \"\x00\x00\x00\n\x04testI\x00\x00\x00\x01\" for { :coordinates => { :latitude => 59.35 } }" do - Table.encode(:coordinates => { :latitude => 59.35 }).should eql("\000\000\000#\vcoordinatesF\000\000\000\022\blatituded\315\314\314\314\314\254M@") + it "should serialize { :coordinates => { :latitude => 59.35 } }" do + Table.encode(:coordinates => { :latitude => 59.35 }).should eql("\x00\x00\x00#\vcoordinatesF\x00\x00\x00\x12\blatituded@M\xAC\xCC\xCC\xCC\xCC\xCD") end - it "should return \"\x00\x00\x00\n\x04testI\x00\x00\x00\x01\" for { :coordinates => { :longitude => 18.066667 } }" do - Table.encode(:coordinates => { :longitude => 18.066667 }).should eql("\000\000\000$\vcoordinatesF\000\000\000\023\tlongituded\361\270\250\026\021\0212@") + it "should serialize { :coordinates => { :longitude => 18.066667 } }" do + Table.encode(:coordinates => { :longitude => 18.066667 }).should eql("\x00\x00\x00$\vcoordinatesF\x00\x00\x00\x13\tlongituded@2\x11\x11\x16\xA8\xB8\xF1") end DATA.each do |data, encoded| @@ -182,7 +182,7 @@ module Protocol it 'is capable of decoding 16bit signed integers' do output = TableValueDecoder.decode_short("\b\xC0",0).first - output.should == -16376 + output.should == -64 end it "is capable of decoding tables" do From 9de1af084d4ceeef048c2f9f035d1171eab682fd Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 10 Apr 2013 18:01:44 +0400 Subject: [PATCH 2/5] 1.3.0 --- lib/amq/protocol/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/amq/protocol/version.rb b/lib/amq/protocol/version.rb index 765aa0b..910865d 100644 --- a/lib/amq/protocol/version.rb +++ b/lib/amq/protocol/version.rb @@ -1,5 +1,5 @@ module AMQ module Protocol - VERSION = "1.3.0.pre1" + VERSION = "1.3.0" end # Protocol end # AMQ From b65da934db2092a0e286a01652f88e4654cc2c92 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 10 Apr 2013 18:04:47 +0400 Subject: [PATCH 3/5] Correct 1.8.7 tests --- spec/amq/protocol/table_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/amq/protocol/table_spec.rb b/spec/amq/protocol/table_spec.rb index 0612cd5..6479ce7 100644 --- a/spec/amq/protocol/table_spec.rb +++ b/spec/amq/protocol/table_spec.rb @@ -18,7 +18,7 @@ module Protocol { {} => "\000\000\000\000", {"test" => 1} => "\000\000\000\n\004testI\000\000\000\001", - {"float" => 1.87} => "\000\000\000\017\005floatd\354Q\270\036\205\353\375?", + {"float" => 1.87} => "\000\000\000\017\005floatd?\375\353\205\036\270Q\354", {"test" => "string"} => "\000\000\000\020\004testS\000\000\000\006string", {"test" => {}} => "\000\000\000\n\004testF\000\000\000\000", {"test" => bigdecimal_1} => "\000\000\000\v\004testD\000\000\000\000\001", From 973f74e918af2651f86eb73c7836d0c3acce6a20 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 10 Apr 2013 18:05:28 +0400 Subject: [PATCH 4/5] Now working on 1.4.0.pre --- lib/amq/protocol/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/amq/protocol/version.rb b/lib/amq/protocol/version.rb index 910865d..f0b774f 100644 --- a/lib/amq/protocol/version.rb +++ b/lib/amq/protocol/version.rb @@ -1,5 +1,5 @@ module AMQ module Protocol - VERSION = "1.3.0" + VERSION = "1.4.0.pre1" end # Protocol end # AMQ From d618043036a39eb7bad338b1a072463eb35692c7 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 10 Apr 2013 18:06:04 +0400 Subject: [PATCH 5/5] Bump minimum RSpec version --- Gemfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 09b68b6..1939587 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ # encoding: utf-8 -source :rubygems +source "https://rubygems.org" group :development do # excludes Windows, Rubinius and JRuby @@ -8,6 +8,6 @@ group :development do end group :test do - gem "rspec", ">= 2.6.0" + gem "rspec", ">= 2.13.0" gem "effin_utf8" end