From 851a7ad8c69bed90c87c896d544f1d954ceb9e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Britto?= Date: Wed, 8 Jul 2020 00:59:11 -0300 Subject: [PATCH 1/7] Extract varint encode/decode concerns into their own module --- lib/protobuf/decoder.ex | 195 ++++------------- lib/protobuf/encoder.ex | 47 ++--- lib/protobuf/wire/varint.ex | 208 +++++++++++++++++++ test/pbt/encode_decode_varint_test.exs | 13 +- test/protobuf/decode/decode_type_test.exs | 28 ++- test/protobuf/decode/decode_varint_test.exs | 58 ------ test/protobuf/encoder/encode_varint_test.exs | 51 ----- test/protobuf/wire/varint_test.exs | 107 ++++++++++ test/support/doctest.ex | 16 ++ 9 files changed, 418 insertions(+), 305 deletions(-) create mode 100644 lib/protobuf/wire/varint.ex delete mode 100644 test/protobuf/decode/decode_varint_test.exs delete mode 100644 test/protobuf/encoder/encode_varint_test.exs create mode 100644 test/protobuf/wire/varint_test.exs diff --git a/lib/protobuf/decoder.ex b/lib/protobuf/decoder.ex index 34270afa..b0739c37 100644 --- a/lib/protobuf/decoder.ex +++ b/lib/protobuf/decoder.ex @@ -1,14 +1,13 @@ defmodule Protobuf.Decoder do @moduledoc false - import Protobuf.WireTypes - import Bitwise, only: [bsl: 2, bsr: 2, band: 2] - require Logger - @max_bits 64 - @mask64 bsl(1, @max_bits) - 1 + import Protobuf.{WireTypes, Wire.Varint} + import Bitwise, only: [bsr: 2, band: 2] alias Protobuf.DecodeError + require Logger + @spec decode(binary, atom) :: any def decode(data, module) do kvs = raw_decode_key(data, [], []) @@ -17,11 +16,6 @@ defmodule Protobuf.Decoder do reverse_repeated(struct, msg_props.repeated_fields) end - @doc false - def decode_raw(data) do - raw_decode_key(data, [], []) - end - @doc false # For performance defmacro decode_type_m(type, key, val) do @@ -216,155 +210,12 @@ defmodule Protobuf.Decoder do end end - @doc false - def decode_varint(bin, type \\ :key) do - raw_decode_varint(bin, [], type, []) - end - defp raw_decode_key(<<>>, result, []), do: Enum.reverse(result) - defp raw_decode_key(<>, result, groups) do - raw_decode_varint(bin, result, :key, groups) - end - - defp raw_decode_varint(<<0::1, x::7, rest::bits>>, result, type, groups) do - raw_handle_varint(type, rest, result, x, groups) - end - - defp raw_decode_varint(<<1::1, x0::7, 0::1, x1::7, rest::bits>>, result, type, groups) do - val = bsl(x1, 7) + x0 - raw_handle_varint(type, rest, result, val, groups) - end - - defp raw_decode_varint( - <<1::1, x0::7, 1::1, x1::7, 0::1, x2::7, rest::bits>>, - result, - type, - groups - ) do - val = bsl(x2, 14) + bsl(x1, 7) + x0 - raw_handle_varint(type, rest, result, val, groups) - end - - defp raw_decode_varint( - <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 0::1, x3::7, rest::bits>>, - result, - type, - groups - ) do - val = bsl(x3, 21) + bsl(x2, 14) + bsl(x1, 7) + x0 - raw_handle_varint(type, rest, result, val, groups) - end - - defp raw_decode_varint( - <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 0::1, x4::7, rest::bits>>, - result, - type, - groups - ) do - val = bsl(x4, 28) + bsl(x3, 21) + bsl(x2, 14) + bsl(x1, 7) + x0 - raw_handle_varint(type, rest, result, val, groups) - end - - defp raw_decode_varint( - <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 0::1, x5::7, - rest::bits>>, - result, - type, - groups - ) do - val = bsl(x5, 35) + bsl(x4, 28) + bsl(x3, 21) + bsl(x2, 14) + bsl(x1, 7) + x0 - raw_handle_varint(type, rest, result, val, groups) - end - - defp raw_decode_varint( - <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 1::1, x5::7, 0::1, - x6::7, rest::bits>>, - result, - type, - groups - ) do - val = bsl(x6, 42) + bsl(x5, 35) + bsl(x4, 28) + bsl(x3, 21) + bsl(x2, 14) + bsl(x1, 7) + x0 - raw_handle_varint(type, rest, result, val, groups) - end - - defp raw_decode_varint( - <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 1::1, x5::7, 1::1, - x6::7, 0::1, x7::7, rest::bits>>, - result, - type, - groups - ) do - val = - bsl(x7, 49) + bsl(x6, 42) + bsl(x5, 35) + bsl(x4, 28) + bsl(x3, 21) + bsl(x2, 14) + - bsl(x1, 7) + x0 - - raw_handle_varint(type, rest, result, val, groups) - end - - defp raw_decode_varint( - <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 1::1, x5::7, 1::1, - x6::7, 1::1, x7::7, 0::1, x8::7, rest::bits>>, - result, - type, - groups - ) do - val = - bsl(x8, 56) + bsl(x7, 49) + bsl(x6, 42) + bsl(x5, 35) + bsl(x4, 28) + bsl(x3, 21) + - bsl(x2, 14) + bsl(x1, 7) + x0 - - raw_handle_varint(type, rest, result, val, groups) - end - - defp raw_decode_varint( - <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 1::1, x5::7, 1::1, - x6::7, 1::1, x7::7, 1::1, x8::7, 0::1, x9::7, rest::bits>>, - result, - type, - groups - ) do - val = - bsl(x9, 63) + bsl(x8, 56) + bsl(x7, 49) + bsl(x6, 42) + bsl(x5, 35) + bsl(x4, 28) + - bsl(x3, 21) + bsl(x2, 14) + bsl(x1, 7) + x0 - - val = band(val, @mask64) - raw_handle_varint(type, rest, result, val, groups) - end - - defp raw_decode_varint(_, _, _, _) do - raise Protobuf.DecodeError, message: "cannot decode binary data" - end - - defp raw_handle_varint(:key, <>, result, key, groups) do - tag = bsr(key, 3) - wire_type = band(key, 7) - raw_handle_key(wire_type, tag, groups, bin, result) - end - - defp raw_handle_varint(:value, <<>>, result, val, []), do: Enum.reverse([val | result]) - - defp raw_handle_varint(:value, <>, result, val, []) do - raw_decode_varint(bin, [val | result], :key, []) - end - - defp raw_handle_varint(:value, <>, result, _val, groups) do - raw_decode_varint(bin, result, :key, groups) - end - - defp raw_handle_varint(:bytes_len, <>, result, len, []) do - <> = bin - raw_decode_key(rest, [bytes | result], []) - end - - defp raw_handle_varint(:bytes_len, <>, result, len, groups) do - <<_bytes::bytes-size(len), rest::bits>> = bin - raw_decode_key(rest, result, groups) - end - - defp raw_handle_varint(:packed, <<>>, result, val, _groups), do: [val | result] - - defp raw_handle_varint(:packed, <>, result, val, groups) do - raw_decode_varint(bin, [val | result], :packed, groups) + decoder :defp, :raw_decode_key, [:result, :groups] do + tag = bsr(value, 3) + wire_type = band(value, 7) + raw_handle_key(wire_type, tag, groups, rest, result) end defp raw_handle_key(wire_start_group(), opening, groups, <>, result) do @@ -395,6 +246,36 @@ defmodule Protobuf.Decoder do raw_decode_value(wire_type, bin, result, groups) end + decoder :defp, :raw_decode_varint, [:result, :type, :groups] do + raw_handle_varint(type, rest, result, value, groups) + end + + defp raw_handle_varint(:value, <<>>, result, val, []), do: Enum.reverse([val | result]) + + defp raw_handle_varint(:value, <>, result, val, []) do + raw_decode_key(bin, [val | result], []) + end + + defp raw_handle_varint(:value, <>, result, _val, groups) do + raw_decode_key(bin, result, groups) + end + + defp raw_handle_varint(:bytes_len, <>, result, len, []) do + <> = bin + raw_decode_key(rest, [bytes | result], []) + end + + defp raw_handle_varint(:bytes_len, <>, result, len, groups) do + <<_bytes::bytes-size(len), rest::bits>> = bin + raw_decode_key(rest, result, groups) + end + + defp raw_handle_varint(:packed, <<>>, result, val, _groups), do: [val | result] + + defp raw_handle_varint(:packed, <>, result, val, groups) do + raw_decode_varint(bin, [val | result], :packed, groups) + end + @doc false def raw_decode_value(wire, bin, result, groups \\ []) diff --git a/lib/protobuf/encoder.ex b/lib/protobuf/encoder.ex index 9e66c2b4..99dd1b3f 100644 --- a/lib/protobuf/encoder.ex +++ b/lib/protobuf/encoder.ex @@ -1,9 +1,9 @@ defmodule Protobuf.Encoder do @moduledoc false import Protobuf.WireTypes - import Bitwise, only: [bsr: 2, band: 2, bsl: 2, bor: 2] + import Bitwise, only: [bsl: 2, bor: 2] - alias Protobuf.{MessageProps, FieldProps} + alias Protobuf.{FieldProps, MessageProps, Wire.Varint} @spec encode(atom, map | struct, keyword) :: iodata def encode(mod, msg, opts) do @@ -114,14 +114,14 @@ defmodule Protobuf.Encoder do # so that oneof {:atom, v} can be encoded encoded = encode(type, v, iolist: true) byte_size = IO.iodata_length(encoded) - [fnum | encode_varint(byte_size)] ++ encoded + [fnum | Varint.encode(byte_size)] ++ encoded end) end defp encode_field(:packed, val, %{type: type, encoded_fnum: fnum}) do encoded = Enum.map(val, fn v -> encode_type(type, v) end) byte_size = IO.iodata_length(encoded) - [fnum | encode_varint(byte_size)] ++ encoded + [fnum | Varint.encode(byte_size)] ++ encoded end @spec class_field(map) :: atom @@ -143,24 +143,24 @@ defmodule Protobuf.Encoder do fnum |> bsl(3) |> bor(wire_type) - |> encode_varint() + |> Varint.encode() |> IO.iodata_to_binary() end @doc false @spec encode_type(atom, any) :: iodata - def encode_type(:int32, n) when n >= -0x80000000 and n <= 0x7FFFFFFF, do: encode_varint(n) + def encode_type(:int32, n) when n >= -0x80000000 and n <= 0x7FFFFFFF, do: Varint.encode(n) def encode_type(:int64, n) when n >= -0x8000000000000000 and n <= 0x7FFFFFFFFFFFFFFF, - do: encode_varint(n) + do: Varint.encode(n) def encode_type(:string, n), do: encode_type(:bytes, n) - def encode_type(:uint32, n) when n >= 0 and n <= 0xFFFFFFFF, do: encode_varint(n) - def encode_type(:uint64, n) when n >= 0 and n <= 0xFFFFFFFFFFFFFFFF, do: encode_varint(n) - def encode_type(:bool, true), do: encode_varint(1) - def encode_type(:bool, false), do: encode_varint(0) - def encode_type({:enum, type}, n) when is_atom(n), do: n |> type.value() |> encode_varint() - def encode_type({:enum, _}, n), do: encode_varint(n) + def encode_type(:uint32, n) when n >= 0 and n <= 0xFFFFFFFF, do: Varint.encode(n) + def encode_type(:uint64, n) when n >= 0 and n <= 0xFFFFFFFFFFFFFFFF, do: Varint.encode(n) + def encode_type(:bool, true), do: Varint.encode(1) + def encode_type(:bool, false), do: Varint.encode(0) + def encode_type({:enum, type}, n) when is_atom(n), do: n |> type.value() |> Varint.encode() + def encode_type({:enum, _}, n), do: Varint.encode(n) def encode_type(:float, :infinity), do: [0, 0, 128, 127] def encode_type(:float, :negative_infinity), do: [0, 0, 128, 255] def encode_type(:float, :nan), do: [0, 0, 192, 127] @@ -171,15 +171,15 @@ defmodule Protobuf.Encoder do def encode_type(:double, n), do: <> def encode_type(:bytes, n) do - len = n |> IO.iodata_length() |> encode_varint() + len = n |> IO.iodata_length() |> Varint.encode() len ++ n end def encode_type(:sint32, n) when n >= -0x80000000 and n <= 0x7FFFFFFF, - do: n |> encode_zigzag |> encode_varint + do: n |> encode_zigzag() |> Varint.encode() def encode_type(:sint64, n) when n >= -0x8000000000000000 and n <= 0x7FFFFFFFFFFFFFFF, - do: n |> encode_zigzag |> encode_varint + do: n |> encode_zigzag() |> Varint.encode() def encode_type(:fixed64, n) when n >= 0 and n <= 0xFFFFFFFFFFFFFFFF, do: <> @@ -199,21 +199,6 @@ defmodule Protobuf.Encoder do defp encode_zigzag(val) when val >= 0, do: val * 2 defp encode_zigzag(val) when val < 0, do: val * -2 - 1 - @doc false - @spec encode_varint(integer) :: iolist - def encode_varint(n) when n < 0 do - <> = <> - encode_varint(n) - end - - def encode_varint(n) when n <= 127 do - [n] - end - - def encode_varint(n) do - [<<1::1, band(n, 127)::7>> | encode_varint(bsr(n, 7))] - end - @doc false @spec wire_type(atom) :: integer def wire_type(:int32), do: wire_varint() diff --git a/lib/protobuf/wire/varint.ex b/lib/protobuf/wire/varint.ex new file mode 100644 index 00000000..78ed520e --- /dev/null +++ b/lib/protobuf/wire/varint.ex @@ -0,0 +1,208 @@ +defmodule Protobuf.Wire.Varint do + @moduledoc """ + Varint encoding and decoding utilities. + + https://developers.google.com/protocol-buffers/docs/encoding#varints + + For performance reasons, varint decoding must be built through a macro, so that binary + match contexts are reused and no new large binaries get allocated. You can define your + own varint decoders with the `decoder` macro, which generates function heads for up to + 10-bytes long varint-encoded data. + + defmodule VarintDecoders do + import Protobuf.Wire.Varint + + decoder :def, :decode_and_sum, [:plus] do + {:ok, value + plus, rest} + end + + def decode_all(<>), do: decode_all(bin, []) + + defp decode_all(<<>>, acc), do: acc + + decoder :defp, :decode_all, [:acc] do + decode_all(rest, [value | acc]) + end + end + + iex> VarintDecoders.decode_and_sum(<<35>>, 7) + {:ok, 42, ""} + + iex> VarintDecoders.decode_all("abcd asdf") + [102, 100, 115, 97, 32, 100, 99, 98, 97] + + Refer to [efficiency guide](http://www1.erlang.org/doc/efficiency_guide/binaryhandling.html) + for more on efficient binary handling. + + Encoding on the other hand is simpler. It takes an integer and returns an iolist with its + varint representation: + + iex> Protobuf.Wire.Varint.encode(35) + [35] + + iex> Protobuf.Wire.Varint.encode(1_234_567) + [<<135>>, <<173>>, 75] + """ + use Bitwise + + @max_bits 64 + @mask64 bsl(1, @max_bits) - 1 + + @varints [ + { + quote(do: <<0::1, value::7>>), + quote(do: value) + }, + { + quote(do: <<1::1, x0::7, 0::1, x1::7>>), + quote(do: x0 + bsl(x1, 7)) + }, + { + quote(do: <<1::1, x0::7, 1::1, x1::7, 0::1, x2::7>>), + quote(do: x0 + bsl(x1, 7) + bsl(x2, 14)) + }, + { + quote(do: <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 0::1, x3::7>>), + quote(do: x0 + bsl(x1, 7) + bsl(x2, 14) + bsl(x3, 21)) + }, + { + quote(do: <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 0::1, x4::7>>), + quote(do: x0 + bsl(x1, 7) + bsl(x2, 14) + bsl(x3, 21) + bsl(x4, 28)) + }, + { + quote do + <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 0::1, x5::7>> + end, + quote do + x0 + + bsl(x1, 7) + + bsl(x2, 14) + + bsl(x3, 21) + + bsl(x4, 28) + + bsl(x5, 35) + end + }, + { + quote do + <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 1::1, x5::7, 0::1, + x6::7>> + end, + quote do + x0 + + bsl(x1, 7) + + bsl(x2, 14) + + bsl(x3, 21) + + bsl(x4, 28) + + bsl(x5, 35) + + bsl(x6, 42) + end + }, + { + quote do + <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 1::1, x5::7, 1::1, + x6::7, 0::1, x7::7>> + end, + quote do + x0 + + bsl(x1, 7) + + bsl(x2, 14) + + bsl(x3, 21) + + bsl(x4, 28) + + bsl(x5, 35) + + bsl(x6, 42) + + bsl(x7, 49) + end + }, + { + quote do + <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 1::1, x5::7, 1::1, + x6::7, 1::1, x7::7, 0::1, x8::7>> + end, + quote do + x0 + + bsl(x1, 7) + + bsl(x2, 14) + + bsl(x3, 21) + + bsl(x4, 28) + + bsl(x5, 35) + + bsl(x6, 42) + + bsl(x7, 49) + + bsl(x8, 56) + end + }, + { + quote do + <<1::1, x0::7, 1::1, x1::7, 1::1, x2::7, 1::1, x3::7, 1::1, x4::7, 1::1, x5::7, 1::1, + x6::7, 1::1, x7::7, 1::1, x8::7, 0::1, x9::7>> + end, + quote do + band( + x0 + + bsl(x1, 7) + + bsl(x2, 14) + + bsl(x3, 21) + + bsl(x4, 28) + + bsl(x5, 35) + + bsl(x6, 42) + + bsl(x7, 49) + + bsl(x8, 56) + + bsl(x9, 63), + unquote(@mask64) + ) + end + } + ] + + defmacro decoder(kind, name, args \\ [], do: block) do + def_success(kind, name, args, block) ++ [def_failure(kind, name, args)] + end + + defp def_success(kind, name, args, block) do + args = Enum.map(args, fn arg -> {arg, [line: 1], nil} end) + + for {pattern, expression} <- @varints do + head = quote(do: unquote(name)(<>, unquote_splicing(args))) + + body = + quote do + var!(value) = unquote(expression) + var!(rest) = rest + unquote(block) + end + + quote do + case unquote(kind) do + :def -> def unquote(head), do: unquote(body) + :defp -> defp unquote(head), do: unquote(body) + end + end + end + end + + defp def_failure(kind, name, args) do + args = Enum.map(args, fn _ -> {:_, [line: 1], nil} end) + head = quote(do: unquote(name)(<<_::bits>>, unquote_splicing(args))) + body = quote(do: raise(Protobuf.DecodeError, message: "cannot decode binary data")) + + quote do + case unquote(kind) do + :def -> def unquote(head), do: unquote(body) + :defp -> defp unquote(head), do: unquote(body) + end + end + end + + @spec encode(integer) :: iolist + def encode(n) when n < 0 do + <> = <> + encode(n) + end + + def encode(n) when n <= 127 do + [n] + end + + def encode(n) do + [<<1::1, band(n, 127)::7>> | encode(bsr(n, 7))] + end +end diff --git a/test/pbt/encode_decode_varint_test.exs b/test/pbt/encode_decode_varint_test.exs index ed04d048..8d1d25d3 100644 --- a/test/pbt/encode_decode_varint_test.exs +++ b/test/pbt/encode_decode_varint_test.exs @@ -2,13 +2,18 @@ defmodule Protobuf.EncodeDecodeVarintTest do use ExUnit.Case, async: true use ExUnitProperties - alias Protobuf.{Encoder, Decoder} + import Protobuf.Wire.Varint + + decoder :defp, :decode do + "" = rest + value + end property "varint roundtrip" do check all n <- large_integer_gen() do - iodata = Encoder.encode_varint(n) + iodata = encode(n) bin = IO.iodata_to_binary(iodata) - [n] = Decoder.decode_varint(bin, :value) + n = decode(bin) assert <> == <> end end @@ -17,7 +22,7 @@ defmodule Protobuf.EncodeDecodeVarintTest do negative_large_integer_gen = map(large_integer_gen(), &(-abs(&1))) check all n <- negative_large_integer_gen do - assert IO.iodata_length(Encoder.encode_varint(n)) == 10 + assert IO.iodata_length(encode(n)) == 10 end end diff --git a/test/protobuf/decode/decode_type_test.exs b/test/protobuf/decode/decode_type_test.exs index d554ca4c..8a01f254 100644 --- a/test/protobuf/decode/decode_type_test.exs +++ b/test/protobuf/decode/decode_type_test.exs @@ -8,16 +8,28 @@ defmodule Protobuf.Decoder.DecodeTypeTest do decode_type_m(type, :fake_key, val) end - test "decode_type/2 varint" do + test "decode_type/3 varint" do assert 42 == decode_type(:int32, 42) end - test "decode_type/2 int64" do + test "decode_type/3 int64" do assert -1 == decode_type(:int64, -1) end - test "decode_type/2 string" do - assert "a" = decode_type(:string, "a") + test "decode_type/3 min int32" do + assert -2_147_483_648 == decode_type(:int32, 18_446_744_071_562_067_968) + end + + test "decode_type/3 max int32" do + assert -2_147_483_647 == decode_type(:int32, 18_446_744_071_562_067_969) + end + + test "decode_type/3 min int64" do + assert -9_223_372_036_854_775_808 == decode_type(:int64, 9_223_372_036_854_775_808) + end + + test "decode_type/3 max int64" do + assert 9_223_372_036_854_775_807 == decode_type(:int64, 9_223_372_036_854_775_807) end test "decode_type/3 min sint32" do @@ -36,6 +48,14 @@ defmodule Protobuf.Decoder.DecodeTypeTest do assert 9_223_372_036_854_775_807 == decode_type(:sint64, 18_446_744_073_709_551_614) end + test "decode_type/3 max uint32" do + assert 4_294_967_295 == decode_type(:uint32, 4_294_967_295) + end + + test "decode_type/3 max uint64" do + assert 9_223_372_036_854_775_807 == decode_type(:uint64, 9_223_372_036_854_775_807) + end + test "decode_type/3 bool works" do assert true == decode_type(:bool, 1) assert false == decode_type(:bool, 0) diff --git a/test/protobuf/decode/decode_varint_test.exs b/test/protobuf/decode/decode_varint_test.exs deleted file mode 100644 index 19df3491..00000000 --- a/test/protobuf/decode/decode_varint_test.exs +++ /dev/null @@ -1,58 +0,0 @@ -defmodule Protobuf.Decoder.DecodeVarintTest do - use ExUnit.Case, async: true - - import Protobuf.Decoder - require Logger - - def decode_type(type, val) do - decode_type_m(type, :fake_key, val) - end - - test "decode_varint 300" do - assert [300] == decode_varint(<<0b1010110000000010::16>>, :value) - end - - test "decode_varint 150" do - assert [1, 0, 150] == decode_varint(<<8, 150, 01>>) - end - - test "decode_varint zero value(int, bool, enum)" do - assert [] == decode_raw(<<>>) - end - - test "decode_varint+decode_type min int32" do - assert [1, 0, val = 18_446_744_071_562_067_968] == - decode_varint(<<8, 128, 128, 128, 128, 248, 255, 255, 255, 255, 1>>) - - assert -2_147_483_648 == decode_type(:int32, val) - end - - test "decode_varint max int32" do - [1, 0, 2_147_483_647] = decode_varint(<<8, 255, 255, 255, 255, 7>>) - end - - test "decode_varint+decode_type min int64" do - [1, 0, val] = decode_varint(<<8, 128, 128, 128, 128, 128, 128, 128, 128, 128, 1>>) - assert -9_223_372_036_854_775_808 == decode_type(:int64, val) - end - - test "decode_varint max int64" do - [1, 0, val] = decode_varint(<<8, 255, 255, 255, 255, 255, 255, 255, 255, 127>>) - assert 9_223_372_036_854_775_807 == decode_type(:int64, val) - end - - test "decode_varint max uint32" do - [1, 0, val] = decode_varint(<<8, 255, 255, 255, 255, 15>>) - assert 4_294_967_295 == decode_type(:uint32, val) - end - - test "decode_varint max uint64" do - [1, 0, val] = decode_varint(<<8, 255, 255, 255, 255, 255, 255, 255, 255, 255, 1>>) - assert 18_446_744_073_709_551_615 == decode_type(:uint64, val) - end - - test "decode_varint true/enum_1" do - [1, 0, val = 1] = decode_varint(<<8, 1>>) - assert true === decode_type(:bool, val) - end -end diff --git a/test/protobuf/encoder/encode_varint_test.exs b/test/protobuf/encoder/encode_varint_test.exs deleted file mode 100644 index 23c4be64..00000000 --- a/test/protobuf/encoder/encode_varint_test.exs +++ /dev/null @@ -1,51 +0,0 @@ -defmodule Protobuf.Encoder.DecodeVarintTest do - use ExUnit.Case, async: true - - alias Protobuf.Encoder - - test "encode_varint 300" do - assert encode(300) == <<0b1010110000000010::16>> - end - - test "encode_varint 150" do - assert encode(150) == <<150, 1>> - end - - test "encode_varint 0" do - assert encode(0) == <<0>> - end - - test "encode_varint/2 min int32" do - assert encode(-2_147_483_648) == - <<128, 128, 128, 128, 248, 255, 255, 255, 255, 1>> - end - - test "encode_varint max int32" do - assert encode(2_147_483_647) == <<255, 255, 255, 255, 7>> - end - - test "encode_varint/2 min int64" do - assert encode(-9_223_372_036_854_775_808) == - <<128, 128, 128, 128, 128, 128, 128, 128, 128, 1>> - end - - test "encode_varint max int64" do - assert encode(9_223_372_036_854_775_807) == - <<255, 255, 255, 255, 255, 255, 255, 255, 127>> - end - - test "encode_varint max uint32" do - assert encode(4_294_967_295) == <<255, 255, 255, 255, 15>> - end - - test "encode_varint max uint64" do - assert encode(18_446_744_073_709_551_615) == - <<255, 255, 255, 255, 255, 255, 255, 255, 255, 1>> - end - - defp encode(varint) do - varint - |> Encoder.encode_varint() - |> IO.iodata_to_binary() - end -end diff --git a/test/protobuf/wire/varint_test.exs b/test/protobuf/wire/varint_test.exs new file mode 100644 index 00000000..c3a60449 --- /dev/null +++ b/test/protobuf/wire/varint_test.exs @@ -0,0 +1,107 @@ +defmodule Protobuf.Wire.VarintTest do + use ExUnit.Case, async: true + doctest Protobuf.Wire.Varint + + alias Protobuf.Wire.Varint + + describe "encode/1" do + test "300" do + assert encode(300) == <<0b10101100, 0b00000010>> + end + + test "150" do + assert encode(150) == <<150, 1>> + end + + test "0" do + assert encode(0) == <<0>> + end + + test "1" do + assert encode(1) == <<1>> + end + + test "min int32" do + assert encode(-2_147_483_648) == <<128, 128, 128, 128, 248, 255, 255, 255, 255, 1>> + end + + test "max int32" do + assert encode(2_147_483_647) == <<255, 255, 255, 255, 7>> + end + + test "min int64" do + assert encode(-9_223_372_036_854_775_808) == + <<128, 128, 128, 128, 128, 128, 128, 128, 128, 1>> + end + + test "max int64" do + assert encode(9_223_372_036_854_775_807) == + <<255, 255, 255, 255, 255, 255, 255, 255, 127>> + end + + test "max uint32" do + assert encode(4_294_967_295) == <<255, 255, 255, 255, 15>> + end + + test "max uint64" do + assert encode(18_446_744_073_709_551_615) == + <<255, 255, 255, 255, 255, 255, 255, 255, 255, 1>> + end + + defp encode(n) do + n + |> Varint.encode() + |> IO.iodata_to_binary() + end + end + + describe "decode/1" do + require Varint + + Varint.decoder(:defp, :decode, do: {value, rest}) + + test "300" do + assert {300, ""} == decode(<<0b1010110000000010::16>>) + end + + test "150" do + assert {150, ""} == decode(<<150, 01>>) + end + + test "0" do + assert {0, ""} == decode(<<0>>) + end + + test "1" do + assert {1, ""} == decode(<<1>>) + end + + test "min int32" do + {val, ""} = decode(<<128, 128, 128, 128, 248, 255, 255, 255, 255, 1>>) + assert <<-2_147_483_648::signed-32>> == <> + end + + test "max int32" do + assert {2_147_483_647, ""} == decode(<<255, 255, 255, 255, 7>>) + end + + test "min int64" do + {val, ""} = decode(<<128, 128, 128, 128, 128, 128, 128, 128, 128, 1>>) + assert <<-9_223_372_036_854_775_808::signed-64>> == <> + end + + test "max int64" do + assert {9_223_372_036_854_775_807, ""} == + decode(<<255, 255, 255, 255, 255, 255, 255, 255, 127>>) + end + + test "max uint32" do + assert {4_294_967_295, ""} == decode(<<255, 255, 255, 255, 15>>) + end + + test "max uint64" do + assert {18_446_744_073_709_551_615, ""} == + decode(<<255, 255, 255, 255, 255, 255, 255, 255, 255, 1>>) + end + end +end diff --git a/test/support/doctest.ex b/test/support/doctest.ex index ea34e8e5..37050efe 100644 --- a/test/support/doctest.ex +++ b/test/support/doctest.ex @@ -17,3 +17,19 @@ defmodule Car do field :color, 1, type: Color, enum: true field :top_speed, 2, type: :float, json_name: "topSpeed" end + +defmodule VarintDecoders do + import Protobuf.Wire.Varint + + decoder :def, :decode_and_sum, [:plus] do + {:ok, value + plus, rest} + end + + def decode_all(<>), do: decode_all(bin, []) + + defp decode_all(<<>>, acc), do: acc + + decoder :defp, :decode_all, [:acc] do + decode_all(rest, [value | acc]) + end +end From c3d487aecaea950c0badff8b66fa0363d535173b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Britto?= Date: Sat, 11 Jul 2020 01:19:58 -0300 Subject: [PATCH 2/7] Refactor property-based tests By making them agnostic to implementation details, preventing direct access to private concerns. --- test/pbt/encode_decode_type_test.exs | 75 ++++++++++++---------------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/test/pbt/encode_decode_type_test.exs b/test/pbt/encode_decode_type_test.exs index 3bde730e..9dbb88e9 100644 --- a/test/pbt/encode_decode_type_test.exs +++ b/test/pbt/encode_decode_type_test.exs @@ -1,21 +1,24 @@ defmodule Protobuf.EncodeDecodeTypeTest.PropertyGenerator do - alias Protobuf.{Encoder} - - require Logger - import Protobuf.Decoder + def decode(type, bin) do + bin + |> TestMsg.Scalars.decode() + |> Map.fetch!(type) + end - def decode_type(wire, type, bin) do - [n] = raw_decode_value(wire, bin, []) - decode_type_m(type, :fake_key, n) + def encode(type, val) do + [{type, val}] + |> TestMsg.Scalars.new!() + |> Protobuf.Encoder.encode(iolist: false) end - defmacro make_property(gen_func, field_type, wire_type) do + defmacro make_property(gen_func, field_type) do quote do - property unquote(Atom.to_string(field_type)) <> " roundtrip" do + property "#{unquote(field_type)} roundtrip" do check all n <- unquote(gen_func) do - iodata = Encoder.encode_type(unquote(field_type), n) - bin = IO.iodata_to_binary(iodata) - assert n == decode_type(unquote(wire_type), unquote(field_type), bin) + field_type = unquote(field_type) + bin = encode(field_type, n) + + assert n == decode(field_type, bin) end end end @@ -24,28 +27,16 @@ defmodule Protobuf.EncodeDecodeTypeTest.PropertyGenerator do # Since float point is not precise, make canonical value before doing PBT # ref: http://hypothesis.works/articles/canonical-serialization/ # and try 0.2 here: https://www.h-schmidt.net/FloatConverter/IEEE754.html - defmacro make_canonical_property(gen_func, field_type, wire_type) do + defmacro make_canonical_property(gen_func, field_type) do quote do - property unquote(Atom.to_string(field_type)) <> " canonical roundtrip" do + property "#{unquote(field_type)} canonical roundtrip" do check all n <- unquote(gen_func) do - encoded_val = - unquote(field_type) - |> Encoder.encode_type(n) - |> IO.iodata_to_binary() - - canonical_val = - decode_type( - unquote(wire_type), - unquote(field_type), - encoded_val - ) - - bin = - unquote(field_type) - |> Encoder.encode_type(canonical_val) - |> IO.iodata_to_binary() + field_type = unquote(field_type) + encoded_val = encode(field_type, n) + canonical_val = decode(field_type, encoded_val) + bin = encode(field_type, canonical_val) - assert canonical_val == decode_type(unquote(wire_type), unquote(field_type), bin) + assert canonical_val == decode(field_type, bin) end end end @@ -74,18 +65,18 @@ defmodule Protobuf.EncodeDecodeTypeTest do map(integer(), &abs/1) end - make_property(integer(), :int32, 0) - make_property(large_integer(), :int64, 0) - make_property(uint32_gen(), :uint32, 0) - make_property(uint64_gen(), :uint64, 0) - make_property(integer(), :sint32, 0) - make_property(large_integer(), :sint64, 0) + make_property(integer(), :int32) + make_property(large_integer(), :int64) + make_property(uint32_gen(), :uint32) + make_property(uint64_gen(), :uint64) + make_property(integer(), :sint32) + make_property(large_integer(), :sint64) - make_property(boolean(), :bool, 0) + make_property(boolean(), :bool) - make_property(natural_number(), :fixed64, 1) - make_property(large_integer(), :sfixed64, 1) + make_property(natural_number(), :fixed64) + make_property(large_integer(), :sfixed64) - make_canonical_property(float(), :double, 1) - make_canonical_property(float(), :float, 5) + make_canonical_property(float(), :double) + make_canonical_property(float(), :float) end From 9881bf9d002151dc42269aa58c3062b330d74823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Britto?= Date: Thu, 9 Jul 2020 03:01:59 -0300 Subject: [PATCH 3/7] Optimize decoding of varints on single, packed and length-delimited fields --- lib/protobuf/decoder.ex | 56 ++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/lib/protobuf/decoder.ex b/lib/protobuf/decoder.ex index b0739c37..dbfb72c6 100644 --- a/lib/protobuf/decoder.ex +++ b/lib/protobuf/decoder.ex @@ -238,53 +238,38 @@ defmodule Protobuf.Decoder do ) end - defp raw_handle_key(wire_type, tag, [], <>, result) do - raw_decode_value(wire_type, bin, [wire_type, tag | result], []) - end - - defp raw_handle_key(wire_type, _tag, groups, <>, result) do - raw_decode_value(wire_type, bin, result, groups) - end - - decoder :defp, :raw_decode_varint, [:result, :type, :groups] do - raw_handle_varint(type, rest, result, value, groups) - end - - defp raw_handle_varint(:value, <<>>, result, val, []), do: Enum.reverse([val | result]) - - defp raw_handle_varint(:value, <>, result, val, []) do - raw_decode_key(bin, [val | result], []) - end - - defp raw_handle_varint(:value, <>, result, _val, groups) do - raw_decode_key(bin, result, groups) - end - - defp raw_handle_varint(:bytes_len, <>, result, len, []) do - <> = bin - raw_decode_key(rest, [bytes | result], []) + defp raw_handle_key(wire_type, tag, groups, <>, result) do + case groups do + [] -> raw_decode_value(wire_type, bin, [wire_type, tag | result], groups) + _ -> raw_decode_value(wire_type, bin, result, groups) + end end - defp raw_handle_varint(:bytes_len, <>, result, len, groups) do - <<_bytes::bytes-size(len), rest::bits>> = bin - raw_decode_key(rest, result, groups) + decoder :defp, :raw_decode_varint, [:result, :groups] do + case groups do + [] -> raw_decode_key(rest, [value | result], groups) + _ -> raw_decode_key(rest, result, groups) + end end - defp raw_handle_varint(:packed, <<>>, result, val, _groups), do: [val | result] + decoder :defp, :raw_decode_delimited, [:result, :groups] do + <> = rest - defp raw_handle_varint(:packed, <>, result, val, groups) do - raw_decode_varint(bin, [val | result], :packed, groups) + case groups do + [] -> raw_decode_key(rest, [bytes | result], groups) + _ -> raw_decode_key(rest, result, groups) + end end @doc false def raw_decode_value(wire, bin, result, groups \\ []) def raw_decode_value(wire_varint(), <>, result, groups) do - raw_decode_varint(bin, result, :value, groups) + raw_decode_varint(bin, result, groups) end def raw_decode_value(wire_delimited(), <>, result, groups) do - raw_decode_varint(bin, result, :bytes_len, groups) + raw_decode_delimited(bin, result, groups) end def raw_decode_value(wire_32bits(), <>, result, []) do @@ -317,7 +302,7 @@ defmodule Protobuf.Decoder do value = case wire_type do - wire_varint() -> raw_decode_varint(bin, acc, :packed, []) + wire_varint() -> decode_varints(bin, acc) wire_32bits() -> decode_fixed32(bin, type, key, acc) wire_64bits() -> decode_fixed64(bin, type, key, acc) end @@ -325,6 +310,9 @@ defmodule Protobuf.Decoder do Map.put(msg, key, value) end + defp decode_varints(<<>>, acc), do: acc + decoder :defp, :decode_varints, [:acc], do: decode_varints(rest, [value | acc]) + @dialyzer {:nowarn_function, decode_fixed32: 4} defp decode_fixed32(<>, type, key, acc) do decode_fixed32(bin, type, key, [decode_type_m(type, key, n) | acc]) From 7b93c1ed9013c4c26a085b432d0994b5a52bbabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Britto?= Date: Thu, 9 Jul 2020 02:26:57 -0300 Subject: [PATCH 4/7] Extract wire format encoding/decoding concerns into their own module --- lib/protobuf/decoder.ex | 155 ++------- lib/protobuf/encoder.ex | 58 +--- lib/protobuf/wire.ex | 136 ++++++++ lib/protobuf/wire/zigzag.ex | 13 + test/protobuf/decode/decode_type_test.exs | 123 ------- test/protobuf/decoder_test.exs | 5 +- test/protobuf/encoder/encode_type_test.exs | 226 ------------ test/protobuf/encoder_validation_test.exs | 4 +- test/protobuf/wire_test.exs | 382 +++++++++++++++++++++ 9 files changed, 559 insertions(+), 543 deletions(-) create mode 100644 lib/protobuf/wire.ex create mode 100644 lib/protobuf/wire/zigzag.ex delete mode 100644 test/protobuf/decode/decode_type_test.exs delete mode 100644 test/protobuf/encoder/encode_type_test.exs create mode 100644 test/protobuf/wire_test.exs diff --git a/lib/protobuf/decoder.ex b/lib/protobuf/decoder.ex index dbfb72c6..9cb123ab 100644 --- a/lib/protobuf/decoder.ex +++ b/lib/protobuf/decoder.ex @@ -4,7 +4,7 @@ defmodule Protobuf.Decoder do import Protobuf.{WireTypes, Wire.Varint} import Bitwise, only: [bsr: 2, band: 2] - alias Protobuf.DecodeError + alias Protobuf.{DecodeError, Wire} require Logger @@ -16,115 +16,6 @@ defmodule Protobuf.Decoder do reverse_repeated(struct, msg_props.repeated_fields) end - @doc false - # For performance - defmacro decode_type_m(type, key, val) do - quote do - case unquote(type) do - :int32 -> - <> = <> - n - - :string -> - unquote(val) - - :bytes -> - unquote(val) - - :int64 -> - <> = <> - n - - :uint32 -> - <> = <> - n - - :uint64 -> - <> = <> - n - - :bool -> - unquote(val) != 0 - - :sint32 -> - decode_zigzag(unquote(val)) - - :sint64 -> - decode_zigzag(unquote(val)) - - :fixed64 -> - <> = unquote(val) - n - - :sfixed64 -> - <> = unquote(val) - n - - :double -> - case unquote(val) do - <> -> - n - - # little endianness - # should be 0b0_11111111111_000000000... - # should be 0b1_11111111111_000000000... - <<0, 0, 0, 0, 0, 0, 0b1111::4, 0::4, 0b01111111::8>> -> - :infinity - - <<0, 0, 0, 0, 0, 0, 0b1111::4, 0::4, 0b11111111::8>> -> - :negative_infinity - - <> when a != 0 or b != 0 -> - :nan - end - - :fixed32 -> - <> = unquote(val) - n - - :sfixed32 -> - <> = unquote(val) - n - - :float -> - case unquote(val) do - <> -> - n - - # little endianness - # should be 0b0_11111111_000000000... - <<0, 0, 0b1000_0000::8, 0b01111111::8>> -> - :infinity - - # little endianness - # should be 0b1_11111111_000000000... - <<0, 0, 0b1000_0000::8, 0b11111111::8>> -> - :negative_infinity - - # should be 0b*_11111111_not_zero... - <> when a != 0 or b != 0 -> - :nan - end - - {:enum, enum_type} -> - try do - enum_type.key(unquote(val)) - rescue - FunctionClauseError -> - Logger.warn( - "unknown enum value #{unquote(val)} when decoding for #{inspect(unquote(type))}" - ) - - unquote(val) - end - - _ -> - raise DecodeError, - message: "can't decode type #{unquote(type)} for field #{unquote(key)}" - end - end - end - defp build_struct([tag, wire, val | rest], %{field_props: f_props} = msg_props, struct) do case f_props do %{ @@ -151,7 +42,7 @@ defmodule Protobuf.Decoder do Map.put(struct, key, val) else - val = decode_type_m(type, key, val) + val = Wire.to_proto(type, val) val = if oneof, do: {name_atom, val}, else: val val = @@ -272,19 +163,19 @@ defmodule Protobuf.Decoder do raw_decode_delimited(bin, result, groups) end - def raw_decode_value(wire_32bits(), <>, result, []) do - raw_decode_key(rest, [<> | result], []) + def raw_decode_value(wire_32bits(), <>, result, []) do + raw_decode_key(rest, [n | result], []) end - def raw_decode_value(wire_32bits(), <<_n::32, rest::bits>>, result, groups) do + def raw_decode_value(wire_32bits(), <<_n::bits-32, rest::bits>>, result, groups) do raw_decode_key(rest, result, groups) end - def raw_decode_value(wire_64bits(), <>, result, []) do - raw_decode_key(rest, [<> | result], []) + def raw_decode_value(wire_64bits(), <>, result, []) do + raw_decode_key(rest, [n | result], []) end - def raw_decode_value(wire_64bits(), <<_n::64, rest::bits>>, result, groups) do + def raw_decode_value(wire_64bits(), <<_n::bits-64, rest::bits>>, result, groups) do raw_decode_key(rest, result, groups) end @@ -303,34 +194,30 @@ defmodule Protobuf.Decoder do value = case wire_type do wire_varint() -> decode_varints(bin, acc) - wire_32bits() -> decode_fixed32(bin, type, key, acc) - wire_64bits() -> decode_fixed64(bin, type, key, acc) + wire_32bits() -> decode_fixed32(bin, type, acc) + wire_64bits() -> decode_fixed64(bin, type, acc) end Map.put(msg, key, value) end defp decode_varints(<<>>, acc), do: acc - decoder :defp, :decode_varints, [:acc], do: decode_varints(rest, [value | acc]) - @dialyzer {:nowarn_function, decode_fixed32: 4} - defp decode_fixed32(<>, type, key, acc) do - decode_fixed32(bin, type, key, [decode_type_m(type, key, n) | acc]) + decoder :defp, :decode_varints, [:acc] do + decode_varints(rest, [value | acc]) end - defp decode_fixed32(<<>>, _, _, acc), do: acc - - @dialyzer {:nowarn_function, decode_fixed64: 4} - defp decode_fixed64(<>, type, key, acc) do - decode_fixed64(bin, type, key, [decode_type_m(type, key, n) | acc]) + defp decode_fixed32(<>, type, acc) do + decode_fixed32(bin, type, [Wire.to_proto(type, n) | acc]) end - defp decode_fixed64(<<>>, _, _, acc), do: acc + defp decode_fixed32(<<>>, _, acc), do: acc - @doc false - @spec decode_zigzag(integer) :: integer - def decode_zigzag(n) when band(n, 1) == 0, do: bsr(n, 1) - def decode_zigzag(n) when band(n, 1) == 1, do: -bsr(n + 1, 1) + defp decode_fixed64(<>, type, acc) do + decode_fixed64(bin, type, [Wire.to_proto(type, n) | acc]) + end + + defp decode_fixed64(<<>>, _, acc), do: acc defp prop_display(prop) do prop.name @@ -370,7 +257,7 @@ defmodule Protobuf.Decoder do embedded_msg = decode(val, type) merge_embedded_value(struct, name_atom, embedded_msg, is_repeated) else - val = decode_type_m(type, name_atom, val) + val = Wire.to_proto(type, val) if is_repeated do merge_simple_repeated_value(struct, name_atom, val) diff --git a/lib/protobuf/encoder.ex b/lib/protobuf/encoder.ex index 99dd1b3f..90959b27 100644 --- a/lib/protobuf/encoder.ex +++ b/lib/protobuf/encoder.ex @@ -3,7 +3,7 @@ defmodule Protobuf.Encoder do import Protobuf.WireTypes import Bitwise, only: [bsl: 2, bor: 2] - alias Protobuf.{FieldProps, MessageProps, Wire.Varint} + alias Protobuf.{FieldProps, MessageProps, Wire, Wire.Varint} @spec encode(atom, map | struct, keyword) :: iodata def encode(mod, msg, opts) do @@ -98,7 +98,7 @@ defmodule Protobuf.Encoder do @spec encode_field(atom, any, FieldProps.t()) :: iodata defp encode_field(:normal, val, %{encoded_fnum: fnum, type: type, repeated?: is_repeated}) do repeated_or_not(val, is_repeated, fn v -> - [fnum | encode_type(type, v)] + [fnum | Wire.from_proto(type, v)] end) end @@ -119,7 +119,7 @@ defmodule Protobuf.Encoder do end defp encode_field(:packed, val, %{type: type, encoded_fnum: fnum}) do - encoded = Enum.map(val, fn v -> encode_type(type, v) end) + encoded = Enum.map(val, fn v -> Wire.from_proto(type, v) end) byte_size = IO.iodata_length(encoded) [fnum | Varint.encode(byte_size)] ++ encoded end @@ -147,58 +147,6 @@ defmodule Protobuf.Encoder do |> IO.iodata_to_binary() end - @doc false - @spec encode_type(atom, any) :: iodata - def encode_type(:int32, n) when n >= -0x80000000 and n <= 0x7FFFFFFF, do: Varint.encode(n) - - def encode_type(:int64, n) when n >= -0x8000000000000000 and n <= 0x7FFFFFFFFFFFFFFF, - do: Varint.encode(n) - - def encode_type(:string, n), do: encode_type(:bytes, n) - def encode_type(:uint32, n) when n >= 0 and n <= 0xFFFFFFFF, do: Varint.encode(n) - def encode_type(:uint64, n) when n >= 0 and n <= 0xFFFFFFFFFFFFFFFF, do: Varint.encode(n) - def encode_type(:bool, true), do: Varint.encode(1) - def encode_type(:bool, false), do: Varint.encode(0) - def encode_type({:enum, type}, n) when is_atom(n), do: n |> type.value() |> Varint.encode() - def encode_type({:enum, _}, n), do: Varint.encode(n) - def encode_type(:float, :infinity), do: [0, 0, 128, 127] - def encode_type(:float, :negative_infinity), do: [0, 0, 128, 255] - def encode_type(:float, :nan), do: [0, 0, 192, 127] - def encode_type(:float, n), do: <> - def encode_type(:double, :infinity), do: [0, 0, 0, 0, 0, 0, 240, 127] - def encode_type(:double, :negative_infinity), do: [0, 0, 0, 0, 0, 0, 240, 255] - def encode_type(:double, :nan), do: [1, 0, 0, 0, 0, 0, 248, 127] - def encode_type(:double, n), do: <> - - def encode_type(:bytes, n) do - len = n |> IO.iodata_length() |> Varint.encode() - len ++ n - end - - def encode_type(:sint32, n) when n >= -0x80000000 and n <= 0x7FFFFFFF, - do: n |> encode_zigzag() |> Varint.encode() - - def encode_type(:sint64, n) when n >= -0x8000000000000000 and n <= 0x7FFFFFFFFFFFFFFF, - do: n |> encode_zigzag() |> Varint.encode() - - def encode_type(:fixed64, n) when n >= 0 and n <= 0xFFFFFFFFFFFFFFFF, do: <> - - def encode_type(:sfixed64, n) when n >= -0x8000000000000000 and n <= 0x7FFFFFFFFFFFFFFF, - do: <> - - def encode_type(:fixed32, n) when n >= 0 and n <= 0xFFFFFFFF, do: <> - - def encode_type(:sfixed32, n) when n >= -0x80000000 and n <= 0x7FFFFFFF, - do: <> - - def encode_type(type, n) do - raise Protobuf.TypeEncodeError, message: "#{inspect(n)} is invalid for type #{type}" - end - - @spec encode_zigzag(integer) :: integer - defp encode_zigzag(val) when val >= 0, do: val * 2 - defp encode_zigzag(val) when val < 0, do: val * -2 - 1 - @doc false @spec wire_type(atom) :: integer def wire_type(:int32), do: wire_varint() diff --git a/lib/protobuf/wire.ex b/lib/protobuf/wire.ex new file mode 100644 index 00000000..6d614d08 --- /dev/null +++ b/lib/protobuf/wire.ex @@ -0,0 +1,136 @@ +defmodule Protobuf.Wire do + @moduledoc """ + Utilities to convert data from wire format to protobuf and back. + """ + + alias Protobuf.Wire.{Varint, Zigzag} + + require Logger + + @type proto_type :: + :int32 + | :int64 + | :fixed32 + | :fixed64 + | :uint32 + | :uint64 + | :sfixed32 + | :sfixed64 + | :sint32 + | :sint64 + | :float + | :double + | :bool + | :string + | :bytes + | {:enum, any} + + @type proto_float :: :infinity | :negative_infinity | :nan | float + + @type proto_value :: binary | integer | bool | proto_float | atom + + @sint32_range -0x80000000..0x7FFFFFFF + @sint64_range -0x8000000000000000..0x7FFFFFFFFFFFFFFF + @uint32_range 0..0xFFFFFFFF + @uint64_range 0..0xFFFFFFFFFFFFFFFF + + @spec from_proto(proto_type, proto_value) :: iodata + # Returns improper list, but still valid iodata. + def from_proto(type, binary) when type in [:string, :bytes] do + len = binary |> IO.iodata_length() |> Varint.encode() + len ++ binary + end + + def from_proto(:int32, n) when n in @sint32_range, do: Varint.encode(n) + def from_proto(:int64, n) when n in @sint64_range, do: Varint.encode(n) + def from_proto(:uint32, n) when n in @uint32_range, do: Varint.encode(n) + def from_proto(:uint64, n) when n in @uint64_range, do: Varint.encode(n) + + def from_proto(:bool, true), do: Varint.encode(1) + def from_proto(:bool, false), do: Varint.encode(0) + + def from_proto({:enum, enum}, key) when is_atom(key), do: Varint.encode(enum.value(key)) + def from_proto({:enum, _}, n) when is_integer(n), do: Varint.encode(n) + + def from_proto(:float, :infinity), do: [0, 0, 128, 127] + def from_proto(:float, :negative_infinity), do: [0, 0, 128, 255] + def from_proto(:float, :nan), do: [0, 0, 192, 127] + def from_proto(:float, n), do: <> + + def from_proto(:double, :infinity), do: [0, 0, 0, 0, 0, 0, 240, 127] + def from_proto(:double, :negative_infinity), do: [0, 0, 0, 0, 0, 0, 240, 255] + def from_proto(:double, :nan), do: [1, 0, 0, 0, 0, 0, 248, 127] + def from_proto(:double, n), do: <> + + def from_proto(:sint32, n) when n in @sint32_range, do: Varint.encode(Zigzag.encode(n)) + def from_proto(:sint64, n) when n in @sint64_range, do: Varint.encode(Zigzag.encode(n)) + def from_proto(:fixed32, n) when n in @uint32_range, do: <> + def from_proto(:fixed64, n) when n in @uint64_range, do: <> + def from_proto(:sfixed32, n) when n in @sint32_range, do: <> + def from_proto(:sfixed64, n) when n in @sint64_range, do: <> + + def from_proto(type, n) do + raise Protobuf.TypeEncodeError, message: "#{inspect(n)} is invalid for type #{type}" + end + + @spec to_proto(proto_type, binary | integer) :: proto_value + def to_proto(type, val) when type in [:string, :bytes], do: val + + def to_proto(:int32, val) do + <> = <> + n + end + + def to_proto(:int64, val) do + <> = <> + n + end + + def to_proto(:uint32, val) do + <> = <> + n + end + + def to_proto(:uint64, val) do + <> = <> + n + end + + def to_proto(:bool, val), do: val != 0 + + def to_proto({:enum, enum}, val) do + enum.key(val) + rescue + FunctionClauseError -> + Logger.warn("unknown enum value #{val} when decoding for #{inspect(enum)}") + val + end + + def to_proto(:float, <>), do: n + # little endianness, should be 0b0_11111111_000000000... + def to_proto(:float, <<0, 0, 0b1000_0000::8, 0b01111111::8>>), do: :infinity + # little endianness, should be 0b1_11111111_000000000... + def to_proto(:float, <<0, 0, 0b1000_0000::8, 0b11111111::8>>), do: :negative_infinity + # should be 0b*_11111111_not_zero... + def to_proto(:float, <>) when a != 0 or b != 0, + do: :nan + + def to_proto(:double, <>), do: n + # little endianness, should be 0b0_11111111111_000000000... + def to_proto(:double, <<0::48, 0b1111::4, 0::4, 0b01111111::8>>), do: :infinity + # little endianness, should be 0b1_11111111111_000000000... + def to_proto(:double, <<0::48, 0b1111::4, 0::4, 0b11111111::8>>), do: :negative_infinity + + def to_proto(:double, <>) when a != 0 or b != 0, + do: :nan + + def to_proto(type, val) when type in [:sint32, :sint64], do: Zigzag.decode(val) + def to_proto(:fixed32, <>), do: n + def to_proto(:fixed64, <>), do: n + def to_proto(:sfixed32, <>), do: n + def to_proto(:sfixed64, <>), do: n + + def to_proto(type, val) do + raise Protobuf.DecodeError, message: "can't decode #{inspect(val)} into type #{type}" + end +end diff --git a/lib/protobuf/wire/zigzag.ex b/lib/protobuf/wire/zigzag.ex new file mode 100644 index 00000000..53932c64 --- /dev/null +++ b/lib/protobuf/wire/zigzag.ex @@ -0,0 +1,13 @@ +defmodule Protobuf.Wire.Zigzag do + @moduledoc false + + use Bitwise, skip_operators: true + + @spec encode(integer) :: integer + def encode(n) when n >= 0, do: n * 2 + def encode(n) when n < 0, do: n * -2 - 1 + + @spec decode(integer) :: integer + def decode(n) when band(n, 1) == 0, do: bsr(n, 1) + def decode(n) when band(n, 1) == 1, do: -bsr(n + 1, 1) +end diff --git a/test/protobuf/decode/decode_type_test.exs b/test/protobuf/decode/decode_type_test.exs deleted file mode 100644 index 8a01f254..00000000 --- a/test/protobuf/decode/decode_type_test.exs +++ /dev/null @@ -1,123 +0,0 @@ -defmodule Protobuf.Decoder.DecodeTypeTest do - use ExUnit.Case, async: true - - require Logger - import Protobuf.Decoder - - def decode_type(type, val) do - decode_type_m(type, :fake_key, val) - end - - test "decode_type/3 varint" do - assert 42 == decode_type(:int32, 42) - end - - test "decode_type/3 int64" do - assert -1 == decode_type(:int64, -1) - end - - test "decode_type/3 min int32" do - assert -2_147_483_648 == decode_type(:int32, 18_446_744_071_562_067_968) - end - - test "decode_type/3 max int32" do - assert -2_147_483_647 == decode_type(:int32, 18_446_744_071_562_067_969) - end - - test "decode_type/3 min int64" do - assert -9_223_372_036_854_775_808 == decode_type(:int64, 9_223_372_036_854_775_808) - end - - test "decode_type/3 max int64" do - assert 9_223_372_036_854_775_807 == decode_type(:int64, 9_223_372_036_854_775_807) - end - - test "decode_type/3 min sint32" do - assert -2_147_483_648 == decode_type(:sint32, 4_294_967_295) - end - - test "decode_type/3 max sint32" do - assert 2_147_483_647 == decode_type(:sint32, 4_294_967_294) - end - - test "decode_type/3 min sint64" do - assert -9_223_372_036_854_775_808 == decode_type(:sint64, 18_446_744_073_709_551_615) - end - - test "decode_type/3 max sint64" do - assert 9_223_372_036_854_775_807 == decode_type(:sint64, 18_446_744_073_709_551_614) - end - - test "decode_type/3 max uint32" do - assert 4_294_967_295 == decode_type(:uint32, 4_294_967_295) - end - - test "decode_type/3 max uint64" do - assert 9_223_372_036_854_775_807 == decode_type(:uint64, 9_223_372_036_854_775_807) - end - - test "decode_type/3 bool works" do - assert true == decode_type(:bool, 1) - assert false == decode_type(:bool, 0) - end - - test "decode_type/3 a fixed64" do - assert 8_446_744_073_709_551_615 == - decode_type(:fixed64, <<255, 255, 23, 118, 251, 220, 56, 117>>) - end - - test "decode_type/3 max fixed64" do - assert 18_446_744_073_709_551_615 == - decode_type(:fixed64, <<255, 255, 255, 255, 255, 255, 255, 255>>) - end - - test "decode_type/3 min sfixed64" do - assert -9_223_372_036_854_775_808 == decode_type(:sfixed64, <<0, 0, 0, 0, 0, 0, 0, 128>>) - end - - test "decode_type/3 max sfixed64" do - assert 9_223_372_036_854_775_807 == - decode_type(:sfixed64, <<255, 255, 255, 255, 255, 255, 255, 127>>) - end - - test "decode_type/3 min double" do - assert 5.0e-324 == decode_type(:double, <<1, 0, 0, 0, 0, 0, 0, 0>>) - end - - test "decode_type/3 max double" do - assert 1.7976931348623157e308 == - decode_type(:double, <<255, 255, 255, 255, 255, 255, 239, 127>>) - end - - test "decode_type/3 string" do - assert "testing" == decode_type(:string, <<116, 101, 115, 116, 105, 110, 103>>) - end - - test "decode_type/3 bytes" do - assert <<42, 43, 44, 45>> == decode_type(:bytes, <<42, 43, 44, 45>>) - end - - test "decode_type/3 fixed32" do - assert 4_294_967_295 == decode_type(:fixed32, <<255, 255, 255, 255>>) - end - - test "decode_type/3 sfixed32" do - assert 2_147_483_647 == decode_type(:sfixed32, <<255, 255, 255, 127>>) - end - - test "decode_type/3 float" do - assert 3.4028234663852886e38 == decode_type(:float, <<255, 255, 127, 127>>) - end - - test "decode_type/3 float infinity,-infinity,nan" do - assert :infinity == decode_type(:float, <<0, 0, 128, 127>>) - assert :negative_infinity == decode_type(:float, <<0, 0, 128, 255>>) - assert :nan == decode_type(:float, <<0, 0, 192, 127>>) - end - - test "decode_type/3 double infinity,-infinity,nan" do - assert :infinity == decode_type(:double, <<0, 0, 0, 0, 0, 0, 240, 127>>) - assert :negative_infinity == decode_type(:double, <<0, 0, 0, 0, 0, 0, 240, 255>>) - assert :nan == decode_type(:double, <<1, 0, 0, 0, 0, 0, 248, 127>>) - end -end diff --git a/test/protobuf/decoder_test.exs b/test/protobuf/decoder_test.exs index 4d729359..00ba7da5 100644 --- a/test/protobuf/decoder_test.exs +++ b/test/protobuf/decoder_test.exs @@ -80,15 +80,16 @@ defmodule Protobuf.DecoderTest do test "decodes enum type" do struct = Decoder.decode(<<88, 1>>, TestMsg.Foo) assert struct == TestMsg.Foo.new(j: :A) + struct = Decoder.decode(<<88, 2>>, TestMsg.Foo) assert struct == TestMsg.Foo.new(j: :B) end - test "decodes unknown enum type" do + test "decodes unknown enum value" do assert ExUnit.CaptureLog.capture_log(fn -> struct = Decoder.decode(<<88, 3>>, TestMsg.Foo) assert struct == TestMsg.Foo.new(j: 3) - end) =~ ~r/unknown enum value 3 when decoding for {:enum, TestMsg.EnumFoo}/ + end) =~ ~r/unknown enum value 3 when decoding for TestMsg\.EnumFoo/ end test "decodes map type" do diff --git a/test/protobuf/encoder/encode_type_test.exs b/test/protobuf/encoder/encode_type_test.exs deleted file mode 100644 index 6261e865..00000000 --- a/test/protobuf/encoder/encode_type_test.exs +++ /dev/null @@ -1,226 +0,0 @@ -defmodule Protobuf.Encoder.DecodeTypeTest do - use ExUnit.Case, async: true - - alias Protobuf.Encoder - alias Protobuf.Decoder - require Logger - import Protobuf.Decoder - - test "encode_type/2 varint" do - assert encode(:int32, 42) == <<42>> - end - - test "encode_type/2 min int32" do - assert encode(:int32, -2_147_483_648) == - <<128, 128, 128, 128, 248, 255, 255, 255, 255, 1>> - end - - test "encode_type/2 min int64" do - assert encode(:int64, -9_223_372_036_854_775_808) == - <<128, 128, 128, 128, 128, 128, 128, 128, 128, 1>> - end - - test "encode_type/3 min sint32" do - assert encode(:sint32, -2_147_483_648) == <<255, 255, 255, 255, 15>> - end - - test "encode_type/3 max sint32" do - assert encode(:sint32, 2_147_483_647) == <<254, 255, 255, 255, 15>> - end - - test "encode_type/3 min sint64" do - assert encode(:sint64, -9_223_372_036_854_775_808) == - <<255, 255, 255, 255, 255, 255, 255, 255, 255, 1>> - end - - test "encode_type/3 max sint64" do - assert encode(:sint64, 9_223_372_036_854_775_807) == - <<254, 255, 255, 255, 255, 255, 255, 255, 255, 1>> - end - - test "encode_type/3 bool false" do - assert encode(:bool, false) == <<0>> - end - - test "encode_type/3 bool true" do - assert encode(:bool, true) == <<1>> - end - - test "encode_type/3 a fixed64" do - assert encode(:fixed64, 8_446_744_073_709_551_615) == - <<255, 255, 23, 118, 251, 220, 56, 117>> - end - - test "encode_type/3 max fixed64" do - assert encode(:fixed64, 18_446_744_073_709_551_615) == - <<255, 255, 255, 255, 255, 255, 255, 255>> - end - - test "encode_type/3 min sfixed64" do - assert encode(:sfixed64, -9_223_372_036_854_775_808) == - <<0, 0, 0, 0, 0, 0, 0, 128>> - end - - test "encode_type/3 max sfixed64" do - assert encode(:sfixed64, 9_223_372_036_854_775_807) == - <<255, 255, 255, 255, 255, 255, 255, 127>> - end - - test "encode_type/3 min double" do - assert encode(:double, 5.0e-324) == <<1, 0, 0, 0, 0, 0, 0, 0>> - end - - test "encode_type/3 max double" do - assert encode(:double, 1.7976931348623157e308) == - <<255, 255, 255, 255, 255, 255, 239, 127>> - end - - test "encode_type/3 int as double" do - assert encode(:double, -9_223_372_036_854_775_808) == - <<0, 0, 0, 0, 0, 0, 224, 195>> - end - - test "encode_type/3 string" do - assert encode(:string, "testing") == <<7, 116, 101, 115, 116, 105, 110, 103>> - end - - test "encode_type/3 bytes" do - assert encode(:bytes, <<42, 43, 44, 45>>) == <<4, 42, 43, 44, 45>> - end - - test "encode_type/3 fixed32" do - assert encode(:fixed32, 4_294_967_295) == <<255, 255, 255, 255>> - end - - test "encode_type/3 sfixed32" do - assert encode(:sfixed32, 2_147_483_647) == <<255, 255, 255, 127>> - end - - test "encode_type/3 float" do - assert encode(:float, 3.4028234663852886e38) == <<255, 255, 127, 127>> - end - - test "encode_type/3 int as float" do - assert encode(:float, 3) == <<0, 0, 64, 64>> - end - - test "encode_type/3 float infinity/-infinity/nan" do - Enum.each([:infinity, :negative_infinity, :nan], fn f -> - bin = encode(:float, f) - assert f == Decoder.decode_type_m(:float, :fake, bin) - end) - end - - test "encode_type/3 double infinity/-infinity/nan" do - Enum.each([:infinity, :negative_infinity, :nan], fn f -> - bin = encode(:double, f) - assert f == Decoder.decode_type_m(:double, :fake, bin) - end) - end - - test "encode_type/2 wrong uint32" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:uint32, 12_345_678_901_234_567_890) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:uint32, -1) - end - end - - test "encode_type/2 wrong uint64" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:uint64, 184_467_440_737_095_516_150) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:uint64, -1) - end - end - - test "encode_type/2 wrong fixed32" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:fixed32, 12_345_678_901_234_567_890) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:fixed32, -1) - end - end - - test "encode_type/2 wrong fixed64" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:fixed64, 184_467_440_737_095_516_150) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:fixed64, -1) - end - end - - test "encode_type/2 wrong int32" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:int32, 2_147_483_648) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:int32, -2_147_483_649) - end - end - - test "encode_type/2 wrong int64" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:int64, 184_467_440_737_095_516_150) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:int64, -184_467_440_737_095_516_150) - end - end - - test "encode_type/2 wrong sint32" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:sint32, 2_147_483_648) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:sint32, -2_147_483_649) - end - end - - test "encode_type/2 wrong sint64" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:sint64, 184_467_440_737_095_516_150) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:sint64, -184_467_440_737_095_516_150) - end - end - - test "encode_type/2 wrong sfixed32" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:sfixed32, 2_147_483_648) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:sfixed32, -2_147_483_649) - end - end - - test "encode_type/2 wrong sfixed64" do - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:sfixed64, 184_467_440_737_095_516_150) - end - - assert_raise Protobuf.TypeEncodeError, fn -> - encode(:sfixed64, -184_467_440_737_095_516_150) - end - end - - defp encode(type, value) do - type - |> Encoder.encode_type(value) - |> IO.iodata_to_binary() - end -end diff --git a/test/protobuf/encoder_validation_test.exs b/test/protobuf/encoder_validation_test.exs index bdc124c2..160af8ea 100644 --- a/test/protobuf/encoder_validation_test.exs +++ b/test/protobuf/encoder_validation_test.exs @@ -1,8 +1,6 @@ defmodule Protobuf.EncoderTest.Validation do use ExUnit.Case, async: true - import Protobuf.Encoder - @valid_vals %{ int32: -32, int64: -64, @@ -33,7 +31,7 @@ defmodule Protobuf.EncoderTest.Validation do assert_invalid = fn type, others -> Enum.each(other_types(others), fn {invalid, err_type} -> assert_raise err_type, fn -> - encode_type(type, invalid) + Protobuf.Wire.from_proto(type, invalid) end end) end diff --git a/test/protobuf/wire_test.exs b/test/protobuf/wire_test.exs new file mode 100644 index 00000000..3710f764 --- /dev/null +++ b/test/protobuf/wire_test.exs @@ -0,0 +1,382 @@ +defmodule Protobuf.WireTest do + use ExUnit.Case, async: true + + alias Protobuf.Wire + + describe "from_proto/2" do + test "varint" do + assert encode(:int32, 42) == <<42>> + end + + test "min int32" do + assert encode(:int32, -2_147_483_648) == + <<128, 128, 128, 128, 248, 255, 255, 255, 255, 1>> + end + + test "min int64" do + assert encode(:int64, -9_223_372_036_854_775_808) == + <<128, 128, 128, 128, 128, 128, 128, 128, 128, 1>> + end + + test "min sint32" do + assert encode(:sint32, -2_147_483_648) == <<255, 255, 255, 255, 15>> + end + + test "max sint32" do + assert encode(:sint32, 2_147_483_647) == <<254, 255, 255, 255, 15>> + end + + test "min sint64" do + assert encode(:sint64, -9_223_372_036_854_775_808) == + <<255, 255, 255, 255, 255, 255, 255, 255, 255, 1>> + end + + test "max sint64" do + assert encode(:sint64, 9_223_372_036_854_775_807) == + <<254, 255, 255, 255, 255, 255, 255, 255, 255, 1>> + end + + test "bool false" do + assert encode(:bool, false) == <<0>> + end + + test "bool true" do + assert encode(:bool, true) == <<1>> + end + + test "enum atom and alias" do + assert encode({:enum, TestMsg.EnumFoo}, :C) == <<4>> + assert encode({:enum, TestMsg.EnumFoo}, :D) == <<4>> + end + + test "enum known and unknown integer" do + assert encode({:enum, TestMsg.EnumFoo}, 1) == <<1>> + assert encode({:enum, TestMsg.EnumFoo}, 5) == <<5>> + end + + test "a fixed64" do + assert encode(:fixed64, 8_446_744_073_709_551_615) == + <<255, 255, 23, 118, 251, 220, 56, 117>> + end + + test "max fixed64" do + assert encode(:fixed64, 18_446_744_073_709_551_615) == + <<255, 255, 255, 255, 255, 255, 255, 255>> + end + + test "min sfixed64" do + assert encode(:sfixed64, -9_223_372_036_854_775_808) == + <<0, 0, 0, 0, 0, 0, 0, 128>> + end + + test "max sfixed64" do + assert encode(:sfixed64, 9_223_372_036_854_775_807) == + <<255, 255, 255, 255, 255, 255, 255, 127>> + end + + test "min double" do + assert encode(:double, 5.0e-324) == <<1, 0, 0, 0, 0, 0, 0, 0>> + end + + test "max double" do + assert encode(:double, 1.7976931348623157e308) == <<255, 255, 255, 255, 255, 255, 239, 127>> + end + + test "int as double" do + assert encode(:double, -9_223_372_036_854_775_808) == <<0, 0, 0, 0, 0, 0, 224, 195>> + end + + test "string" do + assert encode(:string, "testing") == <<7, 116, 101, 115, 116, 105, 110, 103>> + end + + test "bytes" do + assert encode(:bytes, <<42, 43, 44, 45>>) == <<4, 42, 43, 44, 45>> + end + + test "fixed32" do + assert encode(:fixed32, 4_294_967_295) == <<255, 255, 255, 255>> + end + + test "sfixed32" do + assert encode(:sfixed32, 2_147_483_647) == <<255, 255, 255, 127>> + end + + test "float" do + assert encode(:float, 3.4028234663852886e38) == <<255, 255, 127, 127>> + end + + test "int as float" do + assert encode(:float, 3) == <<0, 0, 64, 64>> + end + + test "float infinity/-infinity/nan" do + Enum.each([:infinity, :negative_infinity, :nan], fn f -> + bin = encode(:float, f) + assert f == Wire.to_proto(:float, bin) + end) + end + + test "double infinity, -infinity, nan" do + Enum.each([:infinity, :negative_infinity, :nan], fn f -> + bin = encode(:double, f) + assert f == Wire.to_proto(:double, bin) + end) + end + + test "wrong uint32" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:uint32, 12_345_678_901_234_567_890) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:uint32, -1) + end + end + + test "wrong uint64" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:uint64, 184_467_440_737_095_516_150) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:uint64, -1) + end + end + + test "wrong fixed32" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:fixed32, 12_345_678_901_234_567_890) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:fixed32, -1) + end + end + + test "wrong fixed64" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:fixed64, 184_467_440_737_095_516_150) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:fixed64, -1) + end + end + + test "wrong int32" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:int32, 2_147_483_648) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:int32, -2_147_483_649) + end + end + + test "wrong int64" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:int64, 184_467_440_737_095_516_150) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:int64, -184_467_440_737_095_516_150) + end + end + + test "wrong sint32" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:sint32, 2_147_483_648) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:sint32, -2_147_483_649) + end + end + + test "wrong sint64" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:sint64, 184_467_440_737_095_516_150) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:sint64, -184_467_440_737_095_516_150) + end + end + + test "wrong sfixed32" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:sfixed32, 2_147_483_648) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:sfixed32, -2_147_483_649) + end + end + + test "wrong sfixed64" do + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:sfixed64, 184_467_440_737_095_516_150) + end + + assert_raise Protobuf.TypeEncodeError, fn -> + encode(:sfixed64, -184_467_440_737_095_516_150) + end + end + + defp encode(type, value) do + type + |> Wire.from_proto(value) + |> IO.iodata_to_binary() + end + end + + describe "to_proto/2" do + test "varint" do + assert 42 == Wire.to_proto(:int32, 42) + end + + test "int64" do + assert -1 == Wire.to_proto(:int64, -1) + end + + test "min int32" do + assert -2_147_483_648 == Wire.to_proto(:int32, 18_446_744_071_562_067_968) + end + + test "max int32" do + assert -2_147_483_647 == Wire.to_proto(:int32, 18_446_744_071_562_067_969) + end + + test "min int64" do + assert -9_223_372_036_854_775_808 == Wire.to_proto(:int64, 9_223_372_036_854_775_808) + end + + test "max int64" do + assert 9_223_372_036_854_775_807 == Wire.to_proto(:int64, 9_223_372_036_854_775_807) + end + + test "min sint32" do + assert -2_147_483_648 == Wire.to_proto(:sint32, 4_294_967_295) + end + + test "max sint32" do + assert 2_147_483_647 == Wire.to_proto(:sint32, 4_294_967_294) + end + + test "min sint64" do + assert -9_223_372_036_854_775_808 == Wire.to_proto(:sint64, 18_446_744_073_709_551_615) + end + + test "max sint64" do + assert 9_223_372_036_854_775_807 == Wire.to_proto(:sint64, 18_446_744_073_709_551_614) + end + + test "max uint32" do + assert 4_294_967_295 == Wire.to_proto(:uint32, 4_294_967_295) + end + + test "max uint64" do + assert 9_223_372_036_854_775_807 == Wire.to_proto(:uint64, 9_223_372_036_854_775_807) + end + + test "bool works" do + assert true == Wire.to_proto(:bool, 1) + assert false == Wire.to_proto(:bool, 0) + end + + test "enum known and unknown integer" do + assert :A == Wire.to_proto({:enum, TestMsg.EnumFoo}, 1) + + assert ExUnit.CaptureLog.capture_log(fn -> + assert 5 == Wire.to_proto({:enum, TestMsg.EnumFoo}, 5) + end) =~ ~r/unknown enum value 5 when decoding for TestMsg\.EnumFoo/ + end + + test "a fixed64" do + assert 8_446_744_073_709_551_615 == + Wire.to_proto(:fixed64, <<255, 255, 23, 118, 251, 220, 56, 117>>) + end + + test "max fixed64" do + assert 18_446_744_073_709_551_615 == + Wire.to_proto(:fixed64, <<255, 255, 255, 255, 255, 255, 255, 255>>) + end + + test "min sfixed64" do + assert -9_223_372_036_854_775_808 == Wire.to_proto(:sfixed64, <<0, 0, 0, 0, 0, 0, 0, 128>>) + end + + test "max sfixed64" do + assert 9_223_372_036_854_775_807 == + Wire.to_proto(:sfixed64, <<255, 255, 255, 255, 255, 255, 255, 127>>) + end + + test "min double" do + assert 5.0e-324 == Wire.to_proto(:double, <<1, 0, 0, 0, 0, 0, 0, 0>>) + end + + test "max double" do + assert 1.7976931348623157e308 == + Wire.to_proto(:double, <<255, 255, 255, 255, 255, 255, 239, 127>>) + end + + test "string" do + assert "testing" == Wire.to_proto(:string, <<116, 101, 115, 116, 105, 110, 103>>) + end + + test "bytes" do + assert <<42, 43, 44, 45>> == Wire.to_proto(:bytes, <<42, 43, 44, 45>>) + end + + test "fixed32" do + assert 4_294_967_295 == Wire.to_proto(:fixed32, <<255, 255, 255, 255>>) + end + + test "sfixed32" do + assert 2_147_483_647 == Wire.to_proto(:sfixed32, <<255, 255, 255, 127>>) + end + + test "float" do + assert 3.4028234663852886e38 == Wire.to_proto(:float, <<255, 255, 127, 127>>) + end + + test "float infinity, -infinity, nan" do + assert :infinity == Wire.to_proto(:float, <<0, 0, 128, 127>>) + assert :negative_infinity == Wire.to_proto(:float, <<0, 0, 128, 255>>) + assert :nan == Wire.to_proto(:float, <<0, 0, 192, 127>>) + end + + test "double infinity, -infinity, nan" do + assert :infinity == Wire.to_proto(:double, <<0, 0, 0, 0, 0, 0, 240, 127>>) + assert :negative_infinity == Wire.to_proto(:double, <<0, 0, 0, 0, 0, 0, 240, 255>>) + assert :nan == Wire.to_proto(:double, <<1, 0, 0, 0, 0, 0, 248, 127>>) + end + + test "mismatching fixed-length sizes" do + msg = "can't decode <<0, 0, 0>> into type fixed32" + + assert_raise Protobuf.DecodeError, msg, fn -> + Wire.to_proto(:fixed32, <<0, 0, 0>>) + end + + msg = "can't decode <<0, 0, 0, 0, 0>> into type fixed32" + + assert_raise Protobuf.DecodeError, msg, fn -> + Wire.to_proto(:fixed32, <<0, 0, 0, 0, 0>>) + end + + msg = "can't decode <<0, 0, 0, 0, 0, 0, 0>> into type fixed64" + + assert_raise Protobuf.DecodeError, msg, fn -> + Wire.to_proto(:fixed64, <<0, 0, 0, 0, 0, 0, 0>>) + end + + msg = "can't decode <<0, 0, 0, 0, 0, 0, 0, 0, 0>> into type fixed64" + + assert_raise Protobuf.DecodeError, msg, fn -> + Wire.to_proto(:fixed64, <<0, 0, 0, 0, 0, 0, 0, 0, 0>>) + end + end + end +end From 83ef63f5cdfc2de892cf0d9611906dd1be9c57b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Britto?= Date: Mon, 6 Jul 2020 11:45:44 -0300 Subject: [PATCH 5/7] Rename WireTypes to Wire.Types --- lib/protobuf/decoder.ex | 2 +- lib/protobuf/encoder.ex | 2 +- lib/protobuf/{wire_types.ex => wire/types.ex} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename lib/protobuf/{wire_types.ex => wire/types.ex} (86%) diff --git a/lib/protobuf/decoder.ex b/lib/protobuf/decoder.ex index 9cb123ab..56377638 100644 --- a/lib/protobuf/decoder.ex +++ b/lib/protobuf/decoder.ex @@ -1,7 +1,7 @@ defmodule Protobuf.Decoder do @moduledoc false - import Protobuf.{WireTypes, Wire.Varint} + import Protobuf.{Wire.Types, Wire.Varint} import Bitwise, only: [bsr: 2, band: 2] alias Protobuf.{DecodeError, Wire} diff --git a/lib/protobuf/encoder.ex b/lib/protobuf/encoder.ex index 90959b27..dde67527 100644 --- a/lib/protobuf/encoder.ex +++ b/lib/protobuf/encoder.ex @@ -1,6 +1,6 @@ defmodule Protobuf.Encoder do @moduledoc false - import Protobuf.WireTypes + import Protobuf.Wire.Types import Bitwise, only: [bsl: 2, bor: 2] alias Protobuf.{FieldProps, MessageProps, Wire, Wire.Varint} diff --git a/lib/protobuf/wire_types.ex b/lib/protobuf/wire/types.ex similarity index 86% rename from lib/protobuf/wire_types.ex rename to lib/protobuf/wire/types.ex index 6a1f6be3..de778322 100644 --- a/lib/protobuf/wire_types.ex +++ b/lib/protobuf/wire/types.ex @@ -1,4 +1,4 @@ -defmodule Protobuf.WireTypes do +defmodule Protobuf.Wire.Types do @moduledoc false defmacro wire_varint, do: 0 From 62d706b9a1038ee89c0655836e979be4df0b283e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Britto?= Date: Fri, 10 Jul 2020 18:57:18 -0300 Subject: [PATCH 6/7] Refactor protobuf decoder This version improves group-skipping by avoiding unnecessary allocation of intermediate values and better separating that logic. It also reduces memory consumption by preventing the creation of an intermediate list with all read values. Now the message is updated as soon as values are read from the wire. --- lib/protobuf/decoder.ex | 341 +++++++++++++++++----------------------- 1 file changed, 144 insertions(+), 197 deletions(-) diff --git a/lib/protobuf/decoder.ex b/lib/protobuf/decoder.ex index 56377638..903e7f22 100644 --- a/lib/protobuf/decoder.ex +++ b/lib/protobuf/decoder.ex @@ -8,197 +8,187 @@ defmodule Protobuf.Decoder do require Logger - @spec decode(binary, atom) :: any - def decode(data, module) do - kvs = raw_decode_key(data, [], []) - msg_props = module.__message_props__() - struct = build_struct(kvs, msg_props, module.new()) - reverse_repeated(struct, msg_props.repeated_fields) - end - - defp build_struct([tag, wire, val | rest], %{field_props: f_props} = msg_props, struct) do - case f_props do - %{ - ^tag => - %{ - wire_type: ^wire, - repeated?: is_repeated, - map?: is_map, - type: type, - oneof: oneof, - name_atom: name_atom, - embedded?: embedded - } = prop - } -> - key = if oneof, do: oneof_field(prop, msg_props), else: name_atom - - struct = - if embedded do - embedded_msg = decode(val, type) - val = if is_map, do: %{embedded_msg.key => embedded_msg.value}, else: embedded_msg - val = if oneof, do: {name_atom, val}, else: val - - val = merge_embedded_value(struct, key, val, is_repeated) - - Map.put(struct, key, val) - else - val = Wire.to_proto(type, val) - val = if oneof, do: {name_atom, val}, else: val - - val = - if is_repeated do - merge_simple_repeated_value(struct, key, val) - else - val - end - - Map.put(struct, key, val) - end + @compile {:inline, + decode_field: 3, skip_varint: 4, skip_delimited: 4, reverse_repeated: 2, field_key: 2} - build_struct(rest, msg_props, struct) + @spec decode(binary, atom) :: any + def decode(bin, module) do + props = module.__message_props__() - %{^tag => %{packed?: true} = f_prop} -> - struct = put_packed_field(struct, f_prop, val) - build_struct(rest, msg_props, struct) + bin + |> build_message(module.new(), props) + |> reverse_repeated(props.repeated_fields) + end - %{^tag => %{wire_type: wire2} = f_prop} -> - raise DecodeError, - "wrong wire_type for #{prop_display(f_prop)}: got #{wire}, want #{wire2}" + defp build_message(<<>>, message, _props), do: message - _ -> - struct = try_decode_extension(struct, tag, wire, val) - build_struct(rest, msg_props, struct) - end + defp build_message(<>, message, props) do + decode_field(bin, message, props) end - defp build_struct([], _, struct) do - struct + decoder :defp, :decode_field, [:message, :props] do + field_number = bsr(value, 3) + wire_type = band(value, 7) + handle_field(rest, field_number, wire_type, message, props) end - defp merge_embedded_value(struct, key, val, is_repeated) do - case struct do - %{^key => nil} -> - if is_repeated, do: [val], else: val + defp handle_field(<>, field_number, wire_start_group(), message, props) do + skip_field(bin, message, props, [field_number]) + end - %{^key => value} -> - if is_repeated, do: [val | value], else: Map.merge(value, val) + defp handle_field(<<_bin::bits>>, closing, wire_end_group(), _message, _props) do + msg = "closing group #{inspect(closing)} but no groups are open" + raise Protobuf.DecodeError, message: msg + end - _ -> - if is_repeated, do: [val], else: val - end + defp handle_field(<>, field_number, wire_varint(), message, props) do + decode_varint(bin, field_number, message, props) end - defp merge_simple_repeated_value(struct, key, val) do - case struct do - %{^key => nil} -> - [val] + defp handle_field(<>, field_number, wire_delimited(), message, props) do + decode_delimited(bin, field_number, message, props) + end - %{^key => value} -> - [val | value] + defp handle_field(<>, field_number, wire_32bits(), message, props) do + <> = bin + handle_value(rest, field_number, wire_32bits(), value, message, props) + end - _ -> - [val] - end + defp handle_field(<>, field_number, wire_64bits(), message, props) do + <> = bin + handle_value(rest, field_number, wire_64bits(), value, message, props) end - defp raw_decode_key(<<>>, result, []), do: Enum.reverse(result) + defp handle_field(_bin, _field_number, _wire_type, _message, _props) do + raise Protobuf.DecodeError, message: "cannot decode binary data" + end - decoder :defp, :raw_decode_key, [:result, :groups] do - tag = bsr(value, 3) + decoder :defp, :skip_field, [:message, :props, :groups] do + field_number = bsr(value, 3) wire_type = band(value, 7) - raw_handle_key(wire_type, tag, groups, rest, result) - end - defp raw_handle_key(wire_start_group(), opening, groups, <>, result) do - raw_decode_key(bin, result, [opening | groups]) - end + case wire_type do + wire_start_group() -> + skip_field(rest, message, props, [field_number | groups]) - defp raw_handle_key(wire_end_group(), closing, [closing | groups], <>, result) do - raw_decode_key(bin, result, groups) - end + wire_end_group() -> + case groups do + [^field_number] -> + build_message(rest, message, props) - defp raw_handle_key(wire_end_group(), closing, [], _bin, _result) do - raise(Protobuf.DecodeError, - message: "closing group #{inspect(closing)} but no groups are open" - ) - end + [^field_number | groups] -> + skip_field(rest, message, props, groups) - defp raw_handle_key(wire_end_group(), closing, [open | _], _bin, _result) do - raise(Protobuf.DecodeError, - message: "closing group #{inspect(closing)} but group #{inspect(open)} is open" - ) - end + [group | _] -> + msg = "closing group #{inspect(field_number)} but group #{inspect(group)} is open" + raise Protobuf.DecodeError, message: msg + end - defp raw_handle_key(wire_type, tag, groups, <>, result) do - case groups do - [] -> raw_decode_value(wire_type, bin, [wire_type, tag | result], groups) - _ -> raw_decode_value(wire_type, bin, result, groups) - end - end + wire_varint() -> + skip_varint(rest, message, props, groups) - decoder :defp, :raw_decode_varint, [:result, :groups] do - case groups do - [] -> raw_decode_key(rest, [value | result], groups) - _ -> raw_decode_key(rest, result, groups) - end - end + wire_delimited() -> + skip_delimited(rest, message, props, groups) - decoder :defp, :raw_decode_delimited, [:result, :groups] do - <> = rest + wire_32bits() -> + <<_skip::bits-32, rest::bits>> = rest + skip_field(rest, message, props, groups) - case groups do - [] -> raw_decode_key(rest, [bytes | result], groups) - _ -> raw_decode_key(rest, result, groups) + wire_64bits() -> + <<_skip::bits-64, rest::bits>> = rest + skip_field(rest, message, props, groups) end end - @doc false - def raw_decode_value(wire, bin, result, groups \\ []) - - def raw_decode_value(wire_varint(), <>, result, groups) do - raw_decode_varint(bin, result, groups) + decoder :defp, :skip_varint, [:message, :props, :groups] do + _ = var!(value) + skip_field(rest, message, props, groups) end - def raw_decode_value(wire_delimited(), <>, result, groups) do - raw_decode_delimited(bin, result, groups) + decoder :defp, :skip_delimited, [:message, :props, :groups] do + <<_skip::bytes-size(value), rest::bits>> = rest + skip_field(rest, message, props, groups) end - def raw_decode_value(wire_32bits(), <>, result, []) do - raw_decode_key(rest, [n | result], []) + decoder :defp, :decode_varint, [:field_number, :message, :props] do + handle_value(rest, field_number, wire_varint(), value, message, props) end - def raw_decode_value(wire_32bits(), <<_n::bits-32, rest::bits>>, result, groups) do - raw_decode_key(rest, result, groups) - end + decoder :defp, :decode_delimited, [:field_number, :message, :props] do + <> = rest + handle_value(rest, field_number, wire_delimited(), bytes, message, props) + end + + defp handle_value(<>, field_number, wire_type, value, message, props) do + case props.field_props do + %{^field_number => %{packed?: true} = prop} -> + key = prop.name_atom + current_value = Map.get(message, key) + new_value = value_for_packed(value, current_value, prop) + new_message = Map.put(message, key, new_value) + build_message(bin, new_message, props) + + %{^field_number => %{wire_type: ^wire_type} = prop} -> + key = field_key(prop, props) + current_value = Map.get(message, key) + new_value = value_for_field(value, current_value, prop) + new_message = Map.put(message, key, new_value) + build_message(bin, new_message, props) + + %{^field_number => %{wire_type: wanted, name: field}} -> + message = "wrong wire_type for #{field}: got #{wire_type}, want #{wanted}" + raise DecodeError, message: message + + %{} -> + %mod{} = message + + new_message = + case Protobuf.Extension.get_extension_props_by_tag(mod, field_number) do + {ext_mod, %{field_props: prop}} -> + current_value = Protobuf.Extension.get(message, ext_mod, prop.name_atom, nil) + new_value = value_for_field(value, current_value, prop) + Protobuf.Extension.put(mod, message, ext_mod, prop.name_atom, new_value) + + _ -> + message + end - def raw_decode_value(wire_64bits(), <>, result, []) do - raw_decode_key(rest, [n | result], []) + build_message(bin, new_message, props) + end end - def raw_decode_value(wire_64bits(), <<_n::bits-64, rest::bits>>, result, groups) do - raw_decode_key(rest, result, groups) + defp value_for_field(value, current, %{embedded?: false} = prop) do + val = Wire.to_proto(prop.type, value) + val = if prop.oneof, do: {prop.name_atom, val}, else: val + + case {current, prop.repeated?} do + {nil, true} -> [val] + {current, true} -> [val | current] + _ -> val + end end - def raw_decode_value(_, _, _, _) do - raise Protobuf.DecodeError, message: "cannot decode binary data" + defp value_for_field(bin, current, %{embedded?: true} = prop) do + embedded_msg = decode(bin, prop.type) + val = if prop.map?, do: %{embedded_msg.key => embedded_msg.value}, else: embedded_msg + val = if prop.oneof, do: {prop.name_atom, val}, else: val + + case {current, prop.repeated?} do + {nil, true} -> [val] + {nil, false} -> val + {current, true} -> [val | current] + {current, false} -> Map.merge(current, val) + end end - # packed - defp put_packed_field(msg, %{wire_type: wire_type, type: type, name_atom: key}, bin) do - acc = - case msg do - %{^key => value} when is_list(value) -> value - %{} -> [] - end - - value = - case wire_type do - wire_varint() -> decode_varints(bin, acc) - wire_32bits() -> decode_fixed32(bin, type, acc) - wire_64bits() -> decode_fixed64(bin, type, acc) - end - - Map.put(msg, key, value) + defp value_for_packed(bin, current, prop) do + acc = current || [] + + case prop.wire_type do + wire_varint() -> decode_varints(bin, acc) + wire_32bits() -> decode_fixed32(bin, prop.type, acc) + wire_64bits() -> decode_fixed64(bin, prop.type, acc) + end end defp decode_varints(<<>>, acc), do: acc @@ -219,12 +209,6 @@ defmodule Protobuf.Decoder do defp decode_fixed64(<<>>, _, acc), do: acc - defp prop_display(prop) do - prop.name - end - - defp reverse_repeated(msg, []), do: msg - defp reverse_repeated(msg, [h | t]) do case msg do %{^h => val} when is_list(val) -> @@ -235,49 +219,12 @@ defmodule Protobuf.Decoder do end end - defp oneof_field(%{oneof: oneof}, %{oneof: oneofs}) do - {field, ^oneof} = Enum.at(oneofs, oneof) - field - end - - defp try_decode_extension(%mod{} = struct, tag, wire, val) do - case Protobuf.Extension.get_extension_props_by_tag(mod, tag) do - {ext_mod, - %{ - field_props: %{ - wire_type: ^wire, - repeated?: is_repeated, - type: type, - name_atom: name_atom, - embedded?: embedded - } - }} -> - val = - if embedded do - embedded_msg = decode(val, type) - merge_embedded_value(struct, name_atom, embedded_msg, is_repeated) - else - val = Wire.to_proto(type, val) - - if is_repeated do - merge_simple_repeated_value(struct, name_atom, val) - else - val - end - end - - key = {ext_mod, name_atom} - - case struct do - %{__pb_extensions__: pb_ext} -> - Map.put(struct, :__pb_extensions__, Map.put(pb_ext, key, val)) + defp reverse_repeated(msg, []), do: msg - _ -> - Map.put(struct, :__pb_extensions__, %{key => val}) - end + defp field_key(%{oneof: nil, name_atom: field_key}, _message_props), do: field_key - _ -> - struct - end + defp field_key(%{oneof: oneof_number}, %{oneof: oneofs}) do + {field_key, ^oneof_number} = Enum.at(oneofs, oneof_number) + field_key end end From 9d46a5f6d6a2de0db78ba8126ad6a018b108ee03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Britto?= Date: Mon, 7 Jun 2021 10:27:15 -0300 Subject: [PATCH 7/7] Remove unnecessary var! call Compilation warning is still there though :( --- lib/protobuf/decoder.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/protobuf/decoder.ex b/lib/protobuf/decoder.ex index 903e7f22..d0342423 100644 --- a/lib/protobuf/decoder.ex +++ b/lib/protobuf/decoder.ex @@ -101,7 +101,7 @@ defmodule Protobuf.Decoder do end decoder :defp, :skip_varint, [:message, :props, :groups] do - _ = var!(value) + _ = value skip_field(rest, message, props, groups) end