diff --git a/CHANGELOG.md b/CHANGELOG.md index 74e38a8..116ab4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 3.0.0 - [TBD] + * Added the `Msgpax.Unpacker` protocol. + * Refactored how extensions should be packed and unpacked. + * Added default extension implementation for `Date`. + * Added possibility of overwriting protocols through the use of `Mgpax.Ext.defimpl`. + +__Breaking changes:__ + * Packing and unpacking of extensions changed, as there is no more `Msgpax.Ext` struct. Please refer to `Msgpax.Unpacker` docs for more details and examples. + * The `Msgpax.Packer.pack` protocol now expects the `/2` arity to be defined, as the `defimpl` macro no longer introduces a catch-all clause. + * `Msgpax.defimpl` has been moved to `Msgpax.Ext.defimpl`. + * `Date` now has a default implementation using extension type 101. + +## v2.5.0 – [TBD] + * Upgraded `Msgpax.Packer` protocol so that the pack function can receive options. + +__Breaking changes:__ + + * `Msgpax.Packer.pack/1` changed to `Msgpax.Packer.pack/2`, so all protocol + implementations should be updated. See `Msgpax.defimpl/3` for examples. + ## v2.4.0 – 2023-05-27 * Dropped support for Elixir versions before 1.6. diff --git a/README.md b/README.md index 813f6a9..da2e1c2 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Msgpax is a high-performance and comprehensive library for serializing and deser * Packing and unpacking Elixir terms via [`Msgpax.pack/1`][docs-msgpax-pack-1] and [`Msgpax.unpack/1`][docs-msgpax-unpack-1] (and their bang! variants). * Unpacking of partial slices of MessagePack-encoded terms via [`Msgpax.unpack_slice/1`][docs-msgpax-unpack_slice-1]. * Support for "Binary" and "Extension" MessagePack types via [`Msgpax.Bin`][docs-msgpax-bin] and [`Msgpax.Ext`][docs-msgpax-ext], respectively. -* Protocol-based packing through the [`Msgpax.Packer`][docs-msgpax-packer] protocol, that can be derived for user-defined structs. +* Protocol-based packing through the [`Msgpax.Packer`][docs-msgpax-packer] and [`Msgpax.Unpacker`][docs-msgpax-unpacker] protocols, that can be derived for user-defined structs. * A Plug parser ([`Msgpax.PlugParser`][docs-msgpax-plug-parser]) to parse requests with MessagePack-encoded bodies. * Support for MessagePack data fragment manipulation. @@ -24,7 +24,7 @@ Add `:msgpax` as a dependency in your `mix.exs` file: ```elixir def deps do - [{:msgpax, "~> 2.0"}] + [{:msgpax, "~> 3.0"}] end ``` @@ -41,6 +41,7 @@ Msgpax is released under [the ISC license](LICENSE). [docs-msgpax-unpack-1]: http://hexdocs.pm/msgpax/Msgpax.html#unpack/1 [docs-msgpax-unpack_slice-1]: http://hexdocs.pm/msgpax/Msgpax.html#unpack_slice/1 [docs-msgpax-packer]: http://hexdocs.pm/msgpax/Msgpax.Packer.html +[docs-msgpax-unpacker]: http://hexdocs.pm/msgpax/Msgpax.Unpacker.html [docs-msgpax-bin]: http://hexdocs.pm/msgpax/Msgpax.Bin.html [docs-msgpax-ext]: http://hexdocs.pm/msgpax/Msgpax.Ext.html [docs-msgpax-plug-parser]: http://hexdocs.pm/msgpax/Msgpax.PlugParser.html diff --git a/lib/msgpax.ex b/lib/msgpax.ex index 0fbfd0c..5674b18 100644 --- a/lib/msgpax.ex +++ b/lib/msgpax.ex @@ -26,7 +26,8 @@ defmodule Msgpax do `%{foo: "bar"}` | map | `%{"foo" => "bar"}` `[foo: "bar"]` | map | `%{"foo" => "bar"}` `[1, true]` | array | `[1, true]` - `#Msgpax.Ext<4, "02:12">` | extension | `#Msgpax.Ext<4, "02:12">` + `#Msgpax.Ext0<"02:12">`⁴ | extension | `#Msgpax.Ext0<"02:12">` + `#Date<2017-12-06>` | extension | `#Date<2017-12-06>` `#DateTime<2017-12-06 00:00:00Z>` | extension | `#DateTime<2017-12-06 00:00:00Z>` ¹ `Msgpax.Packer` provides helper functions to facilitate the serialization of natively unsupported data types. @@ -34,6 +35,8 @@ defmodule Msgpax do ² NaN and ±infinity are not enabled by default. See `unpack/2` for for more information. ³ To deserialize back to `Msgpax.Bin` structs see the `unpack/2` options. + + ⁴ There are 128 extension ranging from `Msgpax.Ext0` to `Msgpax.Ext127` """ alias __MODULE__.Packer @@ -57,6 +60,7 @@ defmodule Msgpax do * `:iodata` - (boolean) if `true`, this function returns the encoded term as iodata, if `false` as a binary. Defaults to `true`. + Any other options are passed to `Msgpax.Packer.pack/2`. ## Examples iex> {:ok, packed} = Msgpax.pack("foo") @@ -72,10 +76,10 @@ defmodule Msgpax do """ @spec pack(term, Keyword.t()) :: {:ok, iodata} | {:error, Msgpax.PackError.t() | Exception.t()} def pack(term, options \\ []) when is_list(options) do - iodata? = Keyword.get(options, :iodata, true) + {iodata?, remaining_options} = Keyword.pop(options, :iodata, true) try do - Packer.pack(term) + Packer.pack(term, remaining_options) catch :throw, reason -> {:error, %Msgpax.PackError{reason: reason}} @@ -209,15 +213,19 @@ defmodule Msgpax do {:error, %Msgpax.UnpackError{reason: {:invalid_format, 163}}} """ - @spec unpack_slice(iodata, Keyword.t()) :: {:ok, any, binary} | {:error, Msgpax.UnpackError.t()} + @spec unpack_slice(iodata, Keyword.t()) :: + {:ok, any, binary} | {:error, Msgpax.UnpackError.t() | Exception.t()} def unpack_slice(iodata, options \\ []) when is_list(options) do try do iodata |> IO.iodata_to_binary() - |> Unpacker.unpack(options) + |> Unpacker.Helper.unpack(options) catch :throw, reason -> {:error, %Msgpax.UnpackError{reason: reason}} + + :error, %Protocol.UndefinedError{protocol: Msgpax.Unpacker} = exception -> + {:error, exception} else {value, rest} -> {:ok, value, rest} @@ -266,13 +274,12 @@ defmodule Msgpax do * `:binary` - (boolean) if `true`, then binaries are decoded as `Msgpax.Bin` structs instead of plain Elixir binaries. Defaults to `false`. - * `:ext` - (module) a module that implements the `Msgpax.Ext.Unpacker` - behaviour. For more information, see the docs for `Msgpax.Ext.Unpacker`. - * `:nonfinite_floats` - (boolean) if `true`, deserializes NaN and ±infinity to "signalling" atoms (see the "Data conversion" section), otherwise errors. Defaults to `false`. + Any other options are passed to `Msgpax.Unpacker.unpack/2`. + ## Examples iex> Msgpax.unpack(<<163, "foo">>) @@ -287,7 +294,8 @@ defmodule Msgpax do #Msgpax.Bin<<<3, 18, 122, 27, 115>>> """ - @spec unpack(iodata, Keyword.t()) :: {:ok, any} | {:error, Msgpax.UnpackError.t()} + @spec unpack(iodata, Keyword.t()) :: + {:ok, any} | {:error, Msgpax.UnpackError.t() | Exception.t()} def unpack(iodata, options \\ []) do case unpack_slice(iodata, options) do {:ok, value, <<>>} -> diff --git a/lib/msgpax/ext.ex b/lib/msgpax/ext.ex index ea2d858..7938ea2 100644 --- a/lib/msgpax/ext.ex +++ b/lib/msgpax/ext.ex @@ -1,7 +1,29 @@ +for type <- 0..127, s_type = Integer.to_string(type) do + ext_module = Module.concat(Msgpax, "Ext#{type}") + + defmodule ext_module do + @moduledoc false + + defstruct [:data] + + defimpl Msgpax.Packer, for: ext_module do + def pack(%_{data: data}, options), do: Msgpax.Ext.__pack__(unquote(type), data, options) + end + + defimpl Inspect do + import Inspect.Algebra + + def inspect(%{data: data}, opts) do + concat(["#Msgpax.Ext", unquote(s_type), "<", to_doc(data, opts), ">"]) + end + end + end +end + defmodule Msgpax.Ext do @moduledoc """ - A struct used to represent the MessagePack [Extension - type](https://github.com/msgpack/msgpack/blob/master/spec.md#formats-ext). + A module used to create structs representing the MessagePack [Extension + types](https://github.com/msgpack/msgpack/blob/master/spec.md#formats-ext). ## Examples @@ -9,76 +31,74 @@ defmodule Msgpax.Ext do byte `data` repeated `count` times. We could represent this as a `RepByte` struct in Elixir: - defmodule RepByte do - defstruct [:data, :count] - end + defmodule RepByte do + defstruct [:data, :count] + end - A simple (albeit not space efficient) approach to encoding such data is simply - a binary containing `data` for `count` times: `%RepByte{data: ?a, count: 2}` - would be encoded as `"aa"`. + A straightforward (albeit not space-efficient) approach to encoding such data + is by creating a binary containing `data` repeated for `count` times: + `%RepByte{data: ?a, count: 2}` would be encoded as `"aa"`. We can now define the `Msgpax.Packer` protocol for the `RepByte` struct to - tell `Msgpax` how to encode this struct (we'll choose `10` as an arbitrary + inform `Msgpax` how to encode this struct (we'll use `10` as an arbitrary integer to identify the type of this extension). - defimpl Msgpax.Packer, for: RepByte do - @rep_byte_ext_type 10 + defimpl Msgpax.Packer, for: RepByte do + @rep_byte_ext_type 10 - def pack(%RepByte{data: byte, count: count}) do - @rep_byte_ext_type - |> Msgpax.Ext.new(String.duplicate(<>, count)) - |> Msgpax.Packer.pack() + def pack(%RepByte{data: byte, count: count}, options) do + @rep_byte_ext_type + |> Msgpax.Ext.new(String.duplicate(<>, count)) + |> Msgpax.Packer.pack(options) + end end - end - Now, we can pack `RepByte`s: + Note that `Msgpax.Ext.new/2` returns the extension struct `Msgpax.Ext10`, + which is then packed by `Msgpax.Packer` - all struct extensions already + implement `Msgpax.Packer`. - iex> packed = Msgpax.pack!(%RepByte{data: ?a, count: 3}) - iex> Msgpax.unpack!(packed) - #Msgpax.Ext<10, "aaa"> + Now, we can pack `RepByte`s: - ### Unpacking + iex> packed = Msgpax.pack!(%RepByte{data: ?a, count: 3}) + [[199, 3], 10 | "aaa"] - As seen in the example above, since the `RepByte` struct is *packed* as a - MessagePack extension, it will be unpacked as that extension later on; what we - may want, however, is to unpack that extension back to a `RepByte` struct. + iex> Msgpax.unpack!(packed) + ** (Protocol.UndefinedError) protocol Msgpax.Unpacker not implemented for #Msgpax.Ext10<"aaa"> of type Msgpax.Ext10 (a struct). - To do this, we can pass an `:ext` option to `Msgpax.unpack/2` (and other - unpacking functions). This option has to be a module that implements the - `Msgpax.Ext.Unpacker` behaviour; it will be used to unpack extensions to - arbitrary Elixir terms. + ### Unpacking - For our `RepByte` example, we could create an unpacker module like this: + Each extension type is mapped to a predetermined Elixir struct. For example, + extension type `10` is mapped to `Msgpax.Ext10`. This struct contains the + data field with the serialized extension data. There are 128 extensions ranging + from 0 to 127. - defmodule MyExtUnpacker do - @behaviour Msgpax.Ext.Unpacker - @rep_byte_ext_type 10 + To unpack an extension back to a struct, we need to implement the + `Msgpax.Unpacker` protocol for the extension struct. For our `RepByte` example, + implementation might look like this: - @impl true - def unpack(%Msgpax.Ext{type: @rep_byte_ext_type, data: data}) do - <> = data - {:ok, %RepByte{data: byte, count: byte_size(data)}} + defimpl Msgpax.Unpacker, for: Msgpax.Ext10 do + def unpack(%{data: <>}, _options) do + {:ok, %RepByte{data: byte, count: byte_size(data)}} + end end - end - With this in place, we can now unpack a packed `RepByte` back to a `RepByte` - struct: + With this in place, we can unpack a packed `RepByte`: - iex> packed = Msgpax.pack!(%RepByte{data: ?a, count: 3}) - iex> Msgpax.unpack!(packed, ext: MyExtUnpacker) - %RepByte{data: ?a, count: 3} + iex> packed = Msgpax.pack!(%RepByte{data: ?a, count: 3}) + iex> Msgpax.unpack!(packed) + %RepByte{data: ?a, count: 3} + > #### `use Msgpax.Ext` {: .info} + > + > When you `use Msgpax.Ext`, the Msgpax.Ext module replaces the default + > `Kernel.defimpl` with `Msgpax.Ext.defimpl`, allowing the overwriting of the + > protocols defined by `Msgpax`. """ - @type type :: 0..127 - @type t :: %__MODULE__{type: type, data: iodata} - - defstruct [:type, :data] - @doc """ - Creates a new `Msgpax.Ext` struct. + Creates a new `Msgpax.Ext#` struct. - `type` must be an integer in `0..127` and it will be used as the type of the + `type` must be an integer in `0..127`, and it will be used as the type of the extension (whose meaning depends on your application). `data` must be an iodata containing the serialized extension (whose serialization depends on your application). @@ -86,28 +106,116 @@ defmodule Msgpax.Ext do ## Examples iex> Msgpax.Ext.new(24, "foo") - #Msgpax.Ext<24, "foo"> + #Msgpax.Ext24<"foo"> iex> Msgpax.Ext.new(25, 'bar') - #Msgpax.Ext<25, 'bar'> + #Msgpax.Ext25<'bar'> + + """ + for type <- 0..127 do + extension = Module.concat(Msgpax, "Ext#{type}") + + def new(unquote(type), data) when is_binary(data) or is_list(data) do + struct(unquote(extension), data: data) + end + end + + @doc false + def __pack__(type, data, _options) do + [format(data), Bitwise.band(256 + type, 255) | data] + end + + defp format(data) do + size = IO.iodata_length(data) + + cond do + size == 1 -> 0xD4 + size == 2 -> 0xD5 + size == 4 -> 0xD6 + size == 8 -> 0xD7 + size == 16 -> 0xD8 + size < 256 -> [0xC7, size] + size < 0x10000 -> <<0xC8, size::16>> + size < 0x100000000 -> <<0xC9, size::32>> + true -> throw({:too_big, data}) + end + end + + @doc """ + Works similarly to `Kernel.defimpl`, but it also allows overwriting + default protocol implementations provided by `Msgpax`. + + Overriding one of the out-of-the-box types may be necessary if there are conflicts + with your extensions or if a custom serialization format is required. + + Note that overwriting basic types such as `Atom` or `String` needs to be done + through the use of an extension struct so that the proper unpacking + implementation can be provided. + + ## Example + + You can overwrite the default implementation for `Date` as follows: + + use Msgpax.Ext # replaces Kernel.impl with Msgpax.Ext.defimpl + + defimpl Msgpax.Packer, for: Date do + def pack(_date, options) do + 2 + |> Msgpax.Ext.new("A") + |> @protocol.pack(options) + end + end + + defimpl Msgpax.Unpacker, for: Msgpax.Ext2 do + def unpack(%{data: date}, _options) do + {:ok, date} + end + end + + This would lead to: + + iex > Date.utc_now() |> Msgpax.pack!() |> Msgpax.unpack!() + "A" """ - def new(type, data) - when type in 0..127 and (is_binary(data) or is_list(data)) do - %__MODULE__{type: type, data: data} + defmacro defimpl(protocol, opts, do_block \\ []) do + protocol = Macro.expand(protocol, __CALLER__) + + if protocol not in [Msgpax.Packer, Msgpax.Unpacker] do + arity = (do_block == [] && "2") || "3" + + raise ArgumentError, + "`Msgpax.defimpl/#{arity}` is not supported for protocols other than `Msgpax.Packer` and `Msgpax.Unpacker`: got `#{Macro.inspect_atom(:literal, protocol)}`" + end + + for_module = + opts + |> Keyword.get(:for, __CALLER__.module) + |> Macro.expand(__CALLER__) + + do_block = Keyword.get(opts, :do, do_block) + + quote do + current_ignore_module_conflict = Code.get_compiler_option(:ignore_module_conflict) || false + + Code.put_compiler_option(:ignore_module_conflict, true) + + impl = + defimpl unquote(protocol), for: unquote(for_module) do + unquote(do_block) + end + + Code.put_compiler_option(:ignore_module_conflict, current_ignore_module_conflict) + + impl + end end - defimpl Inspect do - import Inspect.Algebra - - def inspect(%{type: type, data: data}, opts) do - concat([ - "#Msgpax.Ext<", - Inspect.Integer.inspect(type, opts), - ", ", - to_doc(data, opts), - ">" - ]) + @doc false + defmacro __using__(_opts) do + quote do + import Kernel, except: [defimpl: 2, defimpl: 3] + import unquote(__MODULE__), only: [defimpl: 2, defimpl: 3] end end end diff --git a/lib/msgpax/ext/date.ex b/lib/msgpax/ext/date.ex new file mode 100644 index 0000000..8d6af84 --- /dev/null +++ b/lib/msgpax/ext/date.ex @@ -0,0 +1,24 @@ +defmodule Msgpax.Ext.Date do + @moduledoc """ + Default implementation for `Date` type. + + Uses extension type `Msgpax.Ext101`. + """ + use Msgpax.Ext + + defimpl Msgpax.Packer, for: Date do + def pack(%{year: year, month: month, day: day}, options) do + 101 + |> Msgpax.Ext.new(<>) + |> @protocol.pack(options) + end + end + + defimpl Msgpax.Unpacker, for: Msgpax.Ext101 do + def unpack(%{data: <>}, _options) do + with {:error, _reason} <- Date.new(year, month, day), do: :error + end + + def unpack(_ext_101, _options), do: :error + end +end diff --git a/lib/msgpax/ext/unpacker.ex b/lib/msgpax/ext/unpacker.ex deleted file mode 100644 index 9b8ba0d..0000000 --- a/lib/msgpax/ext/unpacker.ex +++ /dev/null @@ -1,19 +0,0 @@ -defmodule Msgpax.Ext.Unpacker do - @moduledoc """ - Behaviour to unpack `Msgpax.Ext` structs into arbitrary terms. - - Modules that implement this behaviour can be passed as the value of the `:ext` - option in `Msgpax.unpack/2` and `Msgpax.unpack_slice/2` (and their bang! - variants). - - See the documentation for `Msgpax.Ext` for usage examples. - """ - - @doc """ - Invoked when unpacking the given extension. - - It should return `{:ok, value}` to have Msgpax return `value` when unpacking - the given extension, or `:error` if there's an error while unpacking. - """ - @callback unpack(ext :: Msgpax.Ext.t() | Msgpax.ReservedExt.t()) :: {:ok, any} | :error -end diff --git a/lib/msgpax/fragment.ex b/lib/msgpax/fragment.ex index 94d8278..8af79b4 100644 --- a/lib/msgpax/fragment.ex +++ b/lib/msgpax/fragment.ex @@ -22,7 +22,7 @@ defmodule Msgpax.Fragment do end defimpl Msgpax.Packer do - def pack(%{data: data}), do: data + def pack(%{data: data}, _options), do: data end defimpl Inspect do diff --git a/lib/msgpax/packer.ex b/lib/msgpax/packer.ex index 29dab7f..9a2180a 100644 --- a/lib/msgpax/packer.ex +++ b/lib/msgpax/packer.ex @@ -102,7 +102,7 @@ defprotocol Msgpax.Packer do It returns an iodata result. """ - def pack(term) + def pack(term, options) @doc """ Returns serialized NaN in 64-bit format. @@ -124,23 +124,23 @@ defprotocol Msgpax.Packer do end defimpl Msgpax.Packer, for: Atom do - def pack(nil), do: [0xC0] - def pack(false), do: [0xC2] - def pack(true), do: [0xC3] + def pack(nil, _options), do: [0xC0] + def pack(false, _options), do: [0xC2] + def pack(true, _options), do: [0xC3] - def pack(atom) do + def pack(atom, opts) do atom |> Atom.to_string() - |> @protocol.BitString.pack() + |> @protocol.BitString.pack(opts) end end defimpl Msgpax.Packer, for: BitString do - def pack(binary) when is_binary(binary) do + def pack(binary, _options) when is_binary(binary) do [format(binary) | binary] end - def pack(bits) do + def pack(bits, _options) do throw({:not_encodable, bits}) end @@ -162,15 +162,15 @@ defimpl Msgpax.Packer, for: Map do @protocol.Any.deriving(module, struct, options) end - def pack(map) do - [format(map) | map |> Map.to_list() |> pack([])] + def pack(map, options) do + [format(map) | map |> Map.to_list() |> do_pack([], options)] end - defp pack([{key, value} | rest], result) do - pack(rest, [@protocol.pack(key), @protocol.pack(value) | result]) + defp do_pack([{key, value} | rest], result, opts) do + do_pack(rest, [@protocol.pack(key, opts), @protocol.pack(value, opts) | result], opts) end - defp pack([], result), do: result + defp do_pack([], result, _opts), do: result defp format(map) do length = map_size(map) @@ -185,15 +185,15 @@ defimpl Msgpax.Packer, for: Map do end defimpl Msgpax.Packer, for: List do - def pack(list) do - [format(list) | list |> Enum.reverse() |> pack([])] + def pack(list, options) do + [format(list) | list |> Enum.reverse() |> do_pack([], options)] end - defp pack([item | rest], result) do - pack(rest, [@protocol.pack(item) | result]) + defp do_pack([item | rest], result, opts) do + do_pack(rest, [@protocol.pack(item, opts) | result], opts) end - defp pack([], result), do: result + defp do_pack([], result, _opts), do: result defp format(list) do length = length(list) @@ -208,13 +208,13 @@ defimpl Msgpax.Packer, for: List do end defimpl Msgpax.Packer, for: Float do - def pack(num) do + def pack(num, _options) do <<0xCB, num::64-float>> end end defimpl Msgpax.Packer, for: Integer do - def pack(int) when int < 0 do + def pack(int, _options) when int < 0 do cond do int >= -32 -> [0x100 + int] int >= -128 -> [0xD0, 0x100 + int] @@ -225,7 +225,7 @@ defimpl Msgpax.Packer, for: Integer do end end - def pack(int) do + def pack(int, _options) do cond do int < 128 -> [int] int < 256 -> [0xCC, int] @@ -238,7 +238,7 @@ defimpl Msgpax.Packer, for: Integer do end defimpl Msgpax.Packer, for: Msgpax.Bin do - def pack(%{data: data}) when is_binary(data), do: [format(data) | data] + def pack(%{data: data}, _options) when is_binary(data), do: [format(data) | data] defp format(binary) do size = byte_size(binary) @@ -252,30 +252,6 @@ defimpl Msgpax.Packer, for: Msgpax.Bin do end end -defimpl Msgpax.Packer, for: [Msgpax.Ext, Msgpax.ReservedExt] do - require Bitwise - - def pack(%_{type: type, data: data}) do - [format(data), Bitwise.band(256 + type, 255) | data] - end - - defp format(data) do - size = IO.iodata_length(data) - - cond do - size == 1 -> 0xD4 - size == 2 -> 0xD5 - size == 4 -> 0xD6 - size == 8 -> 0xD7 - size == 16 -> 0xD8 - size < 256 -> [0xC7, size] - size < 0x10000 -> <<0xC8, size::16>> - size < 0x100000000 -> <<0xC9, size::32>> - true -> throw({:too_big, data}) - end - end -end - defimpl Msgpax.Packer, for: Any do defmacro __deriving__(module, struct, options) do deriving(module, struct, options) @@ -304,15 +280,15 @@ defimpl Msgpax.Packer, for: Any do quote do defimpl unquote(@protocol), for: unquote(module) do - def pack(struct) do + def pack(struct, options) do unquote(extractor) - |> @protocol.Map.pack() + |> @protocol.Map.pack(options) end end end end - def pack(term) do + def pack(term, _options) do raise Protocol.UndefinedError, protocol: @protocol, value: term end end diff --git a/lib/msgpax/reserved_ext.ex b/lib/msgpax/reserved_ext.ex index b7dd2a8..3101b9a 100644 --- a/lib/msgpax/reserved_ext.ex +++ b/lib/msgpax/reserved_ext.ex @@ -1,27 +1,16 @@ -defimpl Msgpax.Packer, for: DateTime do - import Bitwise +# Since we are only implementing extension 127, there is no need to define all +# other structs +for reserved_type <- -1..-1, type = Bitwise.band(reserved_type, 127) do + reserved_ext_module = Module.concat(Msgpax, "ReservedExt#{type}") - def pack(datetime) do - -1 - |> Msgpax.ReservedExt.new(build_data(datetime)) - |> @protocol.Msgpax.ReservedExt.pack() - end - - defp build_data(datetime) do - total_nanoseconds = @for.to_unix(datetime, :nanosecond) - seconds = Integer.floor_div(total_nanoseconds, 1_000_000_000) - nanoseconds = Integer.mod(total_nanoseconds, 1_000_000_000) + defmodule reserved_ext_module do + @moduledoc false - if seconds >>> 34 == 0 do - content = nanoseconds <<< 34 ||| seconds + defstruct [:data] - if (content &&& 0xFFFFFFFF00000000) == 0 do - <> - else - <> - end - else - <> + defimpl Msgpax.Packer, for: reserved_ext_module do + def pack(%_{data: data}, options), + do: Msgpax.Ext.__pack__(unquote(reserved_type), data, options) end end end @@ -31,47 +20,12 @@ defmodule Msgpax.ReservedExt do Reserved extensions automatically get handled by Msgpax. """ - @behaviour Msgpax.Ext.Unpacker - - @nanosecond_range -62_167_219_200_000_000_000..253_402_300_799_999_999_999 - - @typep type :: -128..-1 - @opaque t :: %__MODULE__{type: type, data: binary} - - defstruct [:type, :data] - - @doc false - def new(type, data) - when type in -128..-1 and is_binary(data) do - %__MODULE__{type: type, data: data} - end - @doc false - @impl true - def unpack(%__MODULE__{type: -1, data: data}) do - case data do - <> -> - DateTime.from_unix(seconds) - - <> -> - total_nanoseconds = seconds * 1_000_000_000 + nanoseconds - DateTime.from_unix(total_nanoseconds, :nanosecond) - - <> -> - total_nanoseconds = seconds * 1_000_000_000 + nanoseconds + for reserved_type <- -128..-1, type = Bitwise.band(reserved_type, 127) do + extension = Module.concat(Msgpax, "ReservedExt#{type}") - if total_nanoseconds in @nanosecond_range do - DateTime.from_unix(total_nanoseconds, :nanosecond) - else - :error - end - - _ -> - :error + def new(unquote(reserved_type), data) when is_binary(data) do + struct(unquote(extension), data: data) end end - - def unpack(%__MODULE__{} = struct) do - {:ok, struct} - end end diff --git a/lib/msgpax/reserved_ext/datetime.ex b/lib/msgpax/reserved_ext/datetime.ex new file mode 100644 index 0000000..70982bf --- /dev/null +++ b/lib/msgpax/reserved_ext/datetime.ex @@ -0,0 +1,50 @@ +defimpl Msgpax.Packer, for: DateTime do + import Bitwise + + def pack(datetime, options) do + -1 + |> Msgpax.ReservedExt.new(build_data(datetime)) + |> @protocol.pack(options) + end + + defp build_data(datetime) do + total_nanoseconds = @for.to_unix(datetime, :nanosecond) + seconds = Integer.floor_div(total_nanoseconds, 1_000_000_000) + nanoseconds = Integer.mod(total_nanoseconds, 1_000_000_000) + + if seconds >>> 34 == 0 do + content = nanoseconds <<< 34 ||| seconds + + if (content &&& 0xFFFFFFFF00000000) == 0 do + <> + else + <> + end + else + <> + end + end +end + +defimpl Msgpax.Unpacker, for: Msgpax.ReservedExt127 do + @nanosecond_range -62_167_219_200_000_000_000..253_402_300_799_999_999_999 + + def unpack(%{data: <>}, _options), do: DateTime.from_unix(seconds) + + def unpack(%{data: <>}, _options) do + total_nanoseconds = seconds * 1_000_000_000 + nanoseconds + DateTime.from_unix(total_nanoseconds, :nanosecond) + end + + def unpack(%{data: <>}, _options) do + total_nanoseconds = seconds * 1_000_000_000 + nanoseconds + + if total_nanoseconds in @nanosecond_range do + DateTime.from_unix(total_nanoseconds, :nanosecond) + else + :error + end + end + + def unpack(_reserved_ext_1, _options), do: :error +end diff --git a/lib/msgpax/unpacker.ex b/lib/msgpax/unpacker.ex index 9c2f52b..6dd0cf0 100644 --- a/lib/msgpax/unpacker.ex +++ b/lib/msgpax/unpacker.ex @@ -8,7 +8,7 @@ defmodule Msgpax.UnpackError do {:excess_bytes, binary} | {:invalid_format, integer} | :incomplete - | {:ext_unpack_failure, module, Msgpax.Ext.t()} + | {:ext_unpack_failure, module, Msgpax.Unpacker.t()} | {:nonfinite_float, atom} } @@ -34,7 +34,48 @@ defmodule Msgpax.UnpackError do end end -defmodule Msgpax.Unpacker do +defprotocol Msgpax.Unpacker do + @moduledoc """ + The `Msgpax.Unpacker` protocol is responsible for deserializing any `Msgpax` + extension. There are 128 possible extensions, each of which is mapped to a + struct: `Msgpax.Ext0`, `Msgpax.Ext1`, ..., `Msgpax.Ext127`. + + The `Msgpax.Packer` converts the extension data into one of these structs and + then serializes it. + + Therefore, in order to deserialize an extension, we need to implement this + protocol for its corresponding struct. + + ### Example + Below is Msgpax's default implementation for the `Date` type using extension + type 101. + + defimpl Msgpax.Packer, for: Date do + def pack(%{year: year, month: month, day: day}, options) do + 101 + |> Msgpax.Ext.new(<>) ## <-- returns %Msgpax.Ext101{} + |> @protocol.pack(options) + end + end + + defimpl Msgpax.Unpacker, for: Msgpax.Ext101 do + def unpack(%{data: <>}, _options) do + with {:error, _reason} <- Date.new(year, month, day), do: :error + end + + def unpack(_ext_101, _options), do: :error + end + + """ + + @doc """ + This function deserializes one of the 128 possible extensions + into an elixir term. + """ + def unpack(term, options) +end + +defmodule Msgpax.Unpacker.Helper do @moduledoc false alias Msgpax.{ @@ -43,6 +84,7 @@ defmodule Msgpax.Unpacker do NegInfinity } + @doc false def unpack(<>, options) do unpack(buffer, [], Map.new(options), [], 0, 1) end @@ -211,14 +253,14 @@ defmodule Msgpax.Unpacker do type |> Kernel.-(256) |> Msgpax.ReservedExt.new(content) - |> unpack_ext(%{ext: Msgpax.ReservedExt}) + |> unpack_ext(options) end end @compile {:inline, [unpack_ext: 2]} - defp unpack_ext(struct, %{ext: module}) when is_atom(module) do - case module.unpack(struct) do + defp unpack_ext(%module{} = struct, options) do + case Msgpax.Unpacker.unpack(struct, Enum.into(options, [])) do {:ok, result} -> result @@ -227,10 +269,6 @@ defmodule Msgpax.Unpacker do end end - defp unpack_ext(struct, _options) do - struct - end - @compile {:inline, [build_collection: 3]} defp build_collection(result, count, :list) do diff --git a/mix.exs b/mix.exs index 743e8bf..23a3a5a 100644 --- a/mix.exs +++ b/mix.exs @@ -18,7 +18,12 @@ defmodule Msgpax.Mixfile do main: "Msgpax", source_ref: "v#{@version}", source_url: @source_url, - extras: ["CHANGELOG.md"] + extras: ["CHANGELOG.md"], + groups_for_modules: [ + "Default Extensions": [ + Msgpax.Ext.Date + ] + ] ] ] end diff --git a/test/msgpax/ext/date_test.exs b/test/msgpax/ext/date_test.exs new file mode 100644 index 0000000..9023e48 --- /dev/null +++ b/test/msgpax/ext/date_test.exs @@ -0,0 +1,8 @@ +defmodule Msgpax.Ext.DateTest do + use Msgpax.Case, async: true + + test "`Date` has default extension implementation with code 101" do + date = ~D[2023-08-23] + assert_format date, <<0xC7, 3, 101, 15, 207, 23>>, date + end +end diff --git a/test/msgpax/ext/overwrite_test.exs b/test/msgpax/ext/overwrite_test.exs new file mode 100644 index 0000000..2b5811b --- /dev/null +++ b/test/msgpax/ext/overwrite_test.exs @@ -0,0 +1,49 @@ +defmodule Msgpax.Ext.OverwriteTest do + use Msgpax.Case, async: false + + test "default implementations can be overridden" do + defmodule OverrideDefaultImplementation do + use Msgpax.Ext + + defimpl Msgpax.Packer, for: Atom do + def pack(atom, options) do + 3 + |> Msgpax.Ext.new(Atom.to_string(atom)) + |> @protocol.pack(options) + end + end + + defimpl Msgpax.Unpacker, for: Msgpax.Ext3 do + def unpack(%{data: atom}, _options) do + {:ok, String.to_existing_atom(atom)} + end + end + end + + atom = :A + assert_format atom, <<0xD4, 3, ?A>>, :A + end + + test "default extensions can be overridden" do + defmodule OverrideDefaultExtension do + use Msgpax.Ext + + defimpl Msgpax.Packer, for: Date do + def pack(_date, options) do + 2 + |> Msgpax.Ext.new("A") + |> @protocol.pack(options) + end + end + + defimpl Msgpax.Unpacker, for: Msgpax.Ext2 do + def unpack(%{data: date}, _options) do + {:ok, date} + end + end + end + + now = Date.utc_today() + assert_format now, <<0xD4, 2, ?A>>, "A" + end +end diff --git a/test/msgpax/ext_test.exs b/test/msgpax/ext_test.exs index d4d5a57..9b246c0 100644 --- a/test/msgpax/ext_test.exs +++ b/test/msgpax/ext_test.exs @@ -12,95 +12,96 @@ defmodule Msgpax.ExtTest do %__MODULE__{seed: seed, size: size} end - @behaviour Msgpax.Ext.Unpacker - - def unpack(%Msgpax.Ext{type: 42, data: <<>>}) do - {:ok, new(<<>>, 0)} - end - - def unpack(%Msgpax.Ext{type: 42, data: <> = data}) do - {:ok, new(<>, byte_size(data))} - end - defimpl Msgpax.Packer do - def pack(%Sample{seed: seed, size: size}) do + def pack(%Sample{seed: seed, size: size}, options) do module = if is_list(seed), do: List, else: String 42 |> Msgpax.Ext.new(module.duplicate(seed, size)) - |> @protocol.Msgpax.Ext.pack() + |> @protocol.pack(options) end end - end - defmodule Broken do - def unpack(%Msgpax.Ext{}) do - :error + defimpl Msgpax.Unpacker, for: Msgpax.Ext42 do + def unpack(%{data: <<>>}, _options) do + {:ok, Sample.new(<<>>, 0)} + end + + def unpack(%{data: <> = data}, options) do + case Keyword.get(options, :break_me) do + true -> :error + _ -> {:ok, Sample.new(<>, byte_size(data))} + end + end end end test "fixext 1" do data = Sample.new("A", 1) - assert_format data, <<0xD4, 42, ?A>>, {data, [ext: Sample]} + assert_format data, <<0xD4, 42, ?A>>, data end test "fixext 2" do data = Sample.new("B", 2) - assert_format data, <<0xD5, 42, ?B>>, {data, [ext: Sample]} + assert_format data, <<0xD5, 42, ?B>>, data end test "fixext 4" do data = Sample.new("C", 4) - assert_format data, <<0xD6, 42, ?C>>, {data, [ext: Sample]} + assert_format data, <<0xD6, 42, ?C>>, data end test "fixext 8" do data = Sample.new("D", 8) - assert_format data, <<0xD7, 42, ?D>>, {data, [ext: Sample]} + assert_format data, <<0xD7, 42, ?D>>, data end test "fixext 16" do data = Sample.new("E", 16) - assert_format data, <<0xD8, 42, ?E>>, {data, [ext: Sample]} + assert_format data, <<0xD8, 42, ?E>>, data end test "ext 8" do input = Sample.new("0", 0) output = Sample.new("", 0) - assert_format input, <<0xC7, 0, 42>>, {output, [ext: Sample]} + assert_format input, <<0xC7, 0, 42>>, output data = Sample.new("1", 255) - assert_format data, <<0xC7, 255, 42, ?1>>, {data, [ext: Sample]} + assert_format data, <<0xC7, 255, 42, ?1>>, data end test "ext 16" do data = Sample.new("2", 0x100) - assert_format data, <<0xC8, 0x100::16, 42, ?2>>, {data, [ext: Sample]} + assert_format data, <<0xC8, 0x100::16, 42, ?2>>, data data = Sample.new("3", 0xFFFF) - assert_format data, <<0xC8, 0xFFFF::16, 42, ?3>>, {data, [ext: Sample]} + assert_format data, <<0xC8, 0xFFFF::16, 42, ?3>>, data end test "ext 32" do data = Sample.new("4", 0x10000) - assert_format data, <<0xC9, 0x10000::32, 42, ?4>>, {data, [ext: Sample]} + assert_format data, <<0xC9, 0x10000::32, 42, ?4>>, data end test "empty options" do - output = Msgpax.Ext.new(42, "G") - assert_format Sample.new("G", 1), <<0xD4, 42, ?G>>, output + data = Sample.new("G", 1) + assert_format data, <<0xD4, 42, ?G>>, data end test "iodata input" do - output = Msgpax.Ext.new(42, "HH") - assert_format Sample.new('H', 2), <<0xD5, 42, ?H>>, output + assert_format Sample.new('H', 2), <<0xD5, 42, ?H>>, Sample.new("H", 2) end - test "broken ext" do - assert {:error, %UnpackError{reason: reason}} = Msgpax.unpack(<<0xD4, 42, ?A>>, ext: Broken) - assert reason == {:ext_unpack_failure, Broken, Msgpax.Ext.new(42, "A")} + test "broken ext unpacker" do + assert {:error, unpack_error} = Msgpax.unpack(<<0xD4, 42, ?A>>, break_me: true) + + assert unpack_error == %UnpackError{ + reason: {:ext_unpack_failure, Msgpax.Ext42, %Msgpax.Ext42{data: "A"}} + } end - test "not supported reserved ext type" do - assert {:ok, result} = Msgpax.unpack(<<0xD4, -5, ?A>>) - assert result == %Msgpax.ReservedExt{data: "A", type: -5} + test "ext unpacker not implemented" do + assert {:error, reason} = Msgpax.unpack(<<0xD4, 43, ?A>>) + + assert %Protocol.UndefinedError{protocol: Msgpax.Unpacker, value: %Msgpax.Ext43{data: "A"}} = + reason end end diff --git a/test/msgpax/plug_parser_test.exs b/test/msgpax/plug_parser_test.exs index 1dca304..a9441ec 100644 --- a/test/msgpax/plug_parser_test.exs +++ b/test/msgpax/plug_parser_test.exs @@ -1,5 +1,5 @@ defmodule Msgpax.PlugParserTest do - use ExUnit.Case + use ExUnit.Case, async: true use Plug.Test test "body with a MessagePack-encoded map" do