Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge branch 'master' of https://github.com/freke/erlang_protobuffs i…

…nto az2-merge-protobuffs-with-upstream

Conflicts:
	Makefile
	rebar
	src/protobuffs_parser.erl
	support/include.mk
	t/Makefile
	t/hasdefault.proto
	t/protobuffs_t_001.t
	t/protobuffs_t_002.t
	t/protobuffs_t_003.t
	t/protobuffs_t_005.t
	t/protobuffs_t_006.t
	t/protobuffs_t_007.t
	t/protobuffs_t_008.t
  • Loading branch information...
commit 47476c1d790afc5d1fcc1b1009ff64df6512109c 2 parents 84bd7e2 + 10f46ae
@Vagabond Vagabond authored
Showing with 2,224 additions and 1,053 deletions.
  1. +8 −1 .gitignore
  2. +3 −0  AUTHORS
  3. +13 −0 ChangeLog
  4. +11 −3 README.markdown
  5. +10 −0 bin/protoc-erl
  6. +0 −18 ebin/protobuffs.app
  7. BIN  rebar
  8. +3 −1 rebar.config
  9. +0 −9 src/Makefile
  10. +39 −10 src/pokemon_pb.erl
  11. +8 −0 src/protobuffs.app.src
  12. +192 −18 src/protobuffs.erl
  13. +481 −94 src/protobuffs_compile.erl
  14. +0 −247 src/protobuffs_parser.erl
  15. +69 −0 src/protobuffs_parser.yrl
  16. +31 −0 src/protobuffs_scanner.xrl
  17. +0 −43 support/include.mk
  18. +0 −12 t/Makefile
  19. +0 −40 t/empty.proto
  20. +0 −13 t/hasdefault.proto
  21. +0 −276 t/protobuffs_eqc.erl
  22. +0 −15 t/protobuffs_t_001.t
  23. +0 −10 t/protobuffs_t_002.t
  24. +0 −42 t/protobuffs_t_003.t
  25. +0 −27 t/protobuffs_t_005.t
  26. +0 −45 t/protobuffs_t_006.t
  27. +0 −73 t/protobuffs_t_007.t
  28. +0 −33 t/protobuffs_t_008.t
  29. +0 −22 t/protobuffs_t_009.t
  30. +346 −0 test/erlang_protobuffs_SUITE.erl
  31. +1 −0  test/erlang_protobuffs_SUITE_data/.gitignore
  32. +34 −0 test/erlang_protobuffs_SUITE_data/addressbook.proto
  33. +4 −0 test/erlang_protobuffs_SUITE_data/boolean.proto
  34. +17 −0 test/erlang_protobuffs_SUITE_data/empty.proto
  35. +8 −0 test/erlang_protobuffs_SUITE_data/enum.proto
  36. +8 −0 test/erlang_protobuffs_SUITE_data/enum_outside.proto
  37. +5 −0 test/erlang_protobuffs_SUITE_data/extend.proto
  38. +5 −0 test/erlang_protobuffs_SUITE_data/extend_in_reserved_range.proto
  39. +5 −0 test/erlang_protobuffs_SUITE_data/extend_out_of_range.proto
  40. +7 −0 test/erlang_protobuffs_SUITE_data/extensions.proto
  41. +18 −0 test/erlang_protobuffs_SUITE_data/hasdefault.proto
  42. +5 −0 test/erlang_protobuffs_SUITE_data/import.proto
  43. +3 −0  test/erlang_protobuffs_SUITE_data/import_target.proto
  44. +18 −0 test/erlang_protobuffs_SUITE_data/nested1.proto
  45. +19 −0 test/erlang_protobuffs_SUITE_data/nested2.proto
  46. +15 −0 test/erlang_protobuffs_SUITE_data/nested3.proto
  47. +15 −0 test/erlang_protobuffs_SUITE_data/nested4.proto
  48. +9 −0 test/erlang_protobuffs_SUITE_data/nested5.proto
  49. +16 −0 test/erlang_protobuffs_SUITE_data/packed_repeated.proto
  50. +1 −0  {t → test/erlang_protobuffs_SUITE_data}/repeater.proto
  51. +11 −0 test/erlang_protobuffs_SUITE_data/service.proto
  52. +2 −1  {t → test/erlang_protobuffs_SUITE_data}/simple.proto
  53. +3 −0  test/erlang_protobuffs_SUITE_data/single.proto
  54. +66 −0 test/erlang_protobuffs_SUITE_data/special_words.proto
  55. +378 −0 test/protobuffs_eqc.erl
  56. +67 −0 test/protobuffs_parser_tests.erl
  57. +270 −0 test/protobuffs_tests.erl
View
9 .gitignore
@@ -1,3 +1,10 @@
ebin
*.beam
-*~
+*.gz
+*~
+doc
+logs
+.eunit
+.directory
+src/protobuffs_parser.erl
+src/protobuffs_scanner.erl
View
3  AUTHORS
@@ -0,0 +1,3 @@
+Nick Gerakines
+Jacob Vorreuter
+erlrc integration by Cliff Moon
View
13 ChangeLog
@@ -0,0 +1,13 @@
+* Thu Dec 03 2009 Anthony Molinaro <anthony.molinaro@openx.org> 0.0.1
+- Fixed path for unit tests
+- Have .gitignore ignore all generated files
+- Fixed bool type, they were being improperly left as 0 and 1 instead of using
+ the true and false atoms
+- Nested types should now work according to spec (as much as there is one)
+- Fixed bool defaults, they were being improperly scanned, so falling back to
+ keywords
+* freke <davabe@hotmail.com>
+- Fixed enum support
+- Changed build system to Rake
+- Added QuickCheck tests
+- Changed so parsing of proto files is done by leex and yecc
View
14 README.markdown
@@ -55,9 +55,7 @@ decode functions for the messages defined.
## CAVEATS
Support for parsing proto files and creating code from it is volatile and
-should be considered alpha software at best. It currently only supports flat
-messages, simple types (ints, strings, etc) and will break on ENUM types and
-any sort of nesting. Please do not use this in production.
+should be considered alpha software at best. Please do not use this in production.
## no_debug_info
@@ -67,6 +65,16 @@ _pb modules. Running protobuffs_compile:scan_file/1 reads the erlang forms from
the pokemon_pb.beam file and expands and alters those forms to create the generated
module.
+## Building with rebar
+To compile
+ %>./rebar compile
+
+To run all tests
+ %>./rebar eunit
+ %>./rebar ct
+
+Se rebar doc for more information.
+
## CREDITS
Some of the protobuffs.erl module came from code written by Brian Buchanan.
View
10 bin/protoc-erl
@@ -0,0 +1,10 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -sasl errlog_type error -boot start_sasl -noshell
+
+main ([File]) ->
+ protobuffs_compile:generate_source (File);
+main (_) ->
+ io:format ("usage: ~s <protofile>~n",
+ [filename:basename (escript:script_name())]),
+ halt (1).
View
18 ebin/protobuffs.app
@@ -1,18 +0,0 @@
-{application, protobuffs,
- [
- {description, "Google protobuffs implementation for Erlang."},
- {vsn, "0.5.0"},
- {modules, [
- pokemon_pb,
- protobuffs,
- protobuffs_compile,
- protobuffs_parser
- ]},
- {registered, []},
- {applications, [
- kernel,
- stdlib,
- sasl,
- crypto
- ]}
-]}.
View
BIN  rebar
Binary file not shown
View
4 rebar.config
@@ -1 +1,3 @@
-{erl_opts, [debug_info]}.
+{erl_opts, [debug_info]}.
+
+{clean_files, ["*~","**/*~"]}.
View
9 src/Makefile
@@ -1,9 +0,0 @@
-include ../support/include.mk
-
-all: $(EBIN_FILES)
-
-debug:
- $(MAKE) DEBUG=-DDEBUG
-
-clean:
- rm -rf $(EBIN_FILES) erl_crash.dump
View
49 src/pokemon_pb.erl
@@ -24,10 +24,13 @@
%% OTHER DEALINGS IN THE SOFTWARE.
-module(pokemon_pb).
-export([encode_pikachu/1, decode_pikachu/1]).
--compile([export_all]).
+-export([encode/1]).
-record(pikachu, {abc, def}).
%% ENCODE
+encode(Record) ->
+ encode(element(1, Record), Record).
+
encode_pikachu(Record) when is_record(Record, pikachu) ->
encode(pikachu, Record).
@@ -44,6 +47,9 @@ with_default(Val, _) -> Val.
pack(_, optional, undefined, _, _) -> [];
pack(_, repeated, undefined, _, _) -> [];
+
+pack(_, repeated_packed, undefined, _, _) -> [];
+pack(_, repeated_packed, [], _, _) -> [];
pack(FNum, required, undefined, Type, _) ->
exit({error, {required_field_is_undefined, FNum, Type}});
@@ -54,12 +60,28 @@ pack(_, repeated, [], _, Acc) ->
pack(FNum, repeated, [Head|Tail], Type, Acc) ->
pack(FNum, repeated, Tail, Type, [pack(FNum, optional, Head, Type, [])|Acc]);
+pack(FNum, repeated_packed, Data, Type, _) ->
+ protobuffs:encode_packed(FNum, Data, Type);
+
pack(FNum, _, Data, _, _) when is_tuple(Data) ->
[RecName|_] = tuple_to_list(Data),
protobuffs:encode(FNum, encode(RecName, Data), bytes);
-pack(FNum, _, Data, Type, _) ->
- protobuffs:encode(FNum, Data, Type).
+pack(FNum, _, Data, Type, _) when Type=:=bool;Type=:=int32;Type=:=uint32;
+ Type=:=int64;Type=:=uint64;Type=:=sint32;
+ Type=:=sint64;Type=:=fixed32;Type=:=sfixed32;
+ Type=:=fixed64;Type=:=sfixed64;Type=:=string;
+ Type=:=bytes;Type=:=float;Type=:=double ->
+ protobuffs:encode(FNum, Data, Type);
+
+pack(FNum, _, Data, Type, _) when is_atom(Data) ->
+ protobuffs:encode(FNum, enum_to_int(Type,Data), enum).
+
+enum_to_int(pikachu,value) ->
+ 1.
+
+int_to_enum(_,Val) ->
+ Val.
%% DECODE
decode_pikachu(Bytes) when is_binary(Bytes) ->
@@ -72,29 +94,36 @@ decode(pikachu, Bytes) when is_binary(Bytes) ->
decode(<<>>, _, Acc) -> Acc;
decode(Bytes, Types, Acc) ->
- {{FNum, WireType}, Rest} = protobuffs:read_field_num_and_wire_type(Bytes),
+ {ok, FNum} = protobuffs:next_field_num(Bytes),
case lists:keysearch(FNum, 1, Types) of
{value, {FNum, Name, Type, Opts}} ->
{Value1, Rest1} =
case lists:member(is_record, Opts) of
true ->
- {V, R} = protobuffs:decode_value(Rest, WireType, bytes),
+ {{FNum, V}, R} = protobuffs:decode(Bytes, bytes),
RecVal = decode(list_to_atom(string:to_lower(atom_to_list(Type))), V),
{RecVal, R};
false ->
- {V, R} = protobuffs:decode_value(Rest, WireType, Type),
- {unpack_value(V, Type), R}
+ case lists:member(repeated_packed, Opts) of
+ true ->
+ {{FNum, V}, R} = protobuffs:decode_packed(Bytes, Type),
+ {V, R};
+ false ->
+ {{FNum, V}, R} = protobuffs:decode(Bytes, Type),
+ {unpack_value(V, Type), R}
+ end
+
end,
case lists:member(repeated, Opts) of
true ->
case lists:keytake(FNum, 1, Acc) of
{value, {FNum, Name, List}, Acc1} ->
- decode(Rest1, Types, [{FNum, Name, lists:reverse([Value1|lists:reverse(List)])}|Acc1]);
+ decode(Rest1, Types, [{FNum, Name, lists:reverse([int_to_enum(Type,Value1)|lists:reverse(List)])}|Acc1]);
false ->
- decode(Rest1, Types, [{FNum, Name, [Value1]}|Acc])
+ decode(Rest1, Types, [{FNum, Name, [int_to_enum(Type,Value1)]}|Acc])
end;
false ->
- decode(Rest1, Types, [{FNum, Name, Value1}|Acc])
+ decode(Rest1, Types, [{FNum, Name, int_to_enum(Type,Value1)}|Acc])
end;
false ->
exit({error, {unexpected_field_index, FNum}})
View
8 src/protobuffs.app.src
@@ -0,0 +1,8 @@
+{ application, protobuffs,
+ [ { description, "Google protobuffs implementation for Erlang." },
+ { vsn, "0.6.0" },
+ { registered, [] },
+ { applications, [kernel,stdlib] },
+ { env, [] }
+ ]
+}.
View
210 src/protobuffs.erl
@@ -25,8 +25,17 @@
%%
%% @doc A protcol buffers encoding and decoding module.
-module(protobuffs).
--export([encode/3, read_field_num_and_wire_type/1, decode/2, decode_value/3]).
--compile(export_all).
+
+%% Pubic
+-export([encode/3, encode_packed/3, decode/2, decode_packed/2]).
+
+%% Used by generated *_pb file. Not intended to used by User
+-export([next_field_num/1]).
+
+%% Will be removed from export, only intended for internal usage
+-deprecated([{read_field_num_and_wire_type,1,next_version}]).
+-deprecated([{decode_value,3,next_version}]).
+-export([read_field_num_and_wire_type/1, decode_value/3]).
-define(TYPE_VARINT, 0).
-define(TYPE_64BIT, 1).
@@ -35,16 +44,47 @@
-define(TYPE_END_GROUP, 4).
-define(TYPE_32BIT, 5).
-%% @spec encode(FieldID, Value, Type) -> Result
-%% FieldID = integer()
-%% Value = any()
-%% Type = bool | enum | int32 | uint32 | int64 | unit64 | sint32 | sint64 | fixed32 | sfixed32 | fixed64 | sfixed64 | string | bytes | float | double
-%% Result = list()
+-type encoded_field_type() ::
+ ?TYPE_VARINT | ?TYPE_64BIT | ?TYPE_STRING |
+ ?TYPE_START_GROUP | ?TYPE_END_GROUP | ?TYPE_32BIT.
+
+-type field_type() :: bool | enum | int32 | uint32 | int64 |
+ unit64 | sint32 | sint64 | fixed32 |
+ sfixed32 | fixed64 | sfixed64 | string |
+ bytes | float | double.
+
+%%--------------------------------------------------------------------
%% @doc Encode an Erlang data structure into a Protocol Buffers value.
+%% @end
+%%--------------------------------------------------------------------
+-spec encode(FieldID :: non_neg_integer(),
+ Value :: any(),
+ Type :: field_type()) ->
+ binary().
+
encode(FieldID, Value, Type) ->
iolist_to_binary(encode_internal(FieldID, Value, Type)).
+
+%%--------------------------------------------------------------------
+%% @doc Encode an list of Erlang data structure into a Protocol Buffers values.
+%% @end
+%%--------------------------------------------------------------------
+-spec encode_packed(FieldID :: non_neg_integer(),
+ Values :: list(),
+ Type :: field_type()) ->
+ binary().
+encode_packed(_FieldID, [], _Type) ->
+ <<>>;
+encode_packed(FieldID, Values, Type) ->
+ PackedValues = iolist_to_binary(encode_packed_internal(Values,Type,[])),
+ Size = encode_varint(size(PackedValues)),
+ iolist_to_binary([encode_field_tag(FieldID, ?TYPE_STRING),Size,PackedValues]).
%% @hidden
+-spec encode_internal(FieldID :: non_neg_integer(),
+ Value :: any(),
+ Type :: field_type()) ->
+ iolist().
encode_internal(FieldID, false, bool) ->
encode_internal(FieldID, 0, int32);
encode_internal(FieldID, true, bool) ->
@@ -95,31 +135,134 @@ encode_internal(FieldID, Float, float) when is_integer(Float) ->
encode_internal(FieldID, Float + 0.0, float);
encode_internal(FieldID, Float, float) when is_float(Float) ->
[encode_field_tag(FieldID, ?TYPE_32BIT), <<Float:32/little-float>>];
-encode_internal(FieldID, Float, double) when is_integer(Float) ->
- encode_internal(FieldID, Float + 0.0, double);
+encode_internal(FieldID, nan, float) ->
+ [encode_field_tag(FieldID, ?TYPE_32BIT), <<0:16,192:8,255:8>>];
+encode_internal(FieldID, infinity, float) ->
+ [encode_field_tag(FieldID, ?TYPE_32BIT), <<0:16,128:8,127:8>>];
+encode_internal(FieldID, '-infinity', float) ->
+ [encode_field_tag(FieldID, ?TYPE_32BIT), <<0:16,128:8,255:8>>];
encode_internal(FieldID, Float, double) when is_float(Float) ->
- [encode_field_tag(FieldID, ?TYPE_64BIT), <<Float:64/little-float>>].
+ [encode_field_tag(FieldID, ?TYPE_64BIT), <<Float:64/little-float>>];
+encode_internal(FieldID, nan, double) ->
+ [encode_field_tag(FieldID, ?TYPE_64BIT), <<0:48,16#F8,16#FF>>];
+encode_internal(FieldID, infinity, double) ->
+ [encode_field_tag(FieldID, ?TYPE_64BIT), <<0:48,16#F0,16#7F>>];
+encode_internal(FieldID, '-infinity', double) ->
+ [encode_field_tag(FieldID, ?TYPE_64BIT), <<0:48,16#F0,16#FF>>].
+
+%% @hidden
+-spec encode_packed_internal(Values :: list(),
+ ExpectedType :: field_type(),
+ Acc :: list()) ->
+ iolist().
+encode_packed_internal([],_Type,Acc) ->
+ lists:reverse(Acc);
+encode_packed_internal([Value|Tail], ExpectedType, Acc) ->
+ [_|V] = encode_internal(1, Value, ExpectedType),
+ encode_packed_internal(Tail, ExpectedType, [V|Acc]).
+
+%%--------------------------------------------------------------------
+%% @doc Will be hidden in future releases
+%% @end
+%%--------------------------------------------------------------------
+-spec read_field_num_and_wire_type(Bytes :: binary()) ->
+ {{non_neg_integer(), encoded_field_type()}, binary()}.
read_field_num_and_wire_type(Bytes) ->
{Tag, Rest} = decode_varint(Bytes),
FieldID = Tag bsr 3,
WireType = Tag band 7,
{{FieldID, WireType}, Rest}.
-%% @spec decode(Bytes, ExpectedType) -> Result
-%% Bytes = binary()
-%% ExpectedType = bool | enum | int32 | uint32 | int64 | unit64 | sint32 | sint64 | fixed32 | sfixed32 | fixed64 | sfixed64 | string | bytes | float | double
-%% Result = {{integer(), any()}, binary()}
+%%--------------------------------------------------------------------
+%% @doc Decode a singel value from a protobuffs data structure
+%% @end
+%%--------------------------------------------------------------------
+-spec decode(Bytes :: binary(), ExpectedType :: field_type()) ->
+ {{non_neg_integer(), any()}, binary()}.
decode(Bytes, ExpectedType) ->
{{FieldID, WireType}, Rest} = read_field_num_and_wire_type(Bytes),
{Value, Rest1} = decode_value(Rest, WireType, ExpectedType),
{{FieldID, Value}, Rest1}.
-%% @hidden
+%%--------------------------------------------------------------------
+%% @doc Decode packed values from a protobuffs data structure
+%% @end
+%%--------------------------------------------------------------------
+-spec decode_packed(Bytes :: binary(), ExpectedType :: field_type()) ->
+ {{non_neg_integer(), any()}, binary()}.
+decode_packed(Bytes, ExpectedType) ->
+ {{FieldID, ?TYPE_STRING}, Rest} = read_field_num_and_wire_type(Bytes),
+ {Length, Rest1} = decode_varint(Rest),
+ {Packed,Rest2} = split_binary(Rest1, Length),
+ Values = decode_packed_values(Packed, ExpectedType, []),
+ {{FieldID, Values},Rest2}.
+
+%%--------------------------------------------------------------------
+%% @doc Returns the next field number id from a protobuffs data structure
+%% @end
+%%--------------------------------------------------------------------
+-spec next_field_num(Bytes :: binary()) ->
+ {ok,non_neg_integer()}.
+next_field_num(Bytes) ->
+ {{FieldID,_WiredType}, _Rest} = read_field_num_and_wire_type(Bytes),
+ {ok,FieldID}.
+
+%% @hidden
+-spec decode_packed_values(Bytes :: binary(),
+ Type :: field_type(),
+ Acc :: list()) ->
+ iolist().
+decode_packed_values(<<>>, _, Acc) ->
+ lists:reverse(Acc);
+decode_packed_values(Bytes, bool, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_VARINT, bool),
+ decode_packed_values(Rest, bool, [Value|Acc]);
+decode_packed_values(Bytes, enum, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_VARINT, enum),
+ decode_packed_values(Rest, bool, [Value|Acc]);
+decode_packed_values(Bytes, int32, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_VARINT, int32),
+ decode_packed_values(Rest, int32, [Value|Acc]);
+decode_packed_values(Bytes, uint32, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_VARINT, uint32),
+ decode_packed_values(Rest, uint32, [Value|Acc]);
+decode_packed_values(Bytes, sint32, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_VARINT, sint32),
+ decode_packed_values(Rest, sint32, [Value|Acc]);
+decode_packed_values(Bytes, int64, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_VARINT, int64),
+ decode_packed_values(Rest, int64, [Value|Acc]);
+decode_packed_values(Bytes, uint64, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_VARINT, uint64),
+ decode_packed_values(Rest, uint64, [Value|Acc]);
+decode_packed_values(Bytes, sint64, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_VARINT, sint64),
+ decode_packed_values(Rest, sint64, [Value|Acc]);
+decode_packed_values(Bytes, float, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_32BIT, float),
+ decode_packed_values(Rest, float, [Value|Acc]);
+decode_packed_values(Bytes, double, Acc) ->
+ {Value,Rest} = decode_value(Bytes,?TYPE_64BIT, double),
+ decode_packed_values(Rest, double, [Value|Acc]).
+
+
+%%--------------------------------------------------------------------
+%% @doc Will be hidden in future releases
+%% @end
+%%--------------------------------------------------------------------
+-spec decode_value(Bytes :: binary(),
+ WireType :: encoded_field_type(),
+ ExpectedType :: field_type()) ->
+ {any(),binary()}.
decode_value(Bytes, ?TYPE_VARINT, ExpectedType) ->
{Value, Rest} = decode_varint(Bytes),
{typecast(Value, ExpectedType), Rest};
-decode_value(Bytes, ?TYPE_STRING, ExpectedType) when ExpectedType =:= string; ExpectedType =:= bytes ->
+decode_value(Bytes, ?TYPE_STRING, string) ->
+ {Length, Rest} = decode_varint(Bytes),
+ {Value,Rest1} = split_binary(Rest, Length),
+ {binary_to_list(Value),Rest1};
+decode_value(Bytes, ?TYPE_STRING, bytes) ->
{Length, Rest} = decode_varint(Bytes),
split_binary(Rest, Length);
decode_value(<<Value:64/little-unsigned-integer, Rest/binary>>, ?TYPE_64BIT, fixed64) ->
@@ -136,12 +279,26 @@ decode_value(<<Value:32/little-signed-integer, Rest/binary>>, ?TYPE_32BIT, Type)
{Value, Rest};
decode_value(<<Value:32/little-float, Rest/binary>>, ?TYPE_32BIT, float) ->
{Value + 0.0, Rest};
+decode_value(<<0:16, 128:8, 127:8, Rest/binary>>, ?TYPE_32BIT, float) ->
+ {infinity, Rest};
+decode_value(<<0:16, 128:8, 255:8, Rest/binary>>, ?TYPE_32BIT, float) ->
+ {'-infinity', Rest};
+decode_value(<<_:16, 2#1:1, _:7, _:1, 2#1111111:7, Rest/binary>>, ?TYPE_32BIT, float) ->
+ {nan, Rest};
decode_value(<<Value:64/little-float, Rest/binary>>, ?TYPE_64BIT, double) ->
{Value + 0.0, Rest};
-decode_value(_, WireType, ExpectedType) ->
- exit({error, {unexpected_value, WireType, ExpectedType}}).
+decode_value(<<0:48, 240:8, 127:8, Rest/binary>>, ?TYPE_64BIT, double) ->
+ {infinity, Rest};
+decode_value(<<0:48, 240:8, 255:8, Rest/binary>>, ?TYPE_64BIT, double) ->
+ {'-infinity', Rest};
+decode_value(<<_:48, 2#1111:4, _:4, _:1, 2#1111111:7, Rest/binary>>, ?TYPE_64BIT, double) ->
+ {nan, Rest};
+decode_value(Value, WireType, ExpectedType) ->
+ exit({error, {unexpected_value, WireType, ExpectedType, Value}}).
%% @hidden
+-spec typecast(Value :: any(), Type :: field_type()) ->
+ any().
typecast(Value, SignedType) when SignedType =:= int32; SignedType =:= int64 ->
if
Value band 16#8000000000000000 =/= 0 -> Value - 16#10000000000000000;
@@ -149,22 +306,34 @@ typecast(Value, SignedType) when SignedType =:= int32; SignedType =:= int64 ->
end;
typecast(Value, SignedType) when SignedType =:= sint32; SignedType =:= sint64 ->
(Value bsr 1) bxor (-(Value band 1));
+typecast(Value, Type) when Type =:= bool ->
+ Value =:= 1;
typecast(Value, _) ->
Value.
%% @hidden
+-spec encode_field_tag(FieldID :: non_neg_integer(),
+ FieldType :: encoded_field_type()) ->
+ binary().
encode_field_tag(FieldID, FieldType) when FieldID band 16#3fffffff =:= FieldID ->
encode_varint((FieldID bsl 3) bor FieldType).
%% @hidden
+-spec encode_varint_field(FieldID :: non_neg_integer(),
+ Integer :: integer()) ->
+ iolist().
encode_varint_field(FieldID, Integer) ->
[encode_field_tag(FieldID, ?TYPE_VARINT), encode_varint(Integer)].
%% @hidden
+-spec encode_varint(I :: integer()) ->
+ binary().
encode_varint(I) ->
encode_varint(I, []).
%% @hidden
+-spec encode_varint(I :: integer(), Acc :: list()) ->
+ binary().
encode_varint(I, Acc) when I =< 16#7f ->
iolist_to_binary(lists:reverse([I | Acc]));
encode_varint(I, Acc) ->
@@ -174,8 +343,13 @@ encode_varint(I, Acc) ->
encode_varint(First_X_Bits, [With_Leading_Bit|Acc]).
%% @hidden
+-spec decode_varint(Bytes :: binary()) ->
+ {integer(), binary()}.
decode_varint(Bytes) ->
decode_varint(Bytes, []).
+
+-spec decode_varint(Bytes :: binary(), list()) ->
+ {integer(), binary()}.
decode_varint(<<0:1, I:7, Rest/binary>>, Acc) ->
Acc1 = [I|Acc],
Result =
View
575 src/protobuffs_compile.erl
@@ -1,4 +1,4 @@
-%% Copyright (c) 2009
+%% Copyright (c) 2009
%% Nick Gerakines <nick@gerakines.net>
%% Jacob Vorreuter <jacob.vorreuter@gmail.com>
%%
@@ -23,76 +23,216 @@
%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
%% OTHER DEALINGS IN THE SOFTWARE.
-module(protobuffs_compile).
--export([scan_file/1, output/2]).
+-export([scan_file/1, scan_file/2, generate_source/1, generate_source/2]).
-scan_file(ProtoFile) when is_list(ProtoFile) ->
+-record(collected,{enum=[], msg=[], extensions=[]}).
+
+%%--------------------------------------------------------------------
+%% @doc Generats a built .beam file and header file .hrl
+%%--------------------------------------------------------------------
+-spec scan_file(ProtoFile :: string()) ->
+ ok | {error, _}.
+scan_file(ProtoFile) ->
+ scan_file(ProtoFile,[]).
+
+%%--------------------------------------------------------------------
+%% @doc Generats a built .beam file and header file .hrl
+%% Considerd option properties: output_include_dir,
+%% output_ebin_dir,
+%% imports_dir
+%%--------------------------------------------------------------------
+-spec scan_file(ProtoFile :: string(), Options :: list()) ->
+ ok | {error, _}.
+scan_file(ProtoFile,Options) when is_list(ProtoFile) ->
+ Basename = filename:basename(ProtoFile, ".proto") ++ "_pb",
+ {ok,FirstParsed} = parse(ProtoFile),
+ ImportPaths = ["./", "src/" | proplists:get_value(imports_dir, Options, [])],
+ Parsed = parse_imports(FirstParsed, ImportPaths),
+ Collected = collect_full_messages(Parsed),
+ Messages = resolve_types(Collected#collected.msg,Collected#collected.enum),
+ output(Basename, Messages, Collected#collected.enum, Options).
+
+%%--------------------------------------------------------------------
+%% @doc Generats a source .erl file and header file .hrl
+%%--------------------------------------------------------------------
+-spec generate_source(ProtoFile :: string()) ->
+ ok | {error, _}.
+generate_source(ProtoFile) ->
+ generate_source(ProtoFile,[]).
+
+%%--------------------------------------------------------------------
+%% @doc Generats a source .erl file and header file .hrl
+%% Consider option properties: output_include_dir,
+%% output_src_dir,
+%% imports_dir
+%%--------------------------------------------------------------------
+-spec generate_source(ProtoFile :: string(), Options :: list()) ->
+ ok | {error, _}.
+generate_source(ProtoFile,Options) when is_list (ProtoFile) ->
Basename = filename:basename(ProtoFile, ".proto") ++ "_pb",
- Parsed = protobuffs_parser:parse_file(ProtoFile),
- Messages = collect_full_messages(Parsed),
- output(Basename, Messages).
+ {ok,FirstParsed} = parse(ProtoFile),
+ ImportPaths = ["./", "src/" | proplists:get_value(imports_dir, Options, [])],
+ Parsed = parse_imports(FirstParsed, ImportPaths),
+ Collected = collect_full_messages(Parsed),
+ Messages = resolve_types(Collected#collected.msg,Collected#collected.enum),
+ output_source (Basename, Messages, Collected#collected.enum, Options).
+
+%% @hidden
+parse_imports(Parsed, Path) ->
+ parse_imports(Parsed, Path, []).
+
+%% @hidden
+parse_imports([], _Path, Acc) ->
+ lists:reverse(Acc);
+parse_imports([{import, File} = Head | Tail], Path, Acc) ->
+ case file:path_open(Path, File, [read]) of
+ {ok, F, Fullname} ->
+ file:close(F),
+ {ok,FirstParsed} = parse(Fullname),
+ Parsed = lists:append(FirstParsed, Tail),
+ parse_imports(Parsed, Path, [Head | Acc]);
+ {error, Error} ->
+ error_logger:error_report([
+ "Could not do import",
+ {import, File},
+ {error, Error},
+ {path, Path}
+ ]),
+ parse_imports(Tail, Path, [Head | Acc])
+ end;
+parse_imports([Head | Tail], Path, Acc) ->
+ parse_imports(Tail, Path, [Head | Acc]).
+
+%% @hidden
+output(Basename, Messages, Enums, Options) ->
+ case proplists:get_value(output_include_dir,Options) of
+ undefined ->
+ HeaderFile = Basename ++ ".hrl";
+ HeaderPath ->
+ HeaderFile = filename:join(HeaderPath,Basename) ++ ".hrl"
+ end,
+ error_logger:info_msg("Writing header file to ~p~n",[HeaderFile]),
+ ok = write_header_include_file(HeaderFile, Messages),
+ PokemonBeamFile = filename:dirname(code:which(?MODULE)) ++ "/pokemon_pb.beam",
+ {ok,{_,[{abstract_code,{_,Forms}}]}} = beam_lib:chunks(PokemonBeamFile, [abstract_code]),
+ Forms1 = filter_forms(Messages, Enums, Forms, Basename, []),
+ {ok, _, Bytes, _Warnings} = compile:forms(Forms1, [return, debug_info]),
+ case proplists:get_value(output_ebin_dir,Options) of
+ undefined ->
+ BeamFile = Basename ++ ".beam";
+ BeamPath ->
+ BeamFile = filename:join(BeamPath,Basename) ++ ".beam"
+ end,
+ error_logger:info_msg("Writing beam file to ~p~n",[BeamFile]),
+ file:write_file(BeamFile, Bytes).
+
+%% @hidden
+output_source (Basename, Messages, Enums, Options) ->
+ case proplists:get_value(output_include_dir,Options) of
+ undefined ->
+ HeaderFile = Basename ++ ".hrl";
+ HeaderPath ->
+ HeaderFile = filename:join(HeaderPath,Basename) ++ ".hrl"
+ end,
+ error_logger:info_msg("Writing header file to ~p~n",[HeaderFile]),
+ ok = write_header_include_file(HeaderFile, Messages),
+ PokemonBeamFile = filename:dirname(code:which(?MODULE)) ++ "/pokemon_pb.beam",
+ {ok,{_,[{abstract_code,{_,Forms}}]}} = beam_lib:chunks(PokemonBeamFile, [abstract_code]),
+ Forms1 = filter_forms(Messages, Enums, Forms, Basename, []),
+ case proplists:get_value(output_src_dir,Options) of
+ undefined ->
+ SrcFile = Basename ++ ".erl";
+ SrcPath ->
+ SrcFile = filename:join(SrcPath,Basename) ++ ".erl"
+ end,
+ error_logger:info_msg("Writing src file to ~p~n",[SrcFile]),
+ file:write_file(SrcFile, erl_prettypr:format(erl_syntax:form_list (Forms1))).
+
+%% @hidden
+parse(FileName) ->
+ {ok, InFile} = file:open(FileName, [read]),
+ Acc = loop(InFile,[]),
+ file:close(InFile),
+ protobuffs_parser:parse(Acc).
-output(Basename, Messages) ->
- ok = write_header_include_file(Basename, Messages),
- BeamFile = filename:dirname(code:which(?MODULE)) ++ "/pokemon_pb.beam",
- {ok,{_,[{abstract_code,{_,Forms}}]}} = beam_lib:chunks(BeamFile, [abstract_code]),
- Forms1 = filter_forms(Messages, Forms, Basename, []),
- {ok, _, Bytes, _} = compile:forms(Forms1, [return, debug_info]),
- file:write_file(Basename ++ ".beam", Bytes).
+%% @hidden
+loop(InFile,Acc) ->
+ case io:request(InFile,{get_until,prompt,protobuffs_scanner,token,[1]}) of
+ {ok,Token,_EndLine} ->
+ loop(InFile,Acc ++ [Token]);
+ {error,token} ->
+ exit(scanning_error);
+ {eof,_} ->
+ Acc
+ end.
-filter_forms(Msgs, [{attribute,L,file,{_,_}}|Tail], Basename, Acc) ->
- filter_forms(Msgs, Tail, Basename, [{attribute,L,file,{"src/" ++ Basename ++ ".erl",L}}|Acc]);
+%% @hidden
+filter_forms(Msgs, Enums, [{attribute,L,file,{_,_}}|Tail], Basename, Acc) ->
+ filter_forms(Msgs, Enums, Tail, Basename, [{attribute,L,file,{"src/" ++ Basename ++ ".erl",L}}|Acc]);
-filter_forms(Msgs, [{attribute,L,module,pokemon_pb}|Tail], Basename, Acc) ->
- filter_forms(Msgs, Tail, Basename, [{attribute,L,module,list_to_atom(Basename)}|Acc]);
+filter_forms(Msgs, Enums, [{attribute,L,module,pokemon_pb}|Tail], Basename, Acc) ->
+ filter_forms(Msgs, Enums, Tail, Basename, [{attribute,L,module,list_to_atom(Basename)}|Acc]);
-filter_forms(Msgs, [{attribute,L,export,[{encode_pikachu,1},{decode_pikachu,1}]}|Tail], Basename, Acc) ->
+filter_forms(Msgs, Enums, [{attribute,L,export,[{encode_pikachu,1},{decode_pikachu,1}]}|Tail], Basename, Acc) ->
Exports = lists:foldl(
- fun({Name,_}, Acc1) ->
- [{list_to_atom("encode_" ++ string:to_lower(Name)),1},
- {list_to_atom("decode_" ++ string:to_lower(Name)),1} | Acc1]
- end, [], Msgs),
- filter_forms(Msgs, Tail, Basename, [{attribute,L,export,Exports}|Acc]);
+ fun({Name,_}, Acc1) ->
+ [{list_to_atom("encode_" ++ string:to_lower(Name)),1},
+ {list_to_atom("decode_" ++ string:to_lower(Name)),1} | Acc1]
+ end, [], Msgs),
+ filter_forms(Msgs, Enums, Tail, Basename, [{attribute,L,export,Exports}|Acc]);
-filter_forms(Msgs, [{attribute,L,record,{pikachu,_}}|Tail], Basename, Acc) ->
+filter_forms(Msgs, Enums, [{attribute,L,record,{pikachu,_}}|Tail], Basename, Acc) ->
Records = [begin
- OutFields = [string:to_lower(A) || {_, _, _, A, _, _} <- lists:keysort(1, Fields)],
- Frm_Fields = [{record_field,L,{atom,L,list_to_atom(OutField)}}|| OutField <- OutFields],
- {attribute, L, record, {atomize(Name), Frm_Fields}}
- end || {Name, Fields} <- Msgs],
- filter_forms(Msgs, Tail, Basename, Records ++ Acc);
+ OutFields = [string:to_lower(A) || {_, _, _, A, _} <- lists:keysort(1, Fields)],
+ Frm_Fields = [{record_field,L,{atom,L,list_to_atom(OutField)}}|| OutField <- OutFields],
+ {attribute, L, record, {atomize(Name), Frm_Fields}}
+ end || {Name, Fields} <- Msgs],
+ filter_forms(Msgs, Enums, Tail, Basename, Records ++ Acc);
-filter_forms(Msgs, [{function,L,encode_pikachu,1,[Clause]}|Tail], Basename, Acc) ->
+filter_forms(Msgs, Enums, [{function,L,encode_pikachu,1,[Clause]}|Tail], Basename, Acc) ->
Functions = [begin
- {function,L,list_to_atom("encode_" ++ string:to_lower(Name)),1,[replace_atom(Clause, pikachu, atomize(Name))]}
- end || {Name, _} <- Msgs],
- filter_forms(Msgs, Tail, Basename, Functions ++ Acc);
+ {function,L,list_to_atom("encode_" ++ string:to_lower(Name)),1,[replace_atom(Clause, pikachu, atomize(Name))]}
+ end || {Name, _} <- Msgs],
+ filter_forms(Msgs, Enums, Tail, Basename, Functions ++ Acc);
-filter_forms(Msgs, [{function,L,encode,2,[Clause]}|Tail], Basename, Acc) ->
- filter_forms(Msgs, Tail, Basename, [expand_encode_function(Msgs, L, Clause)|Acc]);
+filter_forms(Msgs, Enums, [{function,L,encode,2,[Clause]}|Tail], Basename, Acc) ->
+ filter_forms(Msgs, Enums, Tail, Basename, [expand_encode_function(Msgs, L, Clause)|Acc]);
-filter_forms(Msgs, [{function,L,iolist,2,[Clause]}|Tail], Basename, Acc) ->
- filter_forms(Msgs, Tail, Basename, [expand_iolist_function(Msgs, L, Clause)|Acc]);
+ filter_forms(Msgs, Enums, [{function,L,iolist,2,[Clause]}|Tail], Basename, Acc) ->
+ filter_forms(Msgs, Enums, Tail, Basename, [expand_iolist_function(Msgs, L, Clause)|Acc]);
-filter_forms(Msgs, [{function,L,decode_pikachu,1,[Clause]}|Tail], Basename, Acc) ->
+filter_forms(Msgs, Enums, [{function,L,decode_pikachu,1,[Clause]}|Tail], Basename, Acc) ->
Functions = [begin
- {function,L,list_to_atom("decode_" ++ string:to_lower(Name)),1,[replace_atom(Clause, pikachu, atomize(Name))]}
- end || {Name, _} <- Msgs],
- filter_forms(Msgs, Tail, Basename, Functions ++ Acc);
+ {function,
+ L,
+ list_to_atom("decode_" ++ string:to_lower(Name)),
+ 1,
+ [replace_atom(Clause, pikachu, atomize(Name))]}
+ end || {Name, _} <- Msgs],
+ filter_forms(Msgs, Enums, Tail, Basename, Functions ++ Acc);
+
+filter_forms(Msgs, Enums, [{function,L,decode,2,[Clause]}|Tail], Basename, Acc) ->
+ filter_forms(Msgs, Enums, Tail, Basename, [expand_decode_function(Msgs, L, Clause)|Acc]);
+
+filter_forms(Msgs, Enums, [{function,L,to_record,2,[Clause]}|Tail], Basename, Acc) ->
+ filter_forms(Msgs, Enums, Tail, Basename, [expand_to_record_function(Msgs, L, Clause)|Acc]);
-filter_forms(Msgs, [{function,L,decode,2,[Clause]}|Tail], Basename, Acc) ->
- filter_forms(Msgs, Tail, Basename, [expand_decode_function(Msgs, L, Clause)|Acc]);
+filter_forms(Msgs, Enums, [{function,L,enum_to_int,2,[Clause]}|Tail], Basename, Acc) ->
+ filter_forms(Msgs, Enums, Tail, Basename, [expand_enum_to_int_function(Enums, L, Clause)|Acc]);
-filter_forms(Msgs, [{function,L,to_record,2,[Clause]}|Tail], Basename, Acc) ->
- filter_forms(Msgs, Tail, Basename, [expand_to_record_function(Msgs, L, Clause)|Acc]);
+filter_forms(Msgs, Enums, [{function,L,int_to_enum,2,[Clause]}|Tail], Basename, Acc) ->
+ filter_forms(Msgs, Enums, Tail, Basename, [expand_int_to_enum_function(Enums, L, Clause)|Acc]);
-filter_forms(Msgs, [Form|Tail], Basename, Acc) ->
- filter_forms(Msgs, Tail, Basename, [Form|Acc]);
+filter_forms(Msgs, Enums, [Form|Tail], Basename, Acc) ->
+ filter_forms(Msgs, Enums, Tail, Basename, [Form|Acc]);
-filter_forms(_, [], _, Acc) -> lists:reverse(Acc).
+filter_forms(_, _, [], _, Acc) -> lists:reverse(Acc).
+%% @hidden
expand_encode_function(Msgs, Line, Clause) ->
{function,Line,encode,2,[filter_encode_clause(Msg, Clause) || Msg <- Msgs]}.
+%% @hidden
filter_encode_clause({MsgName, _Fields}, {clause,L,_Args,Guards,_Content}) ->
ToBin = {call,L,{atom,L,iolist_to_binary},[{call,L,
{atom,L,iolist},
@@ -104,33 +244,44 @@ expand_iolist_function(Msgs, Line, Clause) ->
filter_iolist_clause({MsgName, Fields}, {clause,L,_Args,Guards,_Content}) ->
Cons = lists:foldl(
- fun({FNum,Tag,SType,SName,_,Default}, Acc) ->
- {cons,L,
- {call,L,{atom,L,pack},[
- {integer,L,FNum},
- {atom,L,Tag},
- {call,L,{atom,L,with_default},[{record_field,L,{var,L,'Record'},atomize(MsgName),{atom,L,atomize(SName)}},erl_parse:abstract(Default)]},
- {atom,L,atomize(SType)},
- {nil,L}]},
- Acc}
- end, {nil,L}, Fields),
+ fun({FNum,Tag,SType,SName,Default}, Acc) ->
+ {cons,L,
+ {call,L,{atom,L,pack},[{integer,L,FNum},
+ {atom,L,Tag},
+ {call,L,
+ {atom,L,with_default},
+ [{record_field,L,
+ {var,L,'Record'},atomize(MsgName),
+ {atom,L,atomize(SName)}},
+ erl_parse:abstract(Default)]},
+ {atom,L,atomize(SType)},
+ {nil,L}]},
+ Acc}
+ end, {nil,L}, Fields),
{clause,L,[{atom,L,atomize(MsgName)},{var,L,'Record'}],Guards,[Cons]}.
+%% @hidden
expand_decode_function(Msgs, Line, Clause) ->
- {function,Line,decode,2,[filter_decode_clause(Msgs, Msg, Clause) || Msg <- Msgs]}.
+ {function,Line,decode,2, [{clause,Line,[{atom,Line,enummsg_values},{integer,Line,1}],[],[{atom,Line,value1}]}] ++
+ [filter_decode_clause(Msgs, Msg, Clause) || Msg <- Msgs]}.
+%% @hidden
filter_decode_clause(Msgs, {MsgName, Fields}, {clause,L,_Args,Guards,[_,B,C]}) ->
- Types = lists:keysort(1, [{FNum, list_to_atom(SName), list_to_atom(SType), decode_opts(Msgs, Tag, SType)} || {FNum,Tag,SType,SName,_,_} <- Fields]),
+ Types = lists:keysort(1, [{FNum, list_to_atom(SName),
+ atomize(SType),
+ decode_opts(Msgs, Tag, SType)} ||
+ {FNum,Tag,SType,SName,_} <- Fields]),
Cons = lists:foldl(
- fun({FNum, FName, Type, Opts}, Acc) ->
- {cons,L,{tuple,L,[{integer,L,FNum},{atom,L,FName},{atom,L,Type},erl_parse:abstract(Opts)]},Acc}
- end, {nil,L}, Types),
+ fun({FNum, FName, Type, Opts}, Acc) ->
+ {cons,L,{tuple,L,[{integer,L,FNum},{atom,L,FName},{atom,L,Type},erl_parse:abstract(Opts)]},Acc}
+ end, {nil,L}, Types),
A = {match,L,{var,L,'Types'},Cons},
C1 = replace_atom(C, pikachu, atomize(MsgName)),
{clause,L,[{atom,L,atomize(MsgName)},{var,L,'Bytes'}],Guards,[A,B,C1]}.
+%% @hidden
decode_opts(Msgs, Tag, Type) ->
- Opts0 = if Tag == repeated -> [repeated]; true -> [] end,
+ Opts0 = if Tag == repeated -> [repeated]; Tag == repeated_packed -> [repeated_packed]; true -> [] end,
case lists:keymember(Type, 1, Msgs) of
true ->
[is_record|Opts0];
@@ -138,62 +289,298 @@ decode_opts(Msgs, Tag, Type) ->
Opts0
end.
+%% @hidden
expand_to_record_function(Msgs, Line, Clause) ->
{function,Line,to_record,2,[filter_to_record_clause(Msg, Clause) || Msg <- Msgs]}.
+%% @hidden
filter_to_record_clause({MsgName, _}, {clause,L,[_Param1,Param2],Guards,[Fold]}) ->
Fold1 = replace_atom(Fold, pikachu, atomize(MsgName)),
{clause,L,[{atom,L,atomize(MsgName)},Param2],Guards,[Fold1]}.
+%% @hidden
+expand_enum_to_int_function([], Line, Clause) ->
+ {function,Line,enum_to_int,2,[Clause]};
+expand_enum_to_int_function(Enums, Line, Clause) ->
+ {function,Line,enum_to_int,2,[filter_enum_to_int_clause(Enum, Clause) || Enum <- Enums]}.
+
+%% @hidden
+filter_enum_to_int_clause({enum,EnumTypeName,IntValue,EnumValue}, {clause,L,_Args,Guards,_}) ->
+ {clause,L,[{atom,L,atomize(EnumTypeName)},{atom,L,EnumValue}],Guards,[{integer,L,IntValue}]}.
+
+%% @hidden
+expand_int_to_enum_function([], Line, Clause) ->
+ {function,Line,int_to_enum,2,[Clause]};
+expand_int_to_enum_function(Enums, Line, Clause) ->
+ {function,Line,int_to_enum,2, [filter_int_to_enum_clause(Enum, Clause) || Enum <- Enums] ++ [Clause]}.
+
+%% @hidden
+filter_int_to_enum_clause({enum,EnumTypeName,IntValue,EnumValue}, {clause,L,_Args,Guards,_}) ->
+ {clause,L,[{atom,L,atomize(EnumTypeName)},{integer,L,IntValue}],Guards,[{atom,L,EnumValue}]}.
+
+%% @hidden
%% [{"Location",
-%% [{2,required,"string","country",number,none},
-%% {1,required,"string","region",number,none}]},
+%% [{2,required,"string","country",none},
+%% {1,required,"string","region",none}]},
%% {"Person",
-%% [{5,optional,"Location","location",number,none},
-%% {4,required,"int32","age",number,none},
-%% {3,required,"string","phone_number",number,none},
-%% {2,required,"string","address",number,none},
-%% {1,required,"string","name",number,none}]}]
-collect_full_messages(Data) -> collect_full_messages(Data, []).
-collect_full_messages([{message, Name, Fields} | Tail], Acc) ->
+%% [{5,optional,"Location","location",none},
+%% {4,required,"int32","age",none},
+%% {3,required,"string","phone_number",none},
+%% {2,required,"string","address",none},
+%% {1,required,"string","name",none}]}]
+collect_full_messages(Data) -> collect_full_messages(Data, #collected{}).
+collect_full_messages([{message, Name, Fields} | Tail], Collected) ->
+ ListName = case erlang:is_list (hd(Name)) of
+ true -> Name;
+ false -> [Name]
+ end,
+
FieldsOut = lists:foldl(
- fun (Input, TmpAcc) ->
- case Input of
- {_, _, _, _, _, _} -> [Input | TmpAcc];
- _ -> TmpAcc
- end
- end, [], Fields),
+ fun ({_,_,_,_,_} = Input, TmpAcc) -> [Input | TmpAcc];
+ (_, TmpAcc) -> TmpAcc
+ end, [], Fields),
+
+ Enums = lists:foldl(
+ fun ({enum,C,D}, TmpAcc) -> [{enum, [C | ListName], D} | TmpAcc];
+ (_, TmpAcc) -> TmpAcc
+ end, [], Fields),
+
+ Extensions = lists:foldl(
+ fun ({extensions, From, To}, TmpAcc) -> [{From,To}|TmpAcc];
+ (_, TmpAcc) -> TmpAcc
+ end, [], Fields),
+
SubMessages = lists:foldl(
- fun ({message, C, D}, TmpAcc) -> [{message, C, D} | TmpAcc];
- (_, TmpAcc) -> TmpAcc
- end, [], Fields),
- collect_full_messages(Tail ++ SubMessages, [{Name, FieldsOut} | Acc]);
-collect_full_messages([{package, _Line1}, {bareword, _Line2, _PackageName}, {';', _Line3} | Tail], Acc) ->
- collect_full_messages(Tail, Acc);
+ fun ({message, C, D}, TmpAcc) -> [{message, [C | ListName], D} | TmpAcc];
+ (_, TmpAcc) -> TmpAcc
+ end, [], Fields),
+
+ NewCollected = Collected#collected{
+ msg=[{ListName, FieldsOut} | Collected#collected.msg],
+ extensions=[{ListName,Extensions} | Collected#collected.extensions]
+ },
+ collect_full_messages(Tail ++ SubMessages ++ Enums, NewCollected);
+collect_full_messages([{enum, Name, Fields} | Tail], Collected) ->
+ ListName = case erlang:is_list (hd(Name)) of
+ true -> Name;
+ false -> [Name]
+ end,
+
+ FieldsOut = lists:foldl(
+ fun (Field, TmpAcc) ->
+ case Field of
+ {EnumAtom, IntValue} -> [{enum,
+ type_path_to_type(ListName),
+ IntValue,
+ EnumAtom} | TmpAcc];
+ _ -> TmpAcc
+ end
+ end, [], Fields),
+
+ NewCollected = Collected#collected{enum=FieldsOut++Collected#collected.enum},
+ collect_full_messages(Tail, NewCollected);
+collect_full_messages([{package, _PackageName} | Tail], Collected) ->
+ collect_full_messages(Tail, Collected);
+collect_full_messages([{option,_,_} | Tail], Collected) ->
+ collect_full_messages(Tail, Collected);
+collect_full_messages([{import, _Filename} | Tail], Collected) ->
+ collect_full_messages(Tail, Collected);
+collect_full_messages([{extend, Name, ExtendedFields} | Tail], Collected) ->
+ ListName = case erlang:is_list (hd(Name)) of
+ true -> Name;
+ false -> [Name]
+ end,
+
+ CollectedMsg = Collected#collected.msg,
+ {ListName,FieldsOut} = lists:keyfind(ListName,1,CollectedMsg),
+ {ListName,Extensions} = lists:keyfind(ListName,1,Collected#collected.extensions),
+
+ FunNotInReservedRange = fun(Id) -> not(19000 =< Id andalso Id =< 19999) end,
+ FunInRange = fun(Id,From,max) -> From =< Id andalso Id =< 16#1fffffff;
+ (Id,From,To) -> From =< Id andalso Id =< To
+ end,
+
+ ExtendedFieldsOut = lists:append(FieldsOut,
+ lists:foldl(
+ fun ({Id, _, _, FieldName, _} = Input, TmpAcc) ->
+ case lists:any(fun({From,To}) -> FunNotInReservedRange(Id)
+ andalso FunInRange(Id,From,To)
+ end,Extensions) of
+ true ->
+ [Input | TmpAcc];
+ _ ->
+ error_logger:error_report(["Extended field not in valid range",
+ {message, Name},
+ {field_id,Id},
+ {field_name,FieldName},
+ {defined_ranges,Extensions},
+ {reserved_range,{19000,19999}},
+ {max,16#1fffffff}]),
+ throw(out_of_range)
+ end;
+ (_, TmpAcc) -> TmpAcc
+ end, [], ExtendedFields)
+ ),
+ NewCollected = Collected#collected{msg=lists:keyreplace(ListName,1,CollectedMsg,{ListName,ExtendedFieldsOut})},
+ collect_full_messages(Tail, NewCollected);
%% Skip anything we don't understand
collect_full_messages([_|Tail], Acc) ->
- collect_full_messages(Tail, Acc);
-collect_full_messages([], Acc) ->
+ collect_full_messages(Tail, Acc);
+collect_full_messages([], Collected) ->
+ Collected.
+
+%% @hidden
+resolve_types (Data, Enums) -> resolve_types (Data, Data, Enums, []).
+resolve_types ([{TypePath, Fields} | Tail], AllPaths, Enums, Acc) ->
+ FieldsOut = lists:foldl(
+ fun (Input, TmpAcc) ->
+ case Input of
+ {Index, Rules, Type, Identifier, Other} ->
+ case is_scalar_type (Type) of
+ true -> [Input | TmpAcc];
+ false ->
+ PossiblePaths =
+ case string:tokens (Type,".") of
+ [Type] ->
+ all_possible_type_paths (Type, TypePath);
+ FullPath ->
+ % handle types of the form Foo.Bar which are absolute,
+ % so we just convert to a type path and check it.
+ [lists:reverse (FullPath)]
+ end,
+ RealPath =
+ case find_type (PossiblePaths, AllPaths) of
+ false ->
+ case is_enum_type(Type, PossiblePaths, Enums) of
+ {true,EnumType} ->
+ EnumType;
+ false ->
+ throw (["Unknown Type ", Type])
+ end;
+ ResultType ->
+ ResultType
+ end,
+ [{Index, Rules, type_path_to_type (RealPath), Identifier, Other} | TmpAcc]
+ end;
+ _ -> TmpAcc
+ end
+ end, [], Fields),
+ resolve_types (Tail, AllPaths, Enums, [{type_path_to_type (TypePath), lists:reverse (FieldsOut) } | Acc]);
+resolve_types ([], _, _, Acc) ->
Acc.
+%% @hidden
write_header_include_file(Basename, Messages) ->
- {ok, FileRef} = file:open(Basename ++ ".hrl", [write]),
+ {ok, FileRef} = file:open(Basename, [write]),
[begin
- OutFields = [string:to_lower(A) || {_, _, _, A, _, _} <- lists:keysort(1, Fields)],
- io:format(FileRef, "-record(~s, {~s}).~n", [string:to_lower(Name), string:join(OutFields, ", ")])
- end || {Name, Fields} <- Messages],
+ OutFields = [{string:to_lower(A), Optional, Default} || {_, Optional, _, A, Default} <- lists:keysort(1, Fields)],
+ if
+ OutFields /= [] ->
+ io:format(FileRef, "-record(~s, {~n ", [string:to_lower(Name)]),
+ WriteFields = generate_field_definitions(OutFields),
+ FormatString = string:join(["~s" || _ <- lists:seq(1, length(WriteFields))], ",~n "),
+ io:format(FileRef, FormatString, WriteFields),
+ io:format(FileRef, "~n}).~n~n", []);
+ true ->
+ ok
+ end
+ end || {Name, Fields} <- Messages],
file:close(FileRef).
+%% @hidden
+generate_field_definitions(Fields) ->
+ generate_field_definitions(Fields, []).
+
+%% @hidden
+generate_field_definitions([], Acc) ->
+ lists:reverse(Acc);
+generate_field_definitions([{Name, required, _} | Tail], Acc) ->
+ Head = lists:flatten(io_lib:format("~s = erlang:error({required, ~s})", [Name, Name])),
+ generate_field_definitions(Tail, [Head | Acc]);
+generate_field_definitions([{Name, _, none} | Tail], Acc) ->
+ Head = lists:flatten(io_lib:format("~s", [Name])),
+ generate_field_definitions(Tail, [Head | Acc]);
+generate_field_definitions([{Name, optional, Default} | Tail], Acc) ->
+ Head = lists:flatten(io_lib:format("~s = ~p", [Name, Default])),
+ generate_field_definitions(Tail, [Head | Acc]).
+
+%% @hidden
atomize(String) ->
list_to_atom(string:to_lower(String)).
+%% @hidden
replace_atom(Find, Find, Replace) -> Replace;
-
replace_atom(Tuple, Find, Replace) when is_tuple(Tuple) ->
list_to_tuple([replace_atom(Term, Find, Replace) || Term <- tuple_to_list(Tuple)]);
-
replace_atom(List, Find, Replace) when is_list(List) ->
[replace_atom(Term, Find, Replace) || Term <- List];
-
replace_atom(Other, _Find, _Replace) ->
Other.
+
+%% @hidden
+is_scalar_type ("double") -> true;
+is_scalar_type ("float") -> true;
+is_scalar_type ("int32") -> true;
+is_scalar_type ("int64") -> true;
+is_scalar_type ("uint32") -> true;
+is_scalar_type ("uint64") -> true;
+is_scalar_type ("sint32") -> true;
+is_scalar_type ("sint64") -> true;
+is_scalar_type ("fixed32") -> true;
+is_scalar_type ("fixed64") -> true;
+is_scalar_type ("sfixed32") -> true;
+is_scalar_type ("sfixed64") -> true;
+is_scalar_type ("bool") -> true;
+is_scalar_type ("string") -> true;
+is_scalar_type ("bytes") -> true;
+is_scalar_type (_) -> false.
+
+%% @hidden
+is_enum_type(_Type, [], _Enums) ->
+ false;
+is_enum_type(Type, [TypePath|Paths], Enums) ->
+ case is_enum_type(type_path_to_type(TypePath), Enums) of
+ true ->
+ {true,TypePath};
+ false ->
+ is_enum_type(Type, Paths, Enums)
+ end.
+is_enum_type(Type, Enums) ->
+ case lists:keysearch(Type,2,Enums) of
+ false ->
+ false;
+ {value,_} ->
+ true
+ end.
+
+%% @hidden
+sublists(List) when is_list(List) ->
+ sublists(List,[]).
+sublists([],Acc) ->
+ [ [] | Acc ];
+sublists(List,Acc) ->
+ sublists (tl (List), [ List | Acc ]).
+
+%% @hidden
+all_possible_type_paths (Type, TypePath) ->
+ lists:foldl (fun (TypeSuffix, AccIn) ->
+ [[Type | TypeSuffix] | AccIn]
+ end,
+ [],
+ sublists (TypePath)).
+
+%% @hidden
+find_type ([], _KnownTypes) ->
+ false;
+find_type ([Type | TailTypes], KnownTypes) ->
+ case lists:keysearch (Type, 1, KnownTypes) of
+ false ->
+ find_type (TailTypes, KnownTypes);
+ {value, {RealType, _}} ->
+ RealType
+ end.
+
+%% @hidden
+type_path_to_type (TypePath) ->
+ string:join (lists:reverse (TypePath), "_").
+
View
247 src/protobuffs_parser.erl
@@ -1,247 +0,0 @@
-%% Copyright (c) 2009
-%% Nick Gerakines <nick@gerakines.net>
-%% Jacob Vorreuter <jacob.vorreuter@gmail.com>
-%%
-%% Permission is hereby granted, free of charge, to any person
-%% obtaining a copy of this software and associated documentation
-%% files (the "Software"), to deal in the Software without
-%% restriction, including without limitation the rights to use,
-%% copy, modify, merge, publish, distribute, sublicense, and/or sell
-%% copies of the Software, and to permit persons to whom the
-%% Software is furnished to do so, subject to the following
-%% conditions:
-%%
-%% The above copyright notice and this permission notice shall be
-%% included in all copies or substantial portions of the Software.
-%%
-%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-%% OTHER DEALINGS IN THE SOFTWARE.
--module(protobuffs_parser).
-
--export([parse_file/1]).
-
-parse_file(Filename) ->
- {ok, Data} = file:read_file(Filename),
- Raw = scan(binary_to_list(Data)),
- parse(Raw).
-
-%% @hidden
-parse(Data) -> parse(Data, []).
-
-%% @hidden
-parse([], Acc) -> lists:reverse(Acc);
-parse([{'}', _Line} | Tail], Acc) -> {Acc, Tail};
-parse([{enum, _Line}, {bareword, _Line, MessageName}, {'{', _Line} | Tail], Acc) ->
- {Res, Tail2} = parse(Tail, []),
- parse(Tail2, [{enum, MessageName, lists:reverse(Res)} | Acc]);
-parse([{message, _Line}, {bareword, _Line, MessageName}, {'{', _Line} | Tail], Acc) ->
- {Res, Tail2} = parse(Tail, []),
- parse(Tail2, [{message, MessageName, lists:reverse(Res)} | Acc]);
-parse([{bareword, _Line, FieldName}, {'=', _Line}, {number, _Line, Value}, {';', _Line} | Tail], Acc) ->
- parse(Tail, [{enum, Value, FieldName} | Acc]);
-parse([{Type, _Line}, {bareword, _Line, Field}, {bareword, _Line, FieldName}, {'=', _Line}, {FieldType, _Line, Position}, {'[', _Line}, {bareword, _Line,"default"}, {'=', _Line}, {_DefaultType, _Line, Default}, {']', _Line}, {';', _Line} | Tail], Acc) ->
- parse(Tail, [{Position, Type, Field, FieldName, FieldType, Default} | Acc]);
-parse([{Type, _Line}, {bareword, _Line, Field}, {bareword, _Line, FieldName}, {'=', _Line}, {FieldType, _Line, Position}, {';', _Line} | Tail], Acc) ->
- parse(Tail, [{Position, Type, Field, FieldName, FieldType, none} | Acc]);
-parse([{'$end', _} | Tail], Acc) ->
- parse(Tail, Acc);
-parse([Head | Tail], Acc) ->
- parse(Tail, [Head | Acc]).
-
-scan(String) ->
- scan(String, [], 1).
-
-%% @hidden
-scan([${|Rest], Accum, Line) ->
- scan(Rest, [{'{', Line}|Accum], Line);
-scan([$}|Rest], Accum, Line) ->
- scan(Rest, [{'}', Line}|Accum], Line);
-scan([$[|Rest], Accum, Line) ->
- scan(Rest, [{'[', Line}|Accum], Line);
-scan([$]|Rest], Accum, Line) ->
- scan(Rest, [{']', Line}|Accum], Line);
-scan([$(|Rest], Accum, Line) ->
- scan(Rest, [{'(', Line}|Accum], Line);
-scan([$)|Rest], Accum, Line) ->
- scan(Rest, [{')', Line}|Accum], Line);
-scan([$=|Rest], Accum, Line) ->
- scan(Rest, [{'=', Line}|Accum], Line);
-scan([$;|Rest], Accum, Line) ->
- scan(Rest, [{';', Line}|Accum], Line);
-scan([$,|Rest], Accum, Line) ->
- scan(Rest, [{',', Line}|Accum], Line);
-scan([Digit|_] = String, Accum, Line)
- when Digit >= $0, Digit =< $9 ->
- {Number, Rest} = scan_number(String),
- scan(Rest, [{number, Line, Number}|Accum], Line);
-scan([$-, Digit|_] = String, Accum, Line)
- when Digit >= $0, Digit =< $9 ->
- {Number, Rest} = scan_number(tl(String)),
- scan(Rest, [{number, Line, -Number}|Accum], Line);
-scan([$\n|Rest], Accum, Line) ->
- scan(Rest, Accum, Line + 1);
-scan([WS|Rest], Accum, Line)
- when WS =:= 32; WS =:= $\t ->
- scan(Rest, Accum, Line);
-scan([$/, $/|Rest], Accum, Line) ->
- scan(skip_to_newline(Rest), Accum, Line);
-scan([$/, $*|Rest], Accum, Line) ->
- {Rest1, Line1} = skip_comment(Rest, Line),
- scan(Rest1, Accum, Line1);
-scan([$"|_] = String, Accum, Line) ->
- {Strval, Rest, Line1} = scan_string(String, Line),
- scan(Rest, [{string, Line, Strval}|Accum], Line1);
-scan([C|_] = String, Accum, Line)
- when C >= $A, C =< $Z;
- C >= $a, C =< $z;
- C =:= $_ ->
- {Identifier, Rest} = scan_identifier(String),
- Token = case get_keyword(Identifier) of
- Keyword when is_atom(Keyword) ->
- {Keyword, Line};
- {bareword, Bareword} ->
- {bareword, Line, Bareword}
- end,
- scan(Rest, [Token|Accum], Line);
-scan([], Accum, Line) ->
- lists:reverse([{'$end', Line}|Accum]);
-scan([C|_], _Accum, Line) ->
- erlang:error({invalid_character, [C], Line}).
-
-%% @hidden
-scan_identifier(String) ->
- scan_identifier(String, "").
-
-%% @hidden
-scan_identifier([C|Rest], Accum)
- when C >= $A, C =< $Z;
- C >= $a, C =< $z;
- C >= $0, C =< $9;
- C =:= $_;
- C =:= $. ->
- scan_identifier(Rest, [C|Accum]);
-scan_identifier(Rest, Accum) ->
- {lists:reverse(Accum), Rest}.
-
-%% @hidden
-scan_number(String) ->
- {A, Rest1} = scan_integer(String),
- case Rest1 of
- [$.|Fraction] ->
- {B, Rest2} = scan_identifier(Fraction),
- {A + list_to_float("0." ++ B), Rest2};
- [$e|Exp] ->
- {B, Rest2} = scan_integer(Exp),
- {list_to_float(integer_to_list(A) ++ ".0e" ++ integer_to_list(B)), Rest2};
- [$x|Rest] when A =:= 0 ->
- {Hex, Rest2} = scan_identifier(Rest),
- {erlang:list_to_integer(Hex, 16), Rest2};
- _ ->
- {A, Rest1}
- end.
-
-%% @hidden
-scan_integer(String) ->
- scan_integer(String, 0).
-
-%% @hidden
-scan_integer([D|Rest], Accum)
- when D >= $0, D =< $9 ->
- scan_integer(Rest, Accum * 10 + (D - $0));
-scan_integer(Rest, Accum) ->
- {Accum, Rest}.
-
-%% @hidden
-scan_string([$"|String], Line) ->
- scan_string(String, "", Line).
-
-%% @hidden
-scan_string([$"|Rest], Accum, Line) ->
- {lists:reverse(Accum), Rest, Line};
-scan_string([$\\, $a|Rest], Accum, Line) ->
- scan_string(Rest, [7|Accum], Line);
-scan_string([$\\, $e|Rest], Accum, Line) ->
- scan_string(Rest, [$\e|Accum], Line);
-scan_string([$\\, $f|Rest], Accum, Line) ->
- scan_string(Rest, [$\f|Accum], Line);
-scan_string([$\\, $n|Rest], Accum, Line) ->
- scan_string(Rest, [$\n|Accum], Line);
-scan_string([$\\, $r|Rest], Accum, Line) ->
- scan_string(Rest, [$\r|Accum], Line);
-scan_string([$\\, $t|Rest], Accum, Line) ->
- scan_string(Rest, [$\t|Accum], Line);
-scan_string([$\\, $v|Rest], Accum, Line) ->
- scan_string(Rest, [$\v|Accum], Line);
-scan_string([$\\, D1, D2, D3|Rest], Accum, Line)
- when D1 >= $0, D1 =< $7, D2 >= $0, D2 =< $7, D3 >= $0, D3 =< $7 ->
- scan_string(Rest, [erlang:list_to_integer([D1, D2, D3], 8)|Accum], Line);
-scan_string([$\\, $x, H1, H2|Rest], Accum, Line) ->
- scan_string(Rest, [erlang:list_to_integer([H1, H2], 16)|Accum], Line);
-scan_string([$\\, Char|Rest], Accum, Line) ->
- scan_string(Rest, [Char|Accum], Line);
-scan_string([$\n|Rest], Accum, Line) ->
- scan_string(Rest, [$\n|Accum], Line + 1);
-scan_string([Char|Rest], Accum, Line) ->
- scan_string(Rest, [Char|Accum], Line).
-
-%% @hidden
-skip_to_newline([$\n|Rest]) ->
- Rest;
-skip_to_newline([]) ->
- [];
-skip_to_newline([_|Rest]) ->
- skip_to_newline(Rest).
-
-%% @hidden
-skip_comment([$*, $/|Rest], Line) ->
- {Rest, Line};
-skip_comment([$\n|Rest], Line) ->
- skip_comment(Rest, Line + 1);
-skip_comment([_|Rest], Line) ->
- skip_comment(Rest, Line).
-
-%% @hidden
-get_keyword("import") ->
- import;
-get_keyword("package") ->
- package;
-get_keyword("option") ->
- option;
-get_keyword("message") ->
- message;
-get_keyword("group") ->
- group;
-get_keyword("enum") ->
- enum;
-get_keyword("extend") ->
- extend;
-get_keyword("service") ->
- service;
-get_keyword("rpc") ->
- rpc;
-get_keyword("required") ->
- required;
-get_keyword("optional") ->
- optional;
-get_keyword("repeated") ->
- repeated;
-get_keyword("returns") ->
- returns;
-get_keyword("extensions") ->
- extensions;
-get_keyword("max") ->
- max;
-get_keyword("to") ->
- to;
-get_keyword("true") ->
- true;
-get_keyword("false") ->
- false;
-get_keyword(Bareword) ->
- {bareword, Bareword}.
View
69 src/protobuffs_parser.yrl
@@ -0,0 +1,69 @@
+Nonterminals
+g_protobuffs g_header g_message g_rpcs g_rpc g_element g_elements g_var g_value g_default.
+
+Terminals ';' '=' '{' '}' '[' ']' '(' ')' string integer float var.
+
+Rootsymbol g_protobuffs.
+Endsymbol '$end'.
+
+g_protobuffs -> '$empty' : [].
+g_protobuffs -> g_header g_protobuffs : ['$1'|'$2'].
+g_protobuffs -> g_message g_protobuffs : ['$1'|'$2'].
+
+g_header -> g_var string ';' : {'$1', unwrap('$2')}.
+g_header -> g_var g_var ';' : {'$1', safe_string('$2')}.
+g_header -> g_var g_var '=' g_value ';' : {'$1', '$2', '$4'}.
+
+g_message -> g_var g_var '{' g_elements '}' : {'$1', safe_string('$2'), '$4'}.
+g_message -> g_var g_var '{' g_rpcs '}' : {'$1', safe_string('$2'), '$4'}.
+
+g_rpcs -> g_rpc : ['$1'].
+g_rpcs -> g_rpc g_rpcs : ['$1' | '$2'].
+
+g_rpc -> g_var g_var '(' g_var ')' g_var '(' g_var ')' ';' : {'$1', safe_string('$2'), safe_string('$4'), safe_string('$8')}.
+
+g_elements -> g_element : ['$1'].
+g_elements -> g_element g_elements : ['$1' | '$2'].
+
+g_element -> g_var g_var g_var '=' integer g_default ';' : {unwrap('$5'), pack_repeated('$1','$6'), safe_string('$2'), safe_string('$3'), default('$6')}.
+g_element -> g_var '=' integer ';' : {'$1', unwrap('$3')}.
+g_element -> g_var integer g_var integer ';' : {'$1', unwrap('$2'), unwrap('$4')}.
+g_element -> g_var integer g_var g_var ';' : {'$1', unwrap('$2'), '$4'}.
+g_element -> g_var g_var '=' g_value ';' : {'$1', '$2', '$4'}.
+g_element -> g_message : '$1'.
+
+g_var -> var : unwrap('$1').
+
+g_value -> g_var : '$1'.
+g_value -> integer : unwrap('$1').
+g_value -> string : unwrap('$1').
+g_value -> float : unwrap('$1').
+
+g_default -> '$empty' : none.
+g_default -> '[' g_var '=' g_value ']' : {'$2', '$4'}.
+
+Erlang code.
+safe_string(A) -> make_safe(atom_to_list(A)).
+
+reserved_words() ->
+ ["after", "and", "andalso", "band", "begin", "bnot", "bor", "bsl", "bsr", "bxor", "case", "catch", "cond", "div", "end", "fun",
+ "if", "let", "not", "of", "or", "orelse", "query", "receive", "rem", "try", "when", "xor"].
+
+make_safe(String) ->
+ case lists:any(fun(Elem) -> string:equal(String,Elem) end, reserved_words()) of
+ true -> "pb_"++String;
+ false -> String
+ end.
+
+unwrap({_,_,V}) -> V;
+unwrap({V,_}) -> V.
+
+default({default,D}) ->
+ D;
+default(_) ->
+ none.
+
+pack_repeated(repeated,{packed,true}) ->
+ repeated_packed;
+pack_repeated(Type,_) ->
+ Type.
View
31 src/protobuffs_scanner.xrl
@@ -0,0 +1,31 @@
+Definitions.
+L = [A-Za-z_\.]
+D = [0-9]
+F = (\+|-)?[0-9]+\.[0-9]+((E|e)(\+|-)?[0-9]+)?
+HEX = 0x[0-9]+
+WS = ([\000-\s]|%.*)
+S = [\(\)\]\[\{\};=]
+
+Rules.
+
+{L}({L}|{D})* : {token, {var, TokenLine,list_to_atom(TokenChars)}}.
+'({L}|{D})+' : S = strip(TokenChars,TokenLen),
+ {token,{string,TokenLine,S}}.
+"({L}|{D}|/)+" : S = strip(TokenChars,TokenLen),
+ {token,{string,TokenLine,S}}.
+{S} : {token, {list_to_atom(TokenChars),TokenLine}}.
+{WS}+ : skip_token.
+//.* : skip_token.
+/\*([^\*]|\*[^/])*\*/ : skip_token.
+{D}+ : {token, {integer, TokenLine, list_to_integer(TokenChars)}}.
+{F} : {token, {float, TokenLine, list_to_float(TokenChars)}}.
+{HEX} : {token, {integer, TokenLine, hex_to_int(TokenChars)}}.
+
+Erlang code.
+strip(TokenChars,TokenLen) ->
+ lists:sublist(TokenChars, 2, TokenLen - 2).
+
+hex_to_int([_,_|R]) ->
+ {ok,[Int],[]} = io_lib:fread("~16u", R),
+ Int.
+
View
43 support/include.mk
@@ -1,43 +0,0 @@
-## -*- makefile -*-
-
-ERL := erl
-ERLC := $(ERL)c
-
-INCLUDE_DIRS := ../include $(wildcard ../deps/*/include)
-EBIN_DIRS := $(wildcard ../deps/*/ebin)
-ERLC_FLAGS := +debug_info -W $(INCLUDE_DIRS:../%=-I ../%) $(EBIN_DIRS:%=-pa %)
-
-ifdef debug
- ERLC_FLAGS += -Ddebug
-endif
-
-EBIN_DIR := ../ebin
-DOC_DIR := ../doc
-EMULATOR := beam
-
-ERL_TEMPLATE := $(wildcard *.et)
-ERL_SOURCES := $(wildcard *.erl)
-ERL_HEADERS := $(wildcard *.hrl) $(wildcard ../include/*.hrl)
-ERL_OBJECTS := $(ERL_SOURCES:%.erl=$(EBIN_DIR)/%.beam)
-ERL_TEMPLATES := $(ERL_TEMPLATE:%.et=$(EBIN_DIR)/%.beam)
-ERL_OBJECTS_LOCAL := $(ERL_SOURCES:%.erl=./%.$(EMULATOR))
-APP_FILES := $(wildcard *.app)
-EBIN_FILES = $(ERL_OBJECTS) $(APP_FILES:%.app=../ebin/%.app) $(ERL_TEMPLATES)
-MODULES = $(ERL_SOURCES:%.erl=%)
-
-../ebin/%.app: %.app
- cp $< $@
-
-$(EBIN_DIR)/%.$(EMULATOR): %.erl
- $(ERLC) $(ERLC_FLAGS) -o $(EBIN_DIR) $<
-
-$(EBIN_DIR)/%.$(EMULATOR): %.et
- $(ERL) -noshell -pa ../../elib/erltl/ebin/ -eval "erltl:compile(atom_to_list('$<'), [{outdir, \"../ebin\"}, report_errors, report_warnings, nowarn_unused_vars])." -s init stop
-
-./%.$(EMULATOR): %.erl
- $(ERLC) $(ERLC_FLAGS) -o . $<
-
-$(DOC_DIR)/%.html: %.erl
- $(ERL) -noshell -run edoc file $< -run init stop
- mv *.html $(DOC_DIR)
-
View
12 t/Makefile
@@ -1,12 +0,0 @@
-include ../support/include.mk
-
-all: $(EBIN_FILES)
-
-clean:
- rm -rf $(EBIN_FILES) erl_crash.dump simple_pb.*
-
-test: $(MODULES)
-
-./$(MODULES):
- @echo "Running tests for $@"
- erl -pa ../ebin -run $@ start -run init stop -noshell
View
40 t/empty.proto
@@ -1,40 +0,0 @@
-message Empty {
- optional int32 id = 1;
- optional string gamername = 2;
- optional string name = 3;
- optional string roles = 4;
- optional string location = 5;
- optional int32 logged_in_at = 6;
- optional int32 created_at = 7;
- optional int32 updated_at = 8;
- optional int32 deleted = 9;
- optional string time_zone = 10;
- optional string security_token = 11;
- optional int32 token_expiry = 12;
- optional int32 verified = 13;
- optional int32 last_seen_when = 14;
- optional string last_seen_game = 15;
- optional int32 fan_nofity = 16;
- optional int32 comment_notify = 17;
- optional int32 delta = 18;
- optional int32 show_welcome = 19;
- optional string tagline = 20;
- optional int32 last_closed_site_announcement_at = 21;
- optional string last_seen_activity = 22;
- optional int64 nucleus_id = 23;
- optional string nucleus_email = 24;
- optional int32 deleted_at = 25;
- optional int32 last_closed_rupture_challenge_at = 26;
- optional int32 message_notify = 27;
- optional int32 comments_count = 28;
- optional int32 challenge_issued_notify = 29;
- optional int32 challenge_accepted_rejected_notify = 30;
- optional string password = 31;
- optional string date_of_birth = 32;
- optional string country = 33;
- optional string language = 34;
- optional string terms_of_service_version = 35;
- optional string status = 36;
- optional bool global_optin = 37;
- optional bool third_party_optin = 38;
-}
View
13 t/hasdefault.proto
@@ -1,13 +0,0 @@
-
-message Person {
- required string name = 1;
- required string address = 2;
- required string phone_number = 3 [default = "+1 (000) 000-0000"];
- required int32 age = 4 [default = 25];
- optional Location location = 5;
-}
-
-message Location {
- required string region = 1;
- required string country = 2;
-}
View
276 t/protobuffs_eqc.erl
@@ -1,276 +0,0 @@
-%%% erlc -o ebin t/*.erl -pa ebin
-%%% erl -name eqc_pb -pa ebin
-%%% eqc_gen:sample(protobuffs_eqc:protobuff_data()).
-%%% eqc:quickcheck(protobuffs_eqc:prop_encode_decode1()).
-%%% eqc:quickcheck(protobuffs_eqc:prop_encode_decode2()).
-%%% eqc:quickcheck(protobuffs_eqc:prop_encode_decode3()).
-%%%
-%%% File : protobuffs_eqc.erl
-%%% Author : <thomas@QUVIQ-THOMAS>
-%%% Description : QuickCheck specification used in class for
-%%% protobuffs-0.2
-%%% Created : 27 Apr 2009 by <thomas@QUVIQ-THOMAS>
--module(protobuffs_eqc).
-
--include_lib("eqc/include/eqc.hrl").
-
--compile(export_all).
-
--define(Mach_Eps, 1.1920928955078125e-7).
--define(NotYetImplemented(Cond,Prop), ?IMPLIES(not (Cond),Prop)).
-
-%% Properties
-
-prop_encode_decode1() ->
- ?FORALL({FieldNum,Data,Type}, protobuff_data(),
- collect(Type,
- begin
- {{N, RData}, <<>>} = protobuffs:decode(protobuffs:encode(FieldNum, Data, Type), Type),
- FieldNum =:= N andalso
- (compare(Data, RData) orelse foreign_type(Type, Data, RData))
- end)).
-
-prop_encode_decode2() ->
- ?FORALL({FieldNum,Data,Type}, fault_rate(5,10,protobuff_data()),
- case catch protobuffs:encode(FieldNum,Data,Type) of
- {'EXIT', _} ->
- not in_range(Data,Type);
- Bin ->
- {{N, RData}, <<>>} = protobuffs:decode(Bin, Type),
- in_range(Data,Type) andalso
- FieldNum =:= N andalso
- (compare(Data,RData) orelse foreign_type(Type,Data,RData))
- end).
-
-prop_encode_decode3() ->
- ?FORALL(Many, protobuff_many(),
- begin
- Sorted = lists:keysort(1, Many),
- IOList = [protobuffs:encode(FNum,Data,Type) || {FNum,Data,Type} <- Sorted],
- Bin = iolist_to_binary(IOList),
- {Decoded0,_} = lists:foldl(
- fun({_,_,Type}, {Acc, Bin1}) ->
- {Val, Rest} = protobuffs:decode(Bin1, Type),
- {[Val|Acc], Rest}
- end, {[],Bin}, Sorted),
- Decoded = lists:reverse(Decoded0),
- lists:foldl(
- fun (_, false) -> false;
- (I, true) ->
- {FNum1, Data1, Type1} = lists:nth(I, Sorted),
- {FNum2, Data2} = lists:nth(I, Decoded),
- (FNum1 =:= FNum2 andalso
- (compare(Data1,Data2) orelse foreign_type(Type1,Data1,Data2)))
- end, true, lists:seq(1, length(Sorted)))
- end).
-
-prop_encode_decode4() ->
- ?FORALL({ProtoName, Msgs}, protobuff_msgs(),
- begin
- code:delete(list_to_atom(ProtoName)),
- code:purge(list_to_atom(ProtoName)),
- protobuffs_compile:output(ProtoName, Msgs),
- ?FORALL(Set, protobuff_set(Defs),
- begin
- Record = build_record_from_defs(MsgName, Defs, Set),
- case catch apply(list_to_atom(ProtoName), list_to_atom("encode_" ++ MsgName), [Record]) of
- {'EXIT', {error, {required_field_is_undefined, FNum, _}}} ->
- {value, {_, required, _, _, _, _}} = lists:keysearch(FNum, 1, Defs),
- true;
- Bin when is_binary(Bin) ->
- Record1 = apply(list_to_atom(ProtoName), list_to_atom("decode_" ++ MsgName), [Bin]),
- lists:foldl(
- fun ({A,B}, true) ->
- io:format("compare ~p and ~p~n", [A,B]),
- compare(A,B);
- (_, false) ->
- false
- end, true, lists:zip(tuple_to_list(Record), tuple_to_list(Record1)))
- end
- end)
- end).
-
-%% Data generators
-
-protobuff_msgs() ->
- {string(), list({msg_name(), protobuff_defs()})}.
-
-protobuff_many() ->
- list(protobuff_data()).
-
-protobuff_set(Defs) ->
- [begin
- {FNum, Name, field_value(Tag, Type)}
- end || {FNum, Tag, Type, Name, _, _Default} <- Defs].
-
-protobuff_data() ->
- fault({field_num(), int(80), oneof([int32,uint32,int64,uint64,sint32,sint64])},
- oneof([
- {field_num(), int(32), int32},
- {field_num(), uint(32), uint32},
- {field_num(), int(64), int64},
- {field_num(), uint(64), uint64},
- {field_num(), bool(), bool},
- {field_num(), sint(32), sint32},
- {field_num(), sint(64), sint64},
- {field_num(), real(), float},
- {field_num(), real(), double},
- {field_num(), list(char()), string},
- {field_num(), binary(), bytes}
- ])
- ).
-
-protobuff_defs() ->
- ?SUCHTHAT(D,orderedlist(protobuff_def()),length(D) > 0).
-
-protobuff_def() ->
- oneof([
- {field_num(), tag(), "int32", field_name(), number, oneof([none, int(32)])},
- {field_num(), tag(), "uint32", field_name(), number, oneof([none, uint(32)])},
- {field_num(), tag(), "int64", field_name(), number, oneof([none, int(64)])},
- {field_num(), tag(), "uint64", field_name(), number, oneof([none, uint(64)])},
- {field_num(), tag(), "bool", field_name(), number, oneof([none, bool()])},
- {field_num(), tag(), "sint32", field_name(), number, oneof([none, sint(32)])},
- {field_num(), tag(), "sint64", field_name(), number, oneof([none, sint(64)])},
- {field_num(), tag(), "float", field_name(), number, oneof([none, real()])},
- {field_num(), tag(), "double", field_name(), number, oneof([none, real()])},
- {field_num(), tag(), "string", field_name(), number, oneof([none, list(char())])},
- {field_num(), tag(), "bytes", field_name(), number, oneof([none, binary()])}
- ]).
-
-field_value(repeated, "int32") -> oneof([undefined, list(int(32))]);
-field_value(repeated, "uint32") -> oneof([undefined, list(uint(32))]);
-field_value(repeated, "int64") -> oneof([undefined, list(int(64))]);
-field_value(repeated, "uint64") -> oneof([undefined, list(uint(64))]);
-field_value(repeated, "bool") -> oneof([undefined, list(bool())]);
-field_value(repeated, "sint32") -> oneof([undefined, list(sint(32))]);
-field_value(repeated, "sint64") -> oneof([undefined, list(sint(64))]);
-field_value(repeated, "float") -> oneof([undefined, list(real())]);
-field_value(repeated, "double") -> oneof([undefined, list(real())]);
-field_value(repeated, "string") -> oneof([undefined, list(list(char()))]);
-field_value(repeated, "bytes") -> oneof([undefined, list(binary())]);
-field_value(_, "int32") -> oneof([undefined, int(32)]);
-field_value(_, "uint32") -> oneof([undefined, uint(32)]);
-field_value(_, "int64") -> oneof([undefined, int(64)]);
-field_value(_, "uint64") -> oneof([undefined, uint(64)]);
-field_value(_, "bool") -> oneof([undefined, bool()]);
-field_value(_, "sint32") -> oneof([undefined, sint(32)]);
-field_value(_, "sint64") -> oneof([undefined, sint(64)]);
-field_value(_, "float") -> oneof([undefined, real()]);
-field_value(_, "double") -> oneof([undefined, real()]);
-field_value(_, "string") -> oneof([undefined, list(char())]);
-field_value(_, "bytes") -> oneof([undefined, binary()]).
-
-field_num() ->
- ?SUCHTHAT(N,nat(),N>0).
-
-tag() ->
- oneof([optional, required, repeated]).
-
-field_name() ->
- ?SUCHTHAT(N,string(),length(N)>0).
-
-msg_name() ->
- ?SUCHTHAT(N,string(),length(N)>0).
-
-string() ->
- list(oneof([choose(97,122), choose(65,90)])).
-
-%% Internal Functions
-
-foreign_type(bool,false,0) ->
- true;
-foreign_type(bool,true,1) ->
- true;
-foreign_type(_,_,_) ->
- false.
-
-prop_varint() ->
- ?FORALL(Base,oneof([32,64]),
- ?FORALL(I,int(Base),
- begin
- {Bits,Data} = decompose(protobuffs:encode_varint(I)),
- right_bits(Bits) andalso
- concatenate(Data) == I
- end
- )
- ).
-
-build_record_from_defs(MsgName, Defs, Set) ->
- lists:foldl(
- fun({FNum, _Tag, _Type, _Name, _, _Default}, Acc) ->
- {value, {_,_,Value}} = lists:keysearch(FNum, 1, Set),
- erlang:append_element(Acc, Value)
- end, {list_to_atom(MsgName)}, Defs).
-
-%% Bits are in reverse order: First bit should be zero, rest should be 1
-right_bits([0|Rest]) ->
- lists:all(fun(B) -> B==1 end,Rest).
-
-int(Base) ->
- ?LET(I,uint(Base),
- begin
- << N:Base/signed >> = <<I:Base>>, N
- end
- ).
-
-uint(Base) ->
- oneof([ choose(0,exp(B)) || B<-lists:seq(1,Base)]).
-
-sint(Base) ->
- int(Base).
-
-exp(1) ->
- 2;
-exp(N) ->
- 2*exp(N-1).
-
-decompose(<<Bit:1,Data:7>>) ->
- {[Bit],<<Data:7>>};
-decompose(<<Bit:1,Data:7,Rest/binary>>) ->
- {Bs,Ds} = decompose(Rest),
- {Bs++[Bit],<<Ds/bitstring,Data:7>>}.
-
-concatenate(Bin) ->
- S = bit_size(Bin),
- << N:S >> = Bin,
- N.
-
-in_range(Int,int32) ->
- fitbits(Int,32);
-in_range(Int,sint32) ->
- fitbits(abs(Int),31);
-in_range(Int,uint32) ->
- fitbits(Int,32);
-in_range(Int,int64) ->
- fitbits(Int,64);
-in_range(Int,sint64) ->
- fitbits(abs(Int),63);
-in_range(Int,uint64) ->
- fitbits(Int,64);
-in_range(Float,float) ->
- fitbits(Float,32);
-in_range(Float,double) ->
- fitbits(Float,64);
-in_range(_,string) ->
- true;
-in_range(_,bytes) ->
- true;
-in_range(false,bool) ->
- true;
-in_range(true,bool) ->
- true.
-
-compare(Float1, Float2) when is_float(Float1), is_float(Float2) ->
- (abs(Float1 - Float2) =< ?Mach_Eps);
-compare(String, Binary) when is_list(String), is_binary(Binary) ->
- String =:= binary_to_list(Binary);
-compare(A,A) -> true;
-compare(_,_) -> false.
-
-fitbits(Float,32) when is_float(Float) -> true;
-fitbits(Float,64) when is_float(Float) -> true;
-fitbits(Int,Bits) ->
- RestBits = 80-Bits,
- << NoFit:RestBits, _:Bits >> = <<Int:80>>,
- NoFit == 0.
View
15 t/protobuffs_t_001.t
@@ -1,15 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-%%! -pa ./ebin -sasl errlog_type error -boot start_sasl -noshell
-
-main(_) ->
- etap:plan(8),
- etap_can:loaded_ok(protobuffs, "module 'protobuffs' loaded"),
- etap_can:can_ok(protobuffs, encode),
- etap_can:can_ok(protobuffs, encode, 3),
- etap_can:can_ok(protobuffs, decode),
- etap_can:can_ok(protobuffs, decode, 2),
- etap_can:loaded_ok(protobuffs_compile, "module 'protobuffs_compile' loaded"),
- etap_can:can_ok(protobuffs_compile, scan_file),
- etap_can:can_ok(protobuffs_compile, scan_file, 1),
- etap:end_tests().
View
10 t/protobuffs_t_002.t
@@ -1,10 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-%%! -pa ./ebin -sasl errlog_type error -boot start_sasl -noshell
-
-main(_) ->
- etap:plan(3),
- etap:is(protobuffs:encode(1, 1, uint32), <<8,1>>, "1, 1, unit32"),
- etap:is(protobuffs:encode(2, 1, uint32), <<16,1>>, "2, 1, unit32"),
- etap:is(protobuffs:encode(3, 1, uint32), <<24,1>>, "3, 1, unit32"),
- etap:end_tests().
View
42 t/protobuffs_t_003.t
@@ -1,42 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-