Skip to content

Commit

Permalink
gRPC view: Adjusted parsing strategy for deprecated protobuf fields (#…
Browse files Browse the repository at this point in the history 
…4858)

* Adjusted parsing strategy for deprecated protobuf fields

* Added test comment

* adjusted comment to reflect correct field IDs
  • Loading branch information
@mame82
mame82 committed Oct 13, 2021
1 parent 77cf2ab commit 8b6345e
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 3 deletions.
4 changes: 2 additions & 2 deletions mitmproxy/contentviews/grpc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,8 @@ def parse_message_fields(self, message: bytes) -> List:
tag = pair.field_tag
wt = pair.wire_type
if wt == GoogleProtobuf.Pair.WireTypes.group_start or wt == GoogleProtobuf.Pair.WireTypes.group_end:
# ignore deprecated types without values
continue
# raise error on deprecated types without values
raise ValueError("deprecated field: {}".format(wt))
v: Union[GoogleProtobuf.DelimitedBytes, VlqBase128Le] = pair.value # for WireType bit-32 and bit-64
preferred_decoding = ProtoParser.DecodedTypes.unknown
# see: https://www.oreilly.com/library/view/grpc-up-and/9781492058328/ch04.html
Expand Down
77 changes: 76 additions & 1 deletion test/mitmproxy/contentviews/test_grpc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,81 @@ def test_view_protobuf_custom_parsing_request(tdata):


def test_view_protobuf_custom_parsing_response(tdata):
# expect to parse 1.3.2 and 1.3.3 as string automatically
# even if there is a length delimeted field containing `b"DC"`, which would translate to
# two deprecated fields [8: group_start, 8: group_end] (and thus represent a valid nested message,
# but containing deprecated wire types)
custom_view_config_parser_rules.parser_rules[1].field_definitions[3].intended_decoding = None
custom_view_config_parser_rules.parser_rules[1].field_definitions[4].intended_decoding = None

v = full_eval(ViewGrpcProtobuf(custom_view_config_parser_rules))
p = tdata.path(datadir + "msg3.bin")

with open(p, "rb") as f:
raw = f.read()
view_text, output = v(raw, flow=sim_flow, http_message=sim_flow.response) # simulate response message
assert view_text == "Protobuf (flattened)"
output = list(output) # assure list conversion if generator
assert output == [
[('text', '[message] '), ('text', ' '), ('text', '1 '), ('text', ' ')], # noqa: E501
[('text', '[string] '), ('text', ' '), ('text', '1.1 '), ('text', '\x15 ')], # noqa: E501
[('text', '[string] '), ('text', 'address '), ('text', '1.2 '), ('text', '1650 Pennsylvania Avenue NW, Washington, DC 20502, USA ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', 'b\'"\' ')], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', '1650 ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', '1650 ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x02' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'Pennsylvania Avenue Northwest ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'Pennsylvania Avenue NW ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x14\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'Northwest Washington ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'Northwest Washington ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x0c\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'Washington ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'Washington ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x06\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'District of Columbia ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'DC ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x05\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'USA ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'US ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x17' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', '20502 ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', '20502 ')], # noqa: E501
[('text', '[message] '), ('text', ' '), ('text', '1.5 '), ('text', ' ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.1 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.1.1 '), ('text', '38.8970309 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.1.2 '), ('text', '-77.03872559999999 ')], # noqa: E501
[('text', '[uint32] '), ('text', ' '), ('text', '1.5.2 '), ('text', '1 ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.3 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.3.1 '), ('text', '38.8962271697085 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.3.2 '), ('text', '-77.0400511802915 ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.4 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.4.1 '), ('text', '38.8989251302915 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.4.2 '), ('text', '-77.03735321970849 ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.5 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.5.1 '), ('text', '38.896898 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.5.2 '), ('text', '-77.03917229999999 ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.6 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.6.1 '), ('text', '38.8982543 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.6.2 '), ('text', '-77.0382321 ')], # noqa: E501
[('text', '[string] '), ('text', ' '), ('text', '1.7 '), ('text', 'ChIJAXiAory3t4kRpkrvas9dYmQ ')], # noqa: E501
[('text', '[message] '), ('text', ' '), ('text', '2 '), ('text', ' ')], # noqa: E501
[('text', '[uint32] '), ('text', ' '), ('text', '2.1 '), ('text', '21 ')], # noqa: E501
]


def test_view_protobuf_custom_parsing_response2(tdata):
# try to parse 1.3.2 and 1.3.3 as string
custom_view_config_parser_rules.parser_rules[1].field_definitions[3].intended_decoding = ProtoParser.DecodedTypes.string # 1.3.2
custom_view_config_parser_rules.parser_rules[1].field_definitions[4].intended_decoding = ProtoParser.DecodedTypes.string # 1.3.3

v = full_eval(ViewGrpcProtobuf(custom_view_config_parser_rules))
p = tdata.path(datadir + "msg3.bin")

Expand Down Expand Up @@ -141,7 +216,7 @@ def test_view_protobuf_custom_parsing_response(tdata):
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x06\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'District of Columbia ')], # noqa: E501
[('text', '[message] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', ' ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'DC ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x05\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'USA ')], # noqa: E501
Expand Down

0 comments on commit 8b6345e

Please sign in to comment.