Skip to content

Commit

Permalink
Fix parsing of lists in the text format and add support for skipping …
Browse files Browse the repository at this point in the history 
…unknown fields.

R=pallosp
DELTA=193  (169 added, 0 deleted, 24 changed)


Revision created by MOE tool push_codebase.
MOE_MIGRATION=1376


git-svn-id: https://closure-library.googlecode.com/svn/trunk@849 0b95b8e8-c90f-11de-9d4f-f947ee5921c8
  • Loading branch information
jschorr@google.com committed Apr 12, 2011
1 parent e263922 commit 30dcd47
Show file tree
Hide file tree
Showing 2 changed files with 191 additions and 22 deletions.
87 changes: 79 additions & 8 deletions closure/goog/proto2/textformatserializer.js
View file
Expand Up @@ -34,10 +34,20 @@ goog.require('goog.string');
/**
* TextFormatSerializer, a serializer which turns Messages into the human
* readable text format.
* @param {boolean=} opt_ignoreMissingFields If true, then fields that cannot be
* found on the proto when parsing the text format will be ignored.
* @constructor
* @extends {goog.proto2.Serializer}
*/
goog.proto2.TextFormatSerializer = function() {};
goog.proto2.TextFormatSerializer = function(opt_ignoreMissingFields) {
/**
* Whether to ignore fields not defined on the proto when parsing the text
* format.
* @type {boolean}
* @private
*/
this.ignoreMissingFields_ = !!opt_ignoreMissingFields;
};
goog.inherits(goog.proto2.TextFormatSerializer, goog.proto2.Serializer);


Expand All @@ -53,7 +63,7 @@ goog.proto2.TextFormatSerializer.prototype.deserializeTo =
var descriptor = message.getDescriptor();
var textData = data.toString();
var parser = new goog.proto2.TextFormatSerializer.Parser();
if (!parser.parse(message, textData)) {
if (!parser.parse(message, textData, this.ignoreMissingFields_)) {
return parser.getError();
}

Expand Down Expand Up @@ -400,6 +410,8 @@ goog.proto2.TextFormatSerializer.Tokenizer_.TokenTypes = {
CLOSE_BRACE: /^}/,
OPEN_TAG: /^</,
CLOSE_TAG: /^>/,
OPEN_LIST: /^\[/,
CLOSE_LIST: /^\]/,
STRING: new RegExp('^"([^"|\\\\]|\\\\.)*"'),
COLON: /^:/,
COMMA: /^,/,
Expand Down Expand Up @@ -499,19 +511,29 @@ goog.proto2.TextFormatSerializer.Parser = function() {
* @private
*/
this.tokenizer_ = null;

/**
* Whether to ignore missing fields in the proto when parsing.
* @type {boolean}
* @private
*/
this.ignoreMissingFields_ = false;
};


/**
* Parses the given data, filling the message as it goes.
* @param {goog.proto2.Message} message The message to fill.
* @param {string} data The text format data.
* @param {boolean=} opt_ignoreMissingFields If true, fields missing in the
* proto will be ignored.
* @return {boolean} True on success, false on failure. On failure, the
* getError method can be called to get the reason for failure.
*/
goog.proto2.TextFormatSerializer.Parser.prototype.parse =
function(message, data) {
function(message, data, opt_ignoreMissingFields) {
this.error_ = null;
this.ignoreMissingFields_ = !!opt_ignoreMissingFields;
this.tokenizer_ = new goog.proto2.TextFormatSerializer.Tokenizer_(data, true);
this.tokenizer_.next();
return this.consumeMessage_(message, '');
Expand Down Expand Up @@ -539,7 +561,9 @@ goog.proto2.TextFormatSerializer.Parser.prototype.reportError_ =

/**
* Attempts to consume the given message.
* @param {goog.proto2.Message} message The message to consume and fill.
* @param {goog.proto2.Message} message The message to consume and fill. If
* null, then the message contents will be consumed without ever being set
* to anything.
* @param {string} delimiter The delimiter expected at the end of the message.
* @return {boolean} True on success, false otherwise.
* @private
Expand Down Expand Up @@ -714,9 +738,48 @@ goog.proto2.TextFormatSerializer.Parser.prototype.consumeNestedMessage_ =
};


/**
* Attempts to consume the value of an unknown field. This method uses
* heuristics to try to consume just the right tokens.
* @return {boolean} True on success, false otherwise.
* @private
*/
goog.proto2.TextFormatSerializer.Parser.prototype.consumeUnknownFieldValue_ =
function() {
// : is optional.
this.tryConsume_(':');

// Handle form: [.. , ... , ..]
if (this.tryConsume_('[')) {
while (true) {
this.tokenizer_.next();
if (this.tryConsume_(']')) {
break;
}
if (!this.consume_(',')) { return false; }
}

return true;
}

// Handle nested messages/groups.
if (this.tryConsume_('<')) {
return this.consumeMessage_(null /* unknown */, '>');
} else if (this.tryConsume_('{')) {
return this.consumeMessage_(null /* unknown */, '}');
} else {
// Otherwise, consume a single token for the field value.
this.tokenizer_.next();
}

return true;
};


/**
* Attempts to consume a field under a message.
* @param {goog.proto2.Message} message The parent message.
* @param {goog.proto2.Message} message The parent message. If null, then the
* field value will be consumed without being assigned to anything.
* @return {boolean} True on success, false otherwise.
* @private
*/
Expand All @@ -728,10 +791,18 @@ goog.proto2.TextFormatSerializer.Parser.prototype.consumeField_ =
return false;
}

var field = message.getDescriptor().findFieldByName(fieldName.toString());
var field = null;
if (message) {
field = message.getDescriptor().findFieldByName(fieldName.toString());
}

if (field == null) {
this.reportError_('Unknown field: ' + fieldName);
return false;
if (this.ignoreMissingFields_) {
return this.consumeUnknownFieldValue_();
} else {
this.reportError_('Unknown field: ' + fieldName);
return false;
}
}

if (field.getFieldType() == goog.proto2.FieldDescriptor.FieldType.MESSAGE ||
Expand Down
126 changes: 112 additions & 14 deletions closure/goog/proto2/textformatserializer_test.html
View file
Expand Up @@ -253,7 +253,7 @@
}

function testDeserialization() {
var message = new proto2.TestAllTypes();
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101\n' +
'repeated_int32: 201\n' +
'repeated_int32: 202\n' +
Expand All @@ -267,12 +267,110 @@
assertEquals(123.4, message.getOptionalFloat());
}

function testDeserializationOfList() {
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101\n' +
'repeated_int32: [201, 202]\n' +
'optional_float: 123.4';

new goog.proto2.TextFormatSerializer().deserializeTo(message, value);

assertEquals(101, message.getOptionalInt32());
assertEquals(201, message.getRepeatedInt32(0));
assertEquals(123.4, message.getOptionalFloat());
}

function testDeserializationSkipUnknown() {
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101\n' +
'repeated_int32: 201\n' +
'some_unknown: true\n' +
'repeated_int32: 202\n' +
'optional_float: 123.4';

var parser = new goog.proto2.TextFormatSerializer.Parser();
assertTrue(parser.parse(message, value, true));

assertEquals(101, message.getOptionalInt32());
assertEquals(201, message.getRepeatedInt32(0));
assertEquals(202, message.getRepeatedInt32(1));
assertEquals(123.4, message.getOptionalFloat());
}

function testDeserializationSkipUnknownList() {
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101\n' +
'repeated_int32: 201\n' +
'some_unknown: [true, 1, 201, "hello"]\n' +
'repeated_int32: 202\n' +
'optional_float: 123.4';

var parser = new goog.proto2.TextFormatSerializer.Parser();
assertTrue(parser.parse(message, value, true));

assertEquals(101, message.getOptionalInt32());
assertEquals(201, message.getRepeatedInt32(0));
assertEquals(202, message.getRepeatedInt32(1));
assertEquals(123.4, message.getOptionalFloat());
}

function testDeserializationSkipUnknownNested() {
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101\n' +
'repeated_int32: 201\n' +
'some_unknown: <\n' +
' a: 1\n' +
' b: 2\n' +
'>\n' +
'repeated_int32: 202\n' +
'optional_float: 123.4';

var parser = new goog.proto2.TextFormatSerializer.Parser();
assertTrue(parser.parse(message, value, true));

assertEquals(101, message.getOptionalInt32());
assertEquals(201, message.getRepeatedInt32(0));
assertEquals(202, message.getRepeatedInt32(1));
assertEquals(123.4, message.getOptionalFloat());
}

function testDeserializationSkipUnknownNestedInvalid() {
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101\n' +
'repeated_int32: 201\n' +
'some_unknown: <\n' +
' a: \n' + // Missing value.
' b: 2\n' +
'>\n' +
'repeated_int32: 202\n' +
'optional_float: 123.4';

var parser = new goog.proto2.TextFormatSerializer.Parser();
assertFalse(parser.parse(message, value, true));
}

function testDeserializationSkipUnknownNestedInvalid2() {
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101\n' +
'repeated_int32: 201\n' +
'some_unknown: <\n' +
' a: 2\n' +
' b: 2\n' +
'}\n' + // Delimiter mismatch
'repeated_int32: 202\n' +
'optional_float: 123.4';

var parser = new goog.proto2.TextFormatSerializer.Parser();
assertFalse(parser.parse(message, value, true));
}


function testDeserializationLegacyFormat() {
var message = new proto2.TestAllTypes();
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101,\n' +
'repeated_int32: 201,\n' +
'repeated_int32: 202;\n' +
'optional_float: 123.4';
'repeated_int32: 201,\n' +
'repeated_int32: 202;\n' +
'optional_float: 123.4';

new goog.proto2.TextFormatSerializer().deserializeTo(message, value);

Expand All @@ -283,19 +381,19 @@
}

function testDeserializationError() {
var message = new proto2.TestAllTypes();
var message = new proto2.TestAllTypes();
var value = 'optional_int33: 101\n';
var result =
new goog.proto2.TextFormatSerializer().deserializeTo(message, value);
assertEquals(result, 'Unknown field: optional_int33');
}

function testNestedDeserialization() {
var message = new proto2.TestAllTypes();
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101\n' +
'optional_nested_message: {\n' +
' b: 301\n' +
'}';
'optional_nested_message: {\n' +
' b: 301\n' +
'}';

new goog.proto2.TextFormatSerializer().deserializeTo(message, value);

Expand All @@ -304,11 +402,11 @@
}

function testNestedDeserializationLegacyFormat() {
var message = new proto2.TestAllTypes();
var message = new proto2.TestAllTypes();
var value = 'optional_int32: 101\n' +
'optional_nested_message: <\n' +
' b: 301\n' +
'>';
'optional_nested_message: <\n' +
' b: 301\n' +
'>';

new goog.proto2.TextFormatSerializer().deserializeTo(message, value);

Expand Down

0 comments on commit 30dcd47

Please sign in to comment.