diff --git a/quickfixj-core/src/main/java/quickfix/DataDictionary.java b/quickfixj-core/src/main/java/quickfix/DataDictionary.java index 92e6d42feb..1b8b6344d8 100644 --- a/quickfixj-core/src/main/java/quickfix/DataDictionary.java +++ b/quickfixj-core/src/main/java/quickfix/DataDictionary.java @@ -518,11 +518,6 @@ public void setAllowUnknownMessageFields(boolean allowUnknownFields) { private void copyFrom(DataDictionary rhs) { hasVersion = rhs.hasVersion; beginString = rhs.beginString; - checkFieldsOutOfOrder = rhs.checkFieldsOutOfOrder; - checkFieldsHaveValues = rhs.checkFieldsHaveValues; - checkUserDefinedFields = rhs.checkUserDefinedFields; - checkUnorderedGroupFields = rhs.checkUnorderedGroupFields; - allowUnknownMessageFields = rhs.allowUnknownMessageFields; copyMap(messageFields, rhs.messageFields); copyMap(requiredFields, rhs.requiredFields); @@ -533,8 +528,14 @@ private void copyFrom(DataDictionary rhs) { copyMap(fieldNames, rhs.fieldNames); copyMap(names, rhs.names); copyMap(valueNames, rhs.valueNames); - copyMap(groups, rhs.groups); + copyGroups(groups, rhs.groups); copyMap(components, rhs.components); + + setCheckFieldsOutOfOrder(rhs.checkFieldsOutOfOrder); + setCheckFieldsHaveValues(rhs.checkFieldsHaveValues); + setCheckUserDefinedFields(rhs.checkUserDefinedFields); + setCheckUnorderedGroupFields(rhs.checkUnorderedGroupFields); + setAllowUnknownMessageFields(rhs.allowUnknownMessageFields); } @SuppressWarnings("unchecked") @@ -558,13 +559,26 @@ private static void copyMap(Map lhs, Map rhs) { } } + /** copy groups including their data dictionaries and validation settings + * + * @param lhs target + * @param rhs source + */ + private static void copyGroups(Map lhs, Map rhs) { + lhs.clear(); + for (Map.Entry entry : rhs.entrySet()) { + GroupInfo value = new GroupInfo(entry.getValue().getDelimiterField(), new DataDictionary(entry.getValue().getDataDictionary())); + lhs.put(entry.getKey(), value); + } + } + private static void copyCollection(Collection lhs, Collection rhs) { lhs.clear(); lhs.addAll(rhs); } /** - * Validate a mesasge, including the header and trailer fields. + * Validate a message, including the header and trailer fields. * * @param message the message * @throws IncorrectTagValue if a field value is not valid @@ -674,16 +688,10 @@ void checkValidTagNumber(Field field) { void checkField(Field field, String msgType, boolean message) { // use different validation for groups and messages boolean messageField = message ? isMsgField(msgType, field.getField()) : fields.contains(field.getField()); - boolean fail; - - if (field.getField() < USER_DEFINED_TAG_MIN) { - fail = !messageField && !allowUnknownMessageFields; - } else { - fail = !messageField && checkUserDefinedFields; - } + boolean fail = checkFieldFailure(field.getField(), messageField); if (fail) { - if (fields.contains(field.getTag())) { + if (fields.contains(field.getField())) { throw new FieldException(SessionRejectReason.TAG_NOT_DEFINED_FOR_THIS_MESSAGE_TYPE, field.getField()); } else { throw new FieldException(SessionRejectReason.INVALID_TAG_NUMBER, field.getField()); @@ -691,6 +699,16 @@ void checkField(Field field, String msgType, boolean message) { } } + boolean checkFieldFailure(int field, boolean messageField) { + boolean fail; + if (field < USER_DEFINED_TAG_MIN) { + fail = !messageField && !allowUnknownMessageFields; + } else { + fail = !messageField && checkUserDefinedFields; + } + return fail; + } + private void checkValidFormat(StringField field) throws IncorrectDataFormat { FieldType fieldType = getFieldType(field.getTag()); if (fieldType == null) { diff --git a/quickfixj-core/src/main/java/quickfix/Message.java b/quickfixj-core/src/main/java/quickfix/Message.java index 32bea95b02..eb9d56c21a 100644 --- a/quickfixj-core/src/main/java/quickfix/Message.java +++ b/quickfixj-core/src/main/java/quickfix/Message.java @@ -77,8 +77,7 @@ public class Message extends FieldMap { protected Header header = new Header(); protected Trailer trailer = new Trailer(); - // @GuardedBy("this") - private FieldException exception; + private volatile FieldException exception; public Message() { // empty @@ -510,7 +509,7 @@ && isNextField(dd, header, BodyLength.FIELD) header.setField(field); if (dd != null && dd.isGroup(DataDictionary.HEADER_ID, field.getField())) { - parseGroup(DataDictionary.HEADER_ID, field, dd, header); + parseGroup(DataDictionary.HEADER_ID, field, dd, header, doValidation); } field = extractField(dd, header); @@ -549,7 +548,7 @@ private void parseBody(DataDictionary dd, boolean doValidation) throws InvalidMe setField(header, field); // Group case if (dd != null && dd.isGroup(DataDictionary.HEADER_ID, field.getField())) { - parseGroup(DataDictionary.HEADER_ID, field, dd, header); + parseGroup(DataDictionary.HEADER_ID, field, dd, header, doValidation); } if (doValidation && dd != null && dd.isCheckFieldsOutOfOrder()) throw new FieldException(SessionRejectReason.TAG_SPECIFIED_OUT_OF_REQUIRED_ORDER, @@ -558,7 +557,7 @@ private void parseBody(DataDictionary dd, boolean doValidation) throws InvalidMe setField(this, field); // Group case if (dd != null && dd.isGroup(getMsgType(), field.getField())) { - parseGroup(getMsgType(), field, dd, this); + parseGroup(getMsgType(), field, dd, this, doValidation); } } @@ -573,7 +572,7 @@ private void setField(FieldMap fields, StringField field) { fields.setField(field); } - private void parseGroup(String msgType, StringField field, DataDictionary dd, FieldMap parent) + private void parseGroup(String msgType, StringField field, DataDictionary dd, FieldMap parent, boolean doValidation) throws InvalidMessage { final DataDictionary.GroupInfo rg = dd.getGroup(msgType, field.getField()); final DataDictionary groupDataDictionary = rg.getDataDictionary(); @@ -603,14 +602,14 @@ private void parseGroup(String msgType, StringField field, DataDictionary dd, Fi previousOffset = -1; // QFJ-742 if (groupDataDictionary.isGroup(msgType, tag)) { - parseGroup(msgType, field, groupDataDictionary, group); + parseGroup(msgType, field, groupDataDictionary, group, doValidation); } } else if (groupDataDictionary.isGroup(msgType, tag)) { if (!firstFieldFound) { throw new InvalidMessage("The group " + groupCountTag + " must set the delimiter field " + firstField + " in " + messageData); } - parseGroup(msgType, field, groupDataDictionary, group); + parseGroup(msgType, field, groupDataDictionary, group, doValidation); } else if (groupDataDictionary.isField(tag)) { if (!firstFieldFound) { throw new FieldException( @@ -629,6 +628,20 @@ private void parseGroup(String msgType, StringField field, DataDictionary dd, Fi } group.setField(field); } else { + // QFJ-169/QFJ-791: handle unknown repeating group fields in the body + if (!(DataDictionary.HEADER_ID.equals(msgType))) { + if (!isTrailerField(tag) && !dd.isMsgField(msgType, tag)) { + if (doValidation) { + boolean fail = dd.checkFieldFailure(tag, false); + if (fail) { + throw new FieldException( + SessionRejectReason.TAG_NOT_DEFINED_FOR_THIS_MESSAGE_TYPE, tag); + } + } + group.setField(field); + continue; + } + } pushBack(field); inGroupParse = false; } @@ -790,11 +803,11 @@ private StringField extractField(DataDictionary dataDictionary, FieldMap fields) * * @return flag indicating whether the message has a valid structure */ - synchronized boolean hasValidStructure() { + boolean hasValidStructure() { return exception == null; } - public synchronized FieldException getException() { + public FieldException getException() { return exception; } @@ -804,7 +817,7 @@ public synchronized FieldException getException() { * * @return the first invalid tag */ - synchronized int getInvalidTag() { + int getInvalidTag() { return exception != null ? exception.getField() : 0; } diff --git a/quickfixj-core/src/test/java/quickfix/DataDictionaryTest.java b/quickfixj-core/src/test/java/quickfix/DataDictionaryTest.java index 504860ca5a..0a0610e4e0 100644 --- a/quickfixj-core/src/test/java/quickfix/DataDictionaryTest.java +++ b/quickfixj-core/src/test/java/quickfix/DataDictionaryTest.java @@ -43,6 +43,7 @@ import quickfix.field.MsgSeqNum; import quickfix.field.MsgType; import quickfix.field.NoHops; +import quickfix.field.NoPartyIDs; import quickfix.field.NoRelatedSym; import quickfix.field.OrdType; import quickfix.field.OrderQty; @@ -476,6 +477,16 @@ public void testCopy() throws Exception { assertEquals(ddCopy.isCheckUnorderedGroupFields(),dataDictionary.isCheckUnorderedGroupFields()); assertEquals(ddCopy.isCheckUserDefinedFields(),dataDictionary.isCheckUserDefinedFields()); + DataDictionary.GroupInfo groupFromDDCopy = ddCopy.getGroup(NewOrderSingle.MSGTYPE, NoPartyIDs.FIELD); + assertTrue(groupFromDDCopy.getDataDictionary().isAllowUnknownMessageFields()); + // set to false on ORIGINAL DD + dataDictionary.setAllowUnknownMessageFields(false); + assertFalse(dataDictionary.isAllowUnknownMessageFields()); + assertFalse(dataDictionary.getGroup(NewOrderSingle.MSGTYPE, NoPartyIDs.FIELD).getDataDictionary().isAllowUnknownMessageFields()); + // should be still true on COPIED DD and its group + assertTrue(ddCopy.isAllowUnknownMessageFields()); + groupFromDDCopy = ddCopy.getGroup(NewOrderSingle.MSGTYPE, NoPartyIDs.FIELD); + assertTrue(groupFromDDCopy.getDataDictionary().isAllowUnknownMessageFields()); } /** diff --git a/quickfixj-core/src/test/java/quickfix/MessageTest.java b/quickfixj-core/src/test/java/quickfix/MessageTest.java index 803913b90e..f4be922d50 100644 --- a/quickfixj-core/src/test/java/quickfix/MessageTest.java +++ b/quickfixj-core/src/test/java/quickfix/MessageTest.java @@ -107,6 +107,24 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import quickfix.field.LastPx; +import quickfix.field.LastQty; +import quickfix.field.LegPrice; +import quickfix.field.LegQty; +import quickfix.field.LegRefID; +import quickfix.field.LegSymbol; +import quickfix.field.MaturityMonthYear; +import quickfix.field.PreviouslyReported; +import quickfix.field.PutOrCall; +import quickfix.field.QuoteAckStatus; +import quickfix.field.SecurityReqID; +import quickfix.field.SecurityRequestResult; +import quickfix.field.SecurityResponseID; +import quickfix.field.StrikePrice; +import quickfix.field.Text; +import quickfix.field.TradeDate; +import quickfix.field.TradeReportID; +import quickfix.fix44.TradeCaptureReport; public class MessageTest { @@ -1352,6 +1370,238 @@ public void testInvalidHeaderFields() throws Exception { assertTrue(msg.isSetField(Account.FIELD)); } + @Test + // QFJ-791 + public void testRepeatingGroupCount() throws Exception { + /* + * Prepare a very simple TradeCaptureReport message template and two + * legs. + */ + Message tcr = new TradeCaptureReport(new TradeReportID("ABC1234"), new PreviouslyReported( + false), new LastQty(1000), new LastPx(5.6789), new TradeDate("20140101"), + new TransactTime(new Date())); + tcr.getHeader().setField(new SenderCompID("SENDER")); + tcr.getHeader().setField(new TargetCompID("TARGET")); + tcr.getHeader().setField(new MsgSeqNum(1)); + tcr.getHeader().setField(new SendingTime(new Date())); + TradeCaptureReport.NoLegs leg1 = new TradeCaptureReport.NoLegs(); + leg1.setField(new LegSymbol("L1-XYZ")); + leg1.setField(new LegRefID("ABC1234-L1")); + leg1.setField(new LegQty(333)); + leg1.setField(new LegPrice(1.2345)); + TradeCaptureReport.NoLegs leg2 = new TradeCaptureReport.NoLegs(); + leg2.setField(new LegSymbol("L2-XYZ")); + leg2.setField(new LegRefID("ABC1234-L2")); + leg2.setField(new LegQty(777)); + leg2.setField(new LegPrice(2.3456)); + + /* + * Create a message from the template and add two legs. Convert the + * message to string and parse it. The parsed message should contain two + * legs. + */ + { + Message m1 = new Message(); + m1.getHeader().setFields(tcr.getHeader()); + m1.setFields(tcr); + m1.addGroup(leg1); + m1.addGroup(leg2); + + String s1 = m1.toString(); + Message parsed1 = new Message(s1, DataDictionaryTest.getDictionary()); + + assertEquals(s1, parsed1.toString()); + assertEquals(2, parsed1.getGroupCount(555)); + } + + /* + * Create a message from the template and add two legs, but the first + * leg contains the additional tag 58 (Text). Convert the message to + * string and parse it. The parsed message should also contain two legs. + */ + { + Message m2 = new Message(); + m2.getHeader().setFields(tcr.getHeader()); + m2.setFields(tcr); + + leg1.setField(new Text("TXT1")); // add unexpected tag to leg1 + m2.addGroup(leg1); + m2.addGroup(leg2); + + String s2 = m2.toString(); + // do not use validation to parse full message + // regardless of errors in message structure + Message parsed2 = new Message(s2, DataDictionaryTest.getDictionary(), false); + + assertEquals(s2, parsed2.toString()); + assertEquals(2, parsed2.getGroupCount(555)); + + /* + * If the above test failed, it means that a simple addition of an + * unexpected tag made the parsing logic fail pretty badly, as the + * number of legs is not 2. + */ + } + } + + @Test + // QFJ-791 + public void testUnknownFieldsInRepeatingGroupsAndValidation() throws Exception { + + Message tcr = new TradeCaptureReport(new TradeReportID("ABC1234"), new PreviouslyReported( + false), new LastQty(1000), new LastPx(5.6789), new TradeDate("20140101"), + new TransactTime(new Date())); + tcr.getHeader().setField(new SenderCompID("SENDER")); + tcr.getHeader().setField(new TargetCompID("TARGET")); + tcr.getHeader().setField(new MsgSeqNum(1)); + tcr.getHeader().setField(new SendingTime(new Date())); + tcr.setField(new Symbol("ABC")); + TradeCaptureReport.NoLegs leg1 = new TradeCaptureReport.NoLegs(); + leg1.setField(new LegSymbol("L1-XYZ")); + leg1.setField(new LegRefID("ABC1234-L1")); + leg1.setField(new LegQty(333)); + leg1.setField(new LegPrice(1.2345)); + TradeCaptureReport.NoLegs leg2 = new TradeCaptureReport.NoLegs(); + leg2.setField(new LegSymbol("L2-XYZ")); + leg2.setField(new LegRefID("ABC1234-L2")); + leg2.setField(new LegQty(777)); + leg2.setField(new LegPrice(2.3456)); + TradeCaptureReport.NoSides sides = new TradeCaptureReport.NoSides(); + sides.setField(new Side(Side.BUY)); + sides.setField(new OrderID("ID")); + + { + // will add a user-defined tag (i.e. greater than 5000) that is not defined in that group + Message m1 = new Message(); + m1.getHeader().setFields(tcr.getHeader()); + m1.setFields(tcr); + + leg1.setField(new StringField(10000, "TXT1")); // add unexpected tag to leg1 + m1.addGroup(leg1); + m1.addGroup(leg2); + m1.addGroup(sides); + + String s1 = m1.toString(); + DataDictionary dictionary = new DataDictionary(DataDictionaryTest.getDictionary()); + // parsing without validation should succeed + Message parsed1 = new Message(s1, dictionary, false); + + // validation should fail + int failingTag = 0; + try { + dictionary.validate(parsed1); + } catch (FieldException e) { + failingTag = e.getField(); + } + assertEquals(10000, failingTag); + + // but without checking user-defined fields, validation should succeed + dictionary.setCheckUserDefinedFields(false); + dictionary.validate(parsed1); + + assertEquals(s1, parsed1.toString()); + assertEquals(2, parsed1.getGroupCount(555)); + } + + { + // will add a normal tag that is not in the dictionary for that group + Message m2 = new Message(); + m2.getHeader().setFields(tcr.getHeader()); + m2.setFields(tcr); + + leg1.removeField(10000); // remove user-defined tag from before + leg1.setField(new Text("TXT1")); // add unexpected tag to leg1 + + m2.addGroup(leg1); + m2.addGroup(leg2); + m2.addGroup(sides); + + String s2 = m2.toString(); + DataDictionary dictionary = new DataDictionary(DataDictionaryTest.getDictionary()); + // parsing without validation should succeed + Message parsed2 = new Message(s2, dictionary, false); + + // validation should fail + int failingTag = 0; + try { + dictionary.validate(parsed2); + } catch (FieldException e) { + failingTag = e.getField(); + } + assertEquals(Text.FIELD, failingTag); + + // but without checking for unknown message fields, validation should succeed + dictionary.setAllowUnknownMessageFields(true); + dictionary.validate(parsed2); + + assertEquals(s2, parsed2.toString()); + assertEquals(2, parsed2.getGroupCount(555)); + } + } + + @Test + // QFJ-169 + public void testInvalidFieldInGroup() throws Exception { + SecurityRequestResult resultCode = new SecurityRequestResult( + SecurityRequestResult.NO_INSTRUMENTS_FOUND_THAT_MATCH_SELECTION_CRITERIA); + + UnderlyingSymbol underlyingSymbolField = new UnderlyingSymbol("UND"); + SecurityReqID id = new SecurityReqID("1234"); + + quickfix.fix44.DerivativeSecurityList responseMessage = new quickfix.fix44.DerivativeSecurityList(); + responseMessage.setField(id); + responseMessage.setField(underlyingSymbolField); + responseMessage.setField(new SecurityResponseID("2345")); + Group optionGroup = new quickfix.fix44.DerivativeSecurityList.NoRelatedSym(); + optionGroup.setField(new Symbol("OPT+RQ")); + optionGroup.setField(new StringField(StrikePrice.FIELD, "10")); + // add invalid field for this FIX version + optionGroup.setField(new QuoteAckStatus(0)); + optionGroup.setField(new PutOrCall(PutOrCall.CALL)); + optionGroup.setField(new MaturityMonthYear("200802")); + responseMessage.addGroup(optionGroup); + + Group group2 = new quickfix.fix44.DerivativeSecurityList.NoRelatedSym(); + group2.setField(new Symbol("OPT+RB")); + group2.setField(new StringField(StrikePrice.FIELD, "10")); + group2.setField(new MaturityMonthYear("200802")); + responseMessage.addGroup(group2); + resultCode.setValue(SecurityRequestResult.VALID_REQUEST); + responseMessage.setField(resultCode); + + DataDictionary dd = new DataDictionary(DataDictionaryTest.getDictionary()); + + int tagNo = 0; + try { + dd.validate(responseMessage, true); + } catch (FieldException e) { + tagNo = e.getField(); + } + // make sure that tag 297 is reported as invalid, NOT tag 55 + // (which is the first field after the invalid 297 field) + assertEquals(QuoteAckStatus.FIELD, tagNo); + + Message msg2 = new Message(responseMessage.toString(), dd); + try { + dd.validate(msg2, true); + } catch (FieldException e) { + tagNo = e.getField(); + } + // make sure that tag 297 is reported as invalid, NOT tag 55 + // (which is the first field after the invalid 297 field) + assertEquals(QuoteAckStatus.FIELD, tagNo); + + // parse message again without validation + msg2 = new Message(responseMessage.toString(), dd, false); + assertEquals(responseMessage.toString(), msg2.toString()); + Group noRelatedSymGroup = new quickfix.fix44.DerivativeSecurityList.NoRelatedSym(); + Group group = responseMessage.getGroup(1, noRelatedSymGroup); + assertTrue(group.isSetField(QuoteAckStatus.FIELD)); + + group = responseMessage.getGroup(2, noRelatedSymGroup); + assertFalse(group.isSetField(QuoteAckStatus.FIELD)); + } + private void assertHeaderField(Message message, String expectedValue, int field) throws FieldNotFound { assertEquals(expectedValue, message.getHeader().getString(field));