Merge branch '97' of github.com:gbirchmeier/quickfixn into 1.4.0beta

Conflicts:
	NEXT_VERSION.md

AcceptanceTest/Reflector.rb

@@ -29,7 +29,14 @@ def identifyMessage(message)
   def processFile(messages)
     lineNum = 0
     messages.each_line do
-      | line |
+      | fooline |
+
+      #utf = String.new(fooline)
+      #puts "UTF8: " + utf.force_encoding("UTF-8")
+
+      line = fooline.force_encoding("ASCII-8BIT")
+      #puts "ASCI: " + line
+
       lineNum += 1
       line.chomp!
       if line.empty? then

AcceptanceTest/Runner.rb

@@ -107,7 +107,7 @@ def createProcess(file, address, port)
   print "<at>\n"
   newarray.each do
     | v |
-    file = File.open(v, "r")
+    file = File.open(v, "rb")
     process = createProcess(file, ARGV[0], ARGV[1])
     if process.nil? then
       print "  <test name='", v,  "' result='", "failure' >\n"

AcceptanceTest/definitions/server/fix44/InternationalCharacters.def

@@ -0,0 +1,25 @@
+# International chars in test
+
+# for reference:
+# http://en.wikipedia.org/wiki/UTF-8
+# http://vim.wikia.com/wiki/Entering_special_characters
+# http://vimdoc.sourceforge.net/htmldoc/digraph.html
+
+iCONNECT
+# logon message and response
+I8=FIX.4.4
35=A
34=1
49=TW
52=<TIME>
56=ISLD
98=0
108=2

+E8=FIX.4.4
9=60
35=A
34=1
49=ISLD
52=00000000-00:00:00.000
56=TW
98=0
108=2
10=0

+
+# for reference:
+# http://en.wikipedia.org/wiki/UTF-8
+# http://vim.wikia.com/wiki/Entering_special_characters
+# http://vimdoc.sourceforge.net/htmldoc/digraph.html
+
+#Send news with intl chars and receive echo
+I8=FIX.4.4
35=B
34=2
49=TW
52=<TIME>
56=ISLD
347=UTF-8
148=ole!
359=olé!
33=0

+E8=FIX.4.4
9=83
35=B
34=2
49=ISLD
52=00000000-00:00:00.000
56=TW
347=UTF-8
33=0
148=ole!
359=olé!
10=69

+
+# logout message and response
+I8=FIX.4.4
35=5
34=3
49=TW
52=<TIME>
56=ISLD

+E8=FIX.4.4
9=49
35=5
34=3
49=ISLD
52=00000000-00:00:00.000
56=TW
10=0

+

NEXT_VERSION.md

@@ -24,4 +24,5 @@ Changes since the last version:
 * (patch) issue #134 - if DD field/group/component is missing "required" attribute, treat it as "required=N" (gbirchmeier)
 * (patch) issue #114 - enum dupe-check script; corrections to FIX43 tag 574/MatchType (gbirchmeier)
 * (patch) scripts and fixes for experimental Mono support (mgatny)
+* (minor) issue #97 - tolerance for non-ASCII (e.g. UTF-8) characters (andbjorn/gbirchmeier)
 

QuickFIXn/Fields/FieldBase.cs

@@ -72,8 +72,7 @@ public override int getLength()
         {
             if (_changed)
                 makeStringFields();
-
-            return _stringField.Length + 1; // +1 for SOH
+            return System.Text.Encoding.UTF8.GetByteCount(_stringField) + 1; // +1 for SOH
         }
 
         /// <summary>
@@ -85,9 +84,10 @@ public override int getTotal()
                 makeStringFields();
 
             int sum = 0;
-            foreach (char c in _stringField)
+            byte[] array = System.Text.Encoding.UTF8.GetBytes(_stringField);
+            foreach (byte b in array)
             {
-                sum += (int)c;
+                sum += b;
             }
             return (sum + 1); // +1 for SOH
         }

QuickFIXn/Parser.cs

@@ -1,17 +1,26 @@
-
 namespace QuickFix
 {
     /// <summary>
-    /// TODO use a byte[]/char[] for buffer_ instead of a string
     /// </summary>
     public class Parser
     {
-        private string buffer_ = "";
+        private byte[] buffer_ = new byte[512];
+        int usedBufferLength = 0;
+        public void AddToStream(ref byte[] data, int bytesAdded)
+        {
+            if (buffer_.Length < usedBufferLength + bytesAdded)
+                System.Array.Resize<byte>(ref buffer_, (usedBufferLength + bytesAdded));
+            System.Buffer.BlockCopy(data, 0, buffer_, usedBufferLength , bytesAdded);
+            usedBufferLength += bytesAdded;
+        }
 
         public void AddToStream(string data)
         {
-            buffer_ += data;
+            byte[] bytes = System.Text.Encoding.UTF8.GetBytes(data);
+            AddToStream(ref bytes, bytes.Length);
+
         }
+
 
         public bool ReadFixMessage(out string msg)
         {
@@ -21,12 +30,11 @@ public bool ReadFixMessage(out string msg)
                 return false;
 
             int pos = 0;
-
-            pos = buffer_.IndexOf("8=");
+            pos = IndexOf(buffer_, "8=", 0);
             if(-1 == pos)
                 return false;
 
-            buffer_ = buffer_.Remove(0, pos);
+            buffer_ = Remove(buffer_, pos);
             pos = 0;
 
             int length = 0;
@@ -39,58 +47,62 @@ public bool ReadFixMessage(out string msg)
                     return false;
 
                 msgBodyStart = pos;
-
                 pos += length;
                 if (buffer_.Length < pos)
                     return false;
-
-                pos = buffer_.IndexOf("\x01" + "10=", pos - 1);
+           
+                pos = IndexOf(buffer_, "\x01" + "10=", pos - 1);
                 if (-1 == pos)
                     return false;
                 msgBodyEnd = pos + 1;
                 pos += 4;
 
-                pos = buffer_.IndexOf("\x01", pos);
+                pos = IndexOf(buffer_, "\x01", pos);
                 if (-1 == pos)
                     return false;
                 pos += 1;
 
-                if ((msgBodyEnd - msgBodyStart) != length)
-                    throw new MessageParseError("Invalid body length");
+                if(length != (msgBodyEnd - msgBodyStart))
+                    throw new MessageParseError("Invalid body length. Calculated:" + (msgBodyEnd - msgBodyStart) + " expected:" + length);
 
-                msg = buffer_.Substring(0, pos);
-                buffer_ = buffer_.Remove(0, pos);
+                msg = Substring(buffer_, 0, pos);
+                buffer_ = Remove(buffer_, pos);
                 return true;
             }
             catch (MessageParseError e)
             {
                 if ((length > 0) && (pos <= buffer_.Length))
-                    buffer_ = buffer_.Remove(0, pos);
+                    buffer_ = Remove(buffer_, pos);
                 else
-                    buffer_ = buffer_.Remove(0, buffer_.Length);
+                    buffer_ = Remove(buffer_, buffer_.Length);
                 throw e;
             }
         }
 
         public bool ExtractLength(out int length, out int pos, string buf)
         {
+            return ExtractLength(out length, out pos, System.Text.Encoding.UTF8.GetBytes(buf));
+        }
+
+        public bool ExtractLength(out int length, out int pos, byte[] buf)
+        {
+            string bufferString = System.Text.Encoding.UTF8.GetString(buf);
             length = 0;
             pos = 0;
 
             if (buf.Length < 1)
                 return false;
 
-            int startPos = buf.IndexOf("\x01" + "9=", 0);
+            int startPos = IndexOf(buf, "\x01" + "9=", 0);
             if(-1 == startPos)
                 return false;
             startPos +=3;
 
-            int endPos = buf.IndexOf("\x01", startPos);
+            int endPos = IndexOf(buf, "\x01", startPos);
             if(-1 == endPos)
                 return false;
 
-            string strLength = buf.Substring(startPos, endPos - startPos);
-
+            string strLength = Substring(buf, startPos, endPos - startPos);
             try
             {
                 length = Fields.Converters.IntConverter.Convert(strLength);
@@ -111,5 +123,43 @@ private bool Fail(string what)
             System.Console.WriteLine("Parser failed: " + what);
             return false;
         }
+
+        private int IndexOf(byte[] arrayToSearchThrough, string stringPatternToFind, int offset)
+        {
+            byte[] patternToFind = System.Text.Encoding.UTF8.GetBytes(stringPatternToFind);
+            if (patternToFind.Length > arrayToSearchThrough.Length)
+                return -1;
+            for (int i = offset; i <= arrayToSearchThrough.Length - patternToFind.Length; i++)
+            {
+                bool found = true;
+                for (int j = 0; j < patternToFind.Length; j++)
+                {
+                    if (arrayToSearchThrough[i + j] != patternToFind[j])
+                    {
+                        found = false;
+                        break;
+                    }
+                }
+                if (found)
+                {
+                    return i;
+                }
+            }
+            return -1;
+        }
+        private byte[] Remove(byte[] array, int count)
+        {
+            byte[] returnByte = new byte[array.Length - count];
+            System.Buffer.BlockCopy(array, count, returnByte,0, array.Length - count);
+            usedBufferLength -= count;
+            return returnByte;
+        }
+
+        private string Substring(byte[] array, int startIndex, int length)
+        {
+            byte[] returnByte = new byte[length];
+            System.Buffer.BlockCopy(array, startIndex, returnByte, 0, length);
+            return System.Text.Encoding.UTF8.GetString(returnByte);
+        }
     }
 }

QuickFIXn/SocketInitiatorThread.cs

@@ -66,7 +66,7 @@ public bool Read()
                     int bytesRead = socket_.Receive(readBuffer_);
                     if (0 == bytesRead)
                         throw new SocketException(System.Convert.ToInt32(SocketError.ConnectionReset));
-                    parser_.AddToStream(System.Text.Encoding.UTF8.GetString(readBuffer_, 0, bytesRead));
+                    parser_.AddToStream(ref readBuffer_, bytesRead);
                 }
                 else if (null != session_)
                 {

QuickFIXn/SocketReader.cs

@@ -30,7 +30,7 @@ public void Read()
                     int bytesRead = tcpClient_.Client.Receive(readBuffer_);
                     if (bytesRead < 1)
                         throw new SocketException(System.Convert.ToInt32(SocketError.ConnectionReset));
-                    parser_.AddToStream(System.Text.Encoding.UTF8.GetString(readBuffer_, 0, bytesRead));
+                    parser_.AddToStream(ref readBuffer_, bytesRead);
                 }
                 else if (null != qfSession_)
                 {

UnitTests/FieldTests.cs

@@ -102,6 +102,15 @@ public void StringFieldTest_TotalAndLength()
             Assert.That(obj.getLength(), Is.EqualTo(9));
         }
 
+        [Test]
+        public void StringFieldTest_NonAscii()
+        {
+            // technically, non-ascii shouldn't be in a StringField, but sometimes it happens, so let's not freak out.
+            StringField obj = new StringField(359, "olé!");
+            Assert.AreEqual(839, obj.getTotal()); // sum of all bytes in "359=olé!"+nul
+            Assert.AreEqual(10, obj.getLength()); // 7 single-byte chars + 1 double=byte char + nul = 10 bytes
+        }
+
         [Test]
         public void DefaultValTest()
         {

UnitTests/ParserTest.cs

@@ -1,5 +1,6 @@
 using NUnit.Framework;
 using QuickFix;
+using System;
 
 namespace UnitTests
 {
@@ -92,5 +93,29 @@ public void ReadFixMessageWithBadLength()
             // nothing thrown now because the previous call removes bad data from buffer:
             Assert.DoesNotThrow(delegate { parser.ReadFixMessage(out readFixMsg); });
         }
+
+        [Test]
+        public void ReadFixMessageWithNonAscii()
+        {
+            string[] fixMsgFields1 = { "8=FIX.4.4", "9=19", "35=B", "148=Ole!", "33=0", "10=0" };
+            string fixMsg1 = String.Join("\x01", fixMsgFields1) + "\x01";
+
+            Assert.AreEqual("é", "\x00E9");
+            Assert.AreEqual("é", "\xE9");
+
+            string[] fixMsgFields2 = { "8=FIX.4.4", "9=20", "35=B", "148=Olé!", "33=0", "10=0" };
+            string fixMsg2 = String.Join("\x01", fixMsgFields2) + "\x01";
+
+            Parser parser = new Parser();
+            parser.AddToStream(fixMsg1 + fixMsg2);
+
+            string readFixMsg1;
+            Assert.True(parser.ReadFixMessage(out readFixMsg1));
+            Assert.AreEqual(fixMsg1, readFixMsg1);
+
+            string readFixMsg2;
+            Assert.True(parser.ReadFixMessage(out readFixMsg2));
+            Assert.AreEqual(fixMsg2, readFixMsg2);
+        }
     }
 }

web/views/about/credits.md

@@ -27,3 +27,4 @@ Contributors
 - Matt Wood
 - Laszlo Ligart
 - Ervin Marguc
+- Bjorn Andersson