Merge pull request #38 from glenebob/native-backend-convert-2

Big rewrite and optimization of code which converts data from Npgsql to Postgresql. Native to backend conversion code.

Fixed BasicNativeToBackendTypeConverter.ToBinary() (broken in commit d2b636a): Reverted back to the correct "\" when escaping byte values.
Added class BackendToNativeTypeConverterOptions to describe UseConformantStrings, Supports_E_StringPrefix and SupportsHexByteFormat. Maintain one on each connector for use during encoding.
Push E string prefix and escaping logic down the conversion stack so that array elements can use the same logic.
Only escape \ if the backend is expecting non conformant strings.
Implement string escapes in a function rather than using multiple string.Replace() calls.
Do all bytea string escaping in ToBinary(), and then do not escape its output again.
Encode bytea data in the new hex format when possible (>= 9.0), which cuts some such strings in half.
Don't use the E prefix if the backend doesn't understand it.
Assume non conformant strings on all version 2 connections, since we can't track changes to the setting. The E prefix causes this to work if the backend is expecting conformant strings.
Expose UseConformantStrings, Supports_E_StringPrefix and SupportsHexByteFormat via NpgsqlConnection.

src/Npgsql/NpgsqlCommand.cs

@@ -88,7 +88,6 @@ public sealed class NpgsqlCommand : DbCommand, ICloneable
 
         private UpdateRowSource updateRowSource = UpdateRowSource.Both;
 
-
         // Constructors
 
         /// <summary>
@@ -672,7 +671,7 @@ private void BindParameters()
                     // TODO: Add binary format support for all supported types. Not only bytea.
                     if (parameters[i].TypeInfo.NpgsqlDbType != NpgsqlDbType.Bytea)
                     {
-                        parameterValues[i] = parameters[i].TypeInfo.ConvertToBackend(parameters[i].Value, true);
+                        parameterValues[i] = parameters[i].TypeInfo.ConvertToBackend(parameters[i].Value, true, Connector.NativeToBackendTypeConverterOptions);
                     }
                     else
                     {
@@ -683,7 +682,7 @@ private void BindParameters()
                         }
                         else
                         {
-                            parameterValues[i] = parameters[i].TypeInfo.ConvertToBackend(parameters[i].Value, true);
+                            parameterValues[i] = parameters[i].TypeInfo.ConvertToBackend(parameters[i].Value, true, Connector.NativeToBackendTypeConverterOptions);
                         }
                     }
                 }
@@ -903,79 +902,30 @@ internal StringBuilder GetCommandText()
             // queries.  Reset command timeout and SQL sent.
             m_Connector.Mediator.ResetResponses();
             m_Connector.Mediator.CommandTimeout = CommandTimeout;
+
             return ret;
 
         }
 
-        private static void PassEscapedArray(StringBuilder query, string array)
-        {
-            bool inTextLiteral = false;
-            int endAt = array.Length - 1;//leave last char for separate append as we don't have to continually check we're safe to add the next char too.
-            for(int i = 0; i != endAt; ++i)
-            {
-                if(array[i] == '\'')
-                {
-                    if(!inTextLiteral)
-                    {
-                        query.Append("E'");
-                        inTextLiteral = true;
-                    }
-                    else if(array[i + 1] == '\'')//SQL-escaped '
-                    {
-                        query.Append("''");
-                        ++i;
-                    }
-                    else
-                    {
-                        query.Append('\'');
-                        inTextLiteral = false;
-                    }
-                }
-                else
-                    query.Append(array[i]);
-            }
-            query.Append(array[endAt]);
-        }
-
         private void PassParam(StringBuilder query, NpgsqlParameter p)
         {
-                string serialised = p.TypeInfo.ConvertToBackend(p.Value, false);
-
-                // Add parentheses wrapping parameter value before the type cast to avoid problems with Int16.MinValue, Int32.MinValue and Int64.MinValue
-                // See bug #1010543
-                // Check if this parenthesis can be collapsed with the previous one about the array support. This way, we could use
-                // only one pair of parentheses for the two purposes instead of two pairs.
-                query.Append('(');
-
-                if(Connector.UseConformantStrings)
-                    switch(serialised[0])
-                    {
-                    case '\''://type passed as string or string with type.
-                        //We could test to see if \ is used anywhere, but then we could be doing quite an expensive check (if the value is large) for little gain.
-                        query.Append("E").Append(serialised);
-                        break;
-                    case 'a':
-                        if(POSTGRES_TEXT_ARRAY.IsMatch(serialised))
-                            PassEscapedArray(query, serialised);
-                        else
-                            query.Append(serialised);
-                        break;
-                    default:
-                        query.Append(serialised);
-                        break;
-                    }
-                else
-                    query.Append(serialised);
+            string serialised = p.TypeInfo.ConvertToBackend(p.Value, false, Connector.NativeToBackendTypeConverterOptions);
 
-                query.Append(')');
+            // Add parentheses wrapping parameter value before the type cast to avoid problems with Int16.MinValue, Int32.MinValue and Int64.MinValue
+            // See bug #1010543
+            // Check if this parenthesis can be collapsed with the previous one about the array support. This way, we could use
+            // only one pair of parentheses for the two purposes instead of two pairs.
+            query.AppendFormat("({0})", serialised);
 
+            if (p.UseCast)
+            {
+                query.AppendFormat("::{0}", p.TypeInfo.CastName);
 
-                if (p.UseCast)
+                if (p.TypeInfo.UseSize && (p.Size > 0))
                 {
-                    query.Append("::").Append(p.TypeInfo.CastName);
-                    if (p.TypeInfo.UseSize && (p.Size > 0))
-                        query.Append('(').Append(p.Size).Append(')');
+                    query.AppendFormat("({0})", p.Size);
                 }
+            }
         }
 
         private StringBuilder GetClearCommandText()
@@ -1238,7 +1188,7 @@ private StringBuilder GetPreparedCommandText()
                     // Add parentheses wrapping parameter value before the type cast to avoid problems with Int16.MinValue, Int32.MinValue and Int64.MinValue
                     // See bug #1010543
                     result.Append('(');
-                    result.Append(p.TypeInfo.ConvertToBackend(p.Value, false));
+                    result.Append(p.TypeInfo.ConvertToBackend(p.Value, false, Connector.NativeToBackendTypeConverterOptions));
                     result.Append(')');
                     if (p.UseCast)
                     {

src/Npgsql/NpgsqlConnection.cs

@@ -474,7 +474,46 @@ public Int32 ProcessID
 				CheckConnectionOpen();
 				return connector.BackEndKeyData.ProcessID;
 			}
-		}
+		}
+
+        /// <summary>
+        /// Report whether the backend is expecting standards conforming strings..
+        /// </summary>
+        [Browsable(false)]
+        public Boolean UseConformantStrings
+        {
+            get
+            {
+                CheckConnectionOpen();
+                return connector.NativeToBackendTypeConverterOptions.UseConformantStrings;
+            }
+        }
+
+        /// <summary>
+        /// Report whether the backend understands the string literal E prefix (>= 8.2).
+        /// </summary>
+        [Browsable(false)]
+        public Boolean Supports_E_StringPrefix
+        {
+            get
+            {
+                CheckConnectionOpen();
+                return connector.NativeToBackendTypeConverterOptions.Supports_E_StringPrefix;
+            }
+        }
+
+        /// <summary>
+        /// Report whether the backend understands the hex byte format (>= 9.0).
+        /// </summary>
+        [Browsable(false)]
+        public Boolean SupportsHexByteFormat
+        {
+            get
+            {
+                CheckConnectionOpen();
+                return connector.NativeToBackendTypeConverterOptions.SupportsHexByteFormat;
+            }
+        }
 
 		/// <summary>
 		/// Begins a database transaction with the specified isolation level.

src/Npgsql/NpgsqlConnector.cs

@@ -142,6 +142,8 @@ internal class NpgsqlConnector
         private const String _planNamePrefix = "npgsqlplan";
         private const String _portalNamePrefix = "npgsqlportal";
 
+        private NativeToBackendTypeConverterOptions _NativeToBackendTypeConverterOptions = NativeToBackendTypeConverterOptions.Default;
+
 
         private Thread _notificationThread;
 
@@ -159,15 +161,11 @@ internal class NpgsqlConnector
         // expect a ready_for_query response.
         private bool _requireReadyForQuery = true;
 
-        private bool? _useConformantStrings;
-
         private readonly Dictionary<string, NpgsqlParameterStatus> _serverParameters =
             new Dictionary<string, NpgsqlParameterStatus>(StringComparer.InvariantCultureIgnoreCase);
 
         // For IsValid test
         private readonly RNGCryptoServiceProvider rng = new RNGCryptoServiceProvider();
-
-
 
 #if WINDOWS && UNMANAGED
 
@@ -407,6 +405,7 @@ internal Boolean IsValid()
 
             return true;
         }
+
         /// <summary>
         /// This method is responsible for releasing all resources associated with this Connector.
         /// </summary>
@@ -674,10 +673,19 @@ internal Boolean SupportsSavepoint
         {
             get { return _supportsSavepoint; }
             set { _supportsSavepoint = value; } 
-
         }
 
-
+        /// <summary>
+        /// Options that control certain aspects of native to backend conversions that depend
+        /// on backend version and status.
+        /// </summary>
+        public NativeToBackendTypeConverterOptions NativeToBackendTypeConverterOptions
+        {
+            get
+            {
+                return _NativeToBackendTypeConverterOptions;
+            }
+        }
 
         /// <summary>
         /// This method is required to set all the version dependent features flags.
@@ -688,6 +696,12 @@ internal void ProcessServerVersion()
         {
             this._supportsPrepare = (ServerVersion >= new Version(7, 3, 0));
             this._supportsSavepoint = (ServerVersion >= new Version(8, 0, 0));
+
+            // Per the PG documentation, E string literal prefix support appeared in PG version 8.1.
+            NativeToBackendTypeConverterOptions.Supports_E_StringPrefix = (ServerVersion >= new Version(8, 1, 0));
+
+            // Per the PG documentation, hex string encoding format support appeared in PG version 9.0.
+            NativeToBackendTypeConverterOptions.SupportsHexByteFormat = (ServerVersion >= new Version(9, 0, 0));
         }
 
         /*/// <value>Counts the numbers of Connections that share
@@ -1110,77 +1124,16 @@ public void AddParameterStatus(NpgsqlParameterStatus ps)
             {
                 _serverParameters.Add(ps.Parameter, ps);
             }
-            _useConformantStrings = null;
-        }
 
-        public IDictionary<string, NpgsqlParameterStatus> ServerParameters
-        {
-            get { return new ReadOnlyDictionary<string, NpgsqlParameterStatus>(_serverParameters); }
-        }
-        public string CheckParameter(string paramName)
-        {
-            NpgsqlParameterStatus ps = null;
-            if(_serverParameters.TryGetValue(paramName, out ps))
-                return ps.ParameterValue;
-            try
+            if (ps.Parameter == "standard_conforming_strings")
             {
-                using(NpgsqlCommand cmd = new NpgsqlCommand("show " + paramName, this))
-                {
-                    string paramValue = (string)cmd.ExecuteScalar();
-                    AddParameterStatus(new NpgsqlParameterStatus(paramName, paramValue));
-                    return paramValue;
-                }
+                NativeToBackendTypeConverterOptions.UseConformantStrings = (ps.ParameterValue == "on");
             }
-
-            /*
-             * In case of problems with the command above, we simply return null in order to 
-             * say we don't support it.
-             */
-
-            catch(NpgsqlException)
-            {
-                return null;
-            }
-            /*
-             * Original catch handler by Jon Hanna.
-             * 7.3 version doesn't support error code. Only 7.4+
-             
-             * catch(NpgsqlException ne)
-            {
-                if(ne.Code == "42704")//unrecognized configuration parameter
-                    return null;
-                else
-                    throw;
-            }*/
-        }
-        private bool CheckStringConformanceRequirements()
-        {
-            //If standards_conforming_strings is "on" then postgres will handle \ in strings differently to how we expect, unless
-            //an escaped-string literal is use (E'example\n' rather than 'example\n'. At time of writing this defaults
-            //to "off", but documentation says this will change to default "on" in the future, and it can still be "on" now.
-            //escape_string_warning being "on" (current default) will result in a warning, but not an error, on every such
-            //string, which is not ideal.
-            //On the other hand, earlier versions of postgres do not have the escaped-literal syntax and will error if it
-            //is used. Version numbers could be checked, but here the approach taken is to check on what the server is
-            //explicitly requesting.
-
-            NpgsqlParameterStatus warning = null;
-            if(_serverParameters.TryGetValue("escape_string_warning", out warning) && warning.ParameterValue == "on")//try the most commonly set at time of coding first
-                return true;
-            NpgsqlParameterStatus insist = null;
-            if(_serverParameters.TryGetValue("standard_conforming_strings", out insist) && insist.ParameterValue == "on")
-                return true;
-            if(warning != null && insist != null)//both where present and "off".
-                return false;
-            //We need to check at least one of these on the server.
-            return CheckParameter("standard_conforming_strings") == "on" || CheckParameter("escape_string_warning") == "on";
         }
-        public bool UseConformantStrings
+
+        public IDictionary<string, NpgsqlParameterStatus> ServerParameters
         {
-            get
-            {
-                return _useConformantStrings ?? (_useConformantStrings = CheckStringConformanceRequirements()).Value;
-            }
+            get { return new ReadOnlyDictionary<string, NpgsqlParameterStatus>(_serverParameters); }
         }
     }
 }

src/Npgsql/SqlGenerators/VisitedExpression.cs

@@ -145,8 +145,9 @@ internal override void WriteSql(StringBuilder sqlText)
                     // Escape syntax is needed for strings with escape values.
                     // We don't check if there are escaped strings for performance reasons.
                     // Check https://github.com/franciscojunior/Npgsql2/pull/10 for more info.
-                    sqlText.Append('E'); 
-                    sqlText.Append(typeInfo.ConvertToBackend(_value, false));
+                    // NativeToBackendTypeConverterOptions.Default should provide the correct
+                    // formatting rules for any backend >= 8.0.
+                    sqlText.Append(typeInfo.ConvertToBackend(_value, false, NativeToBackendTypeConverterOptions.Default));
                     break;
                 case PrimitiveTypeKind.Time:
                     sqlText.AppendFormat(ni, "TIME '{0:T}'", _value);