FIX: Contains filter on lookup column returned too many values

backend/Origam.DA.Service/Generators/AbstractSqlCommandGenerator.cs

@@ -27,6 +27,7 @@
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
+using System.Threading;
 using Origam.DA.Service.CustomCommandParser;
 using Origam.DA.Service.Generators;
 using Origam.Extensions;
@@ -1858,21 +1859,9 @@ private record GroupByData(DataStructureColumn Column, string Expression);
             string orderByExpression="";
             if (concatScalarColumns && columnsInfo != null && columnsInfo.Count > 1)
             {
-                List<ColumnRenderItem> columnRenderData = new List<ColumnRenderItem>();
-                foreach (string columnName in columnsInfo.ColumnNames)
-                {
-                    DataStructureColumn column = entity.Column(columnName);
-                    columnRenderData.Add(
-                        new ColumnRenderItem
-                        {
-                            SchemaItem = column.Field,
-                            Entity = column.Entity ?? entity,
-                            RenderSqlForDetachedFields = columnsInfo.RenderSqlForDetachedFields
-                        });
-                }
                 sqlExpression.Append(" ");
-                sqlExpression.Append(RenderConcat(columnRenderData, sqlValueFormatter.RenderString(", "),
-                    replaceParameterTexts, dynamicParameters, selectParameterReferences));
+                sqlExpression.Append(RenderConcat(selectParameters, isInRecursion, forceDatabaseCalculation, sqlValueFormatter.RenderString(", "),
+                    replaceParameterTexts, dynamicParameters, selectParameterReferences, filterCommandParser, orderByCommandParser));
                 return false;
             }
             i = 0;
@@ -3986,9 +3975,66 @@ internal string RenderConcat(ArrayList concatSchemaItems, DataStructureEntity en
                 concatBuilder.Append(")");
             }
             return concatBuilder.ToString();
+        } 
+
+        internal string RenderConcat(SelectParameters selectParameters, bool isInRecursion, bool forceDatabaseCalculation,
+            string separator, Hashtable replaceParameterTexts, Hashtable dynamicParameters, Hashtable parameterReferences,
+            FilterCommandParser filterCommandParser,
+            OrderByCommandParser orderByCommandParser)
+        {
+            int i = 0;
+            StringBuilder concatBuilder = new StringBuilder();
+            bool shouldTrimSeparator = !string.IsNullOrEmpty(separator) &&
+                                       selectParameters.ColumnsInfo.ColumnNames.Count > 1;
+            if (shouldTrimSeparator)
+            {
+                concatBuilder.Append($"TRIM( {separator} FROM ");
+            }
+            SortedList<int, SortOrder> order = new SortedList<int, SortOrder>();
+            foreach (string columnName in selectParameters.ColumnsInfo.ColumnNames)
+            {
+                DataStructureColumn column = selectParameters.Entity.Column(columnName);
+                if (i > 0)
+                {
+                    concatBuilder.Append(" " + sqlRenderer.StringConcatenationChar + " ");
+                    if (separator != null)
+                    {
+                        concatBuilder.Append(separator);
+                        concatBuilder.Append(" " + sqlRenderer.StringConcatenationChar + " ");
+                    }
+                }
+                string groupByExpression = "";
+                bool groupByNeeded = false;
+                LookupOrderingInfo customOrderingInfo =
+                    LookupOrderingInfo.TryCreate(selectParameters.CustomOrderings.Orderings, column.Name);
+                ColumnRenderData columnRenderData = RenderDataStructureColumn(
+                    selectParameters.DataStructure,
+                    selectParameters.Entity,
+                    replaceParameterTexts, dynamicParameters,
+                    selectParameters.SortSet, parameterReferences, isInRecursion,
+                    forceDatabaseCalculation, ref groupByExpression, order, ref groupByNeeded,
+                    selectParameters.ColumnsInfo ?? ColumnsInfo.Empty, column,
+                    customOrderingInfo, filterCommandParser, orderByCommandParser, 
+                    selectParameters.RowOffset);
+                if (column.DataType == OrigamDataType.Date)
+                {
+                    string nonNullExpression = $"{sqlRenderer.IsNull()} ({sqlRenderer.Format(columnRenderData.Expression, Thread.CurrentThread.CurrentCulture.Name)}, '')";
+                    concatBuilder.Append(nonNullExpression);
+                }
+                else
+                {
+                    string nonNullExpression = $"{sqlRenderer.IsNull()} ({columnRenderData.Expression}, '')";
+                    concatBuilder.Append(nonNullExpression);
+                }
+                i++;
+            }
+            if (shouldTrimSeparator)
+            {
+                concatBuilder.Append(")");
+            }
+            return concatBuilder.ToString();
         }
 
-
         internal string PostProcessCustomCommandParserWhereClauseSegment(
             string input, DataStructureEntity entity,
             Hashtable replaceParameterTexts, Hashtable dynamicParameters, 
@@ -4145,6 +4191,7 @@ internal class ColumnRenderItem
     {
         public bool RenderSqlForDetachedFields { get; set; }
         public ISchemaItem SchemaItem { get; set; }
+        public DataStructureColumn Column { get; set; }
         public DataStructureEntity Entity { get; set; }
     }
 

backend/Origam.DA.Service/Generators/MsSqlRenderer.cs

@@ -59,6 +59,12 @@ internal override string IsNull()
     {
         return "ISNULL";
     }
+
+    internal override string Format(string date, string culture)
+    {
+        return @$" FORMAT({date}, IIF (FORMAT({date}, 'HH:mm:ss tt', 'en-US' ) = '00:00:00 AM', 'd', ''), '{culture}') ";
+    }
+
     internal override string CountAggregate()
     {
         return "COUNT_BIG";

backend/Origam.DA.Service/Generators/PgSqlRenderer.cs

@@ -233,6 +233,12 @@ internal override string IsNull()
     {
         return "COALESCE";
     }
+
+    internal override string Format(string date, string culture)
+    {
+        throw new NotImplementedException();
+    }
+
     internal override string CountAggregate()
     {
         return "COUNT";

backend/Origam.DA.Service/Generators/SqlRenderer.cs

@@ -35,6 +35,11 @@ public abstract class SqlRenderer
     public abstract string ConvertGeoToTextClause(string argument);
     internal abstract string Sequence(string entityName, string primaryKeyName);
     internal abstract string IsNull();
+
+    // Will omit the time part if the time is 00:00:00.
+    // The result should be the same as the the result of C# DateTime.ToString() / DateTime.ToShortDateString()
+    // Because that is what is used in C# code in the UIService/GetLookupLabelsEx endpoint
+    internal abstract string Format(string date, string culture);
     internal abstract string CountAggregate();
     internal abstract string DeclareAsSql();
     internal abstract string FunctionPrefix();